@budibase/server 2.5.9 → 2.5.10-alpha.1

package/src/api/controllers/application.ts

@@ -1,5 +1,4 @@
 import env from "../../environment"
-import packageJson from "../../../package.json"
 import {
   createLinkView,
   createRoutingView,
@@ -24,6 +23,7 @@ import {
   migrations,
   objectStore,
   ErrorCode,
+  env as envCore,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
@@ -223,7 +223,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
   )
 
   ctx.body = {
-    application,
+    application: { ...application, upgradableVersion: envCore.VERSION },
     screens,
     layouts,
     clientLibPath,
@@ -264,7 +264,7 @@ async function performAppCreate(ctx: UserCtx) {
       _rev: undefined,
       appId,
       type: "app",
-      version: packageJson.version,
+      version: envCore.VERSION,
       componentLibraries: ["@budibase/standard-components"],
       name: name,
       url: url,
@@ -433,7 +433,7 @@ export async function updateClient(ctx: UserCtx) {
   }
 
   // Update versions in app package
-  const updatedToVersion = packageJson.version
+  const updatedToVersion = envCore.VERSION
   const appPackageUpdates = {
     version: updatedToVersion,
     revertableVersion: currentVersion,

package/src/api/controllers/automation.ts

@@ -16,9 +16,15 @@ import { setTestFlag, clearTestFlag } from "../../utilities/redis"
 import { context, cache, events } from "@budibase/backend-core"
 import { automations } from "@budibase/pro"
 import { Automation, BBContext } from "@budibase/types"
+import { getActionDefinitions as actionDefs } from "../../automations/actions"
 
-const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
-const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+async function getActionDefinitions() {
+  return removeDeprecated(await actionDefs())
+}
+
+function getTriggerDefinitions() {
+  return removeDeprecated(triggers.TRIGGER_DEFINITIONS)
+}
 
 /*************************
  *                       *
@@ -228,17 +234,17 @@ export async function clearLogError(ctx: BBContext) {
 }
 
 export async function getActionList(ctx: BBContext) {
-  ctx.body = ACTION_DEFS
+  ctx.body = await getActionDefinitions()
 }
 
 export async function getTriggerList(ctx: BBContext) {
-  ctx.body = TRIGGER_DEFS
+  ctx.body = getTriggerDefinitions()
 }
 
 export async function getDefinitionList(ctx: BBContext) {
   ctx.body = {
-    trigger: TRIGGER_DEFS,
-    action: ACTION_DEFS,
+    trigger: getTriggerDefinitions(),
+    action: await getActionDefinitions(),
   }
 }
 

package/src/api/controllers/datasource.ts

@@ -12,7 +12,15 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { UserCtx, Datasource, Row } from "@budibase/types"
+import {
+  UserCtx,
+  Datasource,
+  Row,
+  CreateDatasourceResponse,
+  UpdateDatasourceResponse,
+  UpdateDatasourceRequest,
+  CreateDatasourceRequest,
+} from "@budibase/types"
 import sdk from "../../sdk"
 
 export async function fetch(ctx: UserCtx) {
@@ -146,7 +154,7 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: UserCtx) {
+export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
   let datasource = await sdk.datasources.get(datasourceId)
@@ -187,15 +195,17 @@ export async function update(ctx: UserCtx) {
   }
 }
 
-export async function save(ctx: UserCtx) {
+export async function save(
+  ctx: UserCtx<CreateDatasourceRequest, CreateDatasourceResponse>
+) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
 
   const datasource = {
     _id: generateDatasourceID({ plus }),
-    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
     ...ctx.request.body.datasource,
+    type: plus ? DocumentType.DATASOURCE_PLUS : DocumentType.DATASOURCE,
   }
 
   let schemaError = null
@@ -218,7 +228,7 @@ export async function save(ctx: UserCtx) {
     }
   }
 
-  const response: any = {
+  const response: CreateDatasourceResponse = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
   if (schemaError) {

package/src/api/controllers/dev.ts

@@ -4,7 +4,7 @@ import { checkSlashesInUrl } from "../../utilities"
 import { request } from "../../utilities/workerRequests"
 import { clearLock as redisClearLock } from "../../utilities/redis"
 import { DocumentType } from "../../db/utils"
-import { context } from "@budibase/backend-core"
+import { context, env as envCore } from "@budibase/backend-core"
 import { events, db as dbCore, cache } from "@budibase/backend-core"
 
 async function redirect(ctx: any, method: string, path: string = "global") {
@@ -121,7 +121,7 @@ export async function revert(ctx: any) {
 }
 
 export async function getBudibaseVersion(ctx: any) {
-  const version = require("../../../package.json").version
+  const version = envCore.VERSION
   ctx.body = {
     version,
   }

package/src/api/controllers/ops.ts

@@ -0,0 +1,32 @@
+import { Ctx } from "@budibase/types"
+import { logging } from "@budibase/backend-core"
+
+interface LogRequest {
+  message: string
+  data?: any
+}
+
+interface ErrorRequest {
+  message: string
+}
+
+export async function log(ctx: Ctx<LogRequest>) {
+  const body = ctx.request.body
+  console.trace(body.message, body.data)
+  console.debug(body.message, body.data)
+  console.info(body.message, body.data)
+  console.warn(body.message, body.data)
+  console.error(body.message, body.data)
+  ctx.status = 204
+}
+
+export async function alert(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  logging.logAlert(body.message, new Error(body.message))
+  ctx.status = 204
+}
+
+export async function error(ctx: Ctx<ErrorRequest>) {
+  const body = ctx.request.body
+  throw new Error(body.message)
+}

package/src/api/controllers/plugin/index.ts

@@ -1,31 +1,11 @@
-import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
-import {
-  plugins as pluginCore,
-  db as dbCore,
-  tenancy,
-  objectStore,
-} from "@budibase/backend-core"
-import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types"
+import { npmUpload, urlUpload, githubUpload } from "./uploaders"
+import { plugins as pluginCore } from "@budibase/backend-core"
+import { PluginType, FileType, PluginSource } from "@budibase/types"
 import env from "../../../environment"
-import { ClientAppSocket } from "../../../websocket"
+import { clientAppSocket } from "../../../websockets"
+import sdk from "../../../sdk"
 import { sdk as pro } from "@budibase/pro"
 
-export async function getPlugins(type?: PluginType) {
-  const db = tenancy.getGlobalDB()
-  const response = await db.allDocs(
-    dbCore.getPluginParams(null, {
-      include_docs: true,
-    })
-  )
-  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
-  plugins = objectStore.enrichPluginURLs(plugins)
-  if (type) {
-    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
-  } else {
-    return plugins
-  }
-}
-
 export async function upload(ctx: any) {
   const plugins: FileType[] =
     ctx.request.files.file.length > 1
@@ -35,7 +15,7 @@ export async function upload(ctx: any) {
     let docs = []
     // can do single or multiple plugins
     for (let plugin of plugins) {
-      const doc = await processUploadedPlugin(plugin, PluginSource.FILE)
+      const doc = await sdk.plugins.processUploaded(plugin, PluginSource.FILE)
       docs.push(doc)
     }
     ctx.body = {
@@ -91,7 +71,7 @@ export async function create(ctx: any) {
 
     const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
-    ClientAppSocket.emit("plugins-update", { name, hash: doc.hash })
+    clientAppSocket.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
       message: "Plugin uploaded successfully",
       plugins: [doc],
@@ -105,7 +85,7 @@ export async function create(ctx: any) {
 }
 
 export async function fetch(ctx: any) {
-  ctx.body = await getPlugins()
+  ctx.body = await sdk.plugins.fetch()
 }
 
 export async function destroy(ctx: any) {
@@ -119,20 +99,3 @@ export async function destroy(ctx: any) {
     ctx.throw(400, err.message)
   }
 }
-
-export async function processUploadedPlugin(
-  plugin: FileType,
-  source?: PluginSource
-) {
-  const { metadata, directory } = await fileUpload(plugin)
-  pluginCore.validate(metadata?.schema)
-
-  // Only allow components in cloud
-  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
-    throw new Error("Only component plugins are supported outside of self-host")
-  }
-
-  const doc = await pro.plugins.storePlugin(metadata, directory, source)
-  ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
-  return doc
-}

package/src/api/controllers/query/index.ts

@@ -245,7 +245,7 @@ async function execute(
     }
     const runFn = () => Runner.run(inputs)
 
-    const { rows, pagination, extra } = await quotas.addQuery(runFn, {
+    const { rows, pagination, extra, info } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
     })
     // remove the raw from execution incase transformer being used to hide data
@@ -255,7 +255,7 @@ async function execute(
     if (opts && opts.rowsOnly) {
       ctx.body = rows
     } else {
-      ctx.body = { data: rows, pagination, ...extra }
+      ctx.body = { data: rows, pagination, ...extra, ...info }
     }
   } catch (err) {
     ctx.throw(400, err)

package/src/api/controllers/row/ExternalRequest.ts

@@ -1,31 +1,32 @@
 import {
+  Datasource,
+  FieldSchema,
+  FieldType,
   FilterType,
   IncludeRelationship,
   Operation,
   PaginationJson,
   RelationshipsJson,
+  RelationshipTypes,
+  Row,
   SearchFilters,
   SortJson,
-  Datasource,
-  FieldSchema,
-  Row,
-  Table,
-  RelationshipTypes,
-  FieldType,
   SortType,
+  Table,
 } from "@budibase/types"
 import {
+  breakExternalTableId,
   breakRowIdField,
+  convertRowId,
   generateRowIdField,
   isRowId,
-  convertRowId,
+  isSQL,
 } from "../../../integrations/utils"
 import { getDatasourceAndQuery } from "./utils"
 import { FieldTypes } from "../../../constants"
-import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
 import { cloneDeep } from "lodash/fp"
-import { processFormulas, processDates } from "../../../utilities/rowProcessor"
+import { processDates, processFormulas } from "../../../utilities/rowProcessor"
 import { db as dbCore } from "@budibase/backend-core"
 import sdk from "../../../sdk"
 
@@ -382,10 +383,18 @@ export class ExternalRequest {
       }
       const display = linkedTable.primaryDisplay
       for (let key of Object.keys(row[relationship.column])) {
-        const related: Row = row[relationship.column][key]
+        let relatedRow: Row = row[relationship.column][key]
+        // add this row as context for the relationship
+        for (let col of Object.values(linkedTable.schema)) {
+          if (col.type === FieldType.LINK && col.tableId === table._id) {
+            relatedRow[col.name] = [row]
+          }
+        }
+        relatedRow = processFormulas(linkedTable, relatedRow)
+        const relatedDisplay = display ? relatedRow[display] : undefined
         row[relationship.column][key] = {
-          primaryDisplay: display ? related[display] : undefined,
-          _id: related._id,
+          primaryDisplay: relatedDisplay || "Invalid display column",
+          _id: relatedRow._id,
         }
       }
     }

package/src/api/controllers/row/internal.ts

@@ -30,7 +30,6 @@ import { finaliseRow, updateRelatedFormula } from "./staticFormula"
 import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
-  Ctx,
   UserCtx,
   Database,
   LinkDocumentValue,
@@ -38,7 +37,7 @@ import {
   Table,
 } from "@budibase/types"
 
-const { cleanExportRows } = require("./utils")
+import { cleanExportRows } from "./utils"
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -72,7 +71,7 @@ async function getView(db: Database, viewName: string) {
   return viewInfo
 }
 
-async function getRawTableData(ctx: Ctx, db: Database, tableId: string) {
+async function getRawTableData(ctx: UserCtx, db: Database, tableId: string) {
   let rows
   if (tableId === InternalTables.USER_METADATA) {
     await userController.fetchMetadata(ctx)
@@ -188,7 +187,7 @@ export async function save(ctx: UserCtx) {
   })
 }
 
-export async function fetchView(ctx: Ctx) {
+export async function fetchView(ctx: UserCtx) {
   const viewName = decodeURIComponent(ctx.params.viewName)
 
   // if this is a table view being looked for just transfer to that
@@ -255,7 +254,7 @@ export async function fetchView(ctx: Ctx) {
   return rows
 }
 
-export async function fetch(ctx: Ctx) {
+export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
 
   const tableId = ctx.params.tableId
@@ -264,7 +263,7 @@ export async function fetch(ctx: Ctx) {
   return outputProcessing(table, rows)
 }
 
-export async function find(ctx: Ctx) {
+export async function find(ctx: UserCtx) {
   const db = dbCore.getDB(ctx.appId)
   const table = await db.get(ctx.params.tableId)
   let row = await utils.findRow(ctx, ctx.params.tableId, ctx.params.rowId)
@@ -272,7 +271,7 @@ export async function find(ctx: Ctx) {
   return row
 }
 
-export async function destroy(ctx: Ctx) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const { _id } = ctx.request.body
   let row = await db.get(_id)
@@ -308,7 +307,7 @@ export async function destroy(ctx: Ctx) {
   return { response, row }
 }
 
-export async function bulkDestroy(ctx: Ctx) {
+export async function bulkDestroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const table = await db.get(tableId)
@@ -347,7 +346,7 @@ export async function bulkDestroy(ctx: Ctx) {
   return { response: { ok: true }, rows: processedRows }
 }
 
-export async function search(ctx: Ctx) {
+export async function search(ctx: UserCtx) {
   // Fetch the whole table when running in cypress, as search doesn't work
   if (!env.COUCH_DB_URL && env.isCypress()) {
     return { rows: await fetch(ctx) }
@@ -387,11 +386,14 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function exportRows(ctx: Ctx) {
+export async function exportRows(ctx: UserCtx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
+  if (typeof format !== "string") {
+    ctx.throw(400, "Format parameter is not valid")
+  }
   const { columns, query } = ctx.request.body
 
   let result
@@ -439,7 +441,7 @@ export async function exportRows(ctx: Ctx) {
   }
 }
 
-export async function fetchEnrichedRow(ctx: Ctx) {
+export async function fetchEnrichedRow(ctx: UserCtx) {
   const db = context.getAppDB()
   const tableId = ctx.params.tableId
   const rowId = ctx.params.rowId

package/src/api/controllers/row/utils.ts

@@ -5,7 +5,7 @@ import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
 import { Format } from "../view/exporters"
-import { Ctx } from "@budibase/types"
+import { UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
@@ -26,7 +26,7 @@ export async function getDatasourceAndQuery(json: any) {
   return makeExternalQuery(datasource, json)
 }
 
-export async function findRow(ctx: Ctx, tableId: string, rowId: string) {
+export async function findRow(ctx: UserCtx, tableId: string, rowId: string) {
   const db = context.getAppDB()
   let row
   // TODO remove special user case in future
@@ -137,8 +137,8 @@ export function cleanExportRows(
     delete schema[column]
   })
 
-  // Intended to avoid 'undefined' in export
   if (format === Format.CSV) {
+    // Intended to append empty values in export
     const schemaKeys = Object.keys(schema)
     for (let key of schemaKeys) {
       if (columns?.length && columns.indexOf(key) > 0) {
@@ -146,7 +146,7 @@ export function cleanExportRows(
       }
       for (let row of cleanRows) {
         if (row[key] == null) {
-          row[key] = ""
+          row[key] = undefined
         }
       }
     }

package/src/api/controllers/table/index.ts

@@ -10,7 +10,7 @@ import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
 import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
-import csv from "csvtojson"
+import { jsonFromCsvString } from "../../../utilities/csv"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -104,7 +104,7 @@ export async function bulkImport(ctx: UserCtx) {
 export async function csvToJson(ctx: UserCtx) {
   const { csvString } = ctx.request.body
 
-  const result = await csv().fromString(csvString)
+  const result = await jsonFromCsvString(csvString)
 
   ctx.status = 200
   ctx.body = result

package/src/api/controllers/table/utils.ts

@@ -110,21 +110,28 @@ export function importToRows(
   table: Table,
   user: ContextUser | null = null
 ) {
+  let originalTable = table
   let finalData: any = []
   for (let i = 0; i < data.length; i++) {
     let row = data[i]
     row._id = generateRowID(table._id!)
     row.tableId = table._id
+
+    // We use a reference to table here and update it after input processing,
+    // so that we can auto increment auto IDs in imported data properly
     const processed = inputProcessing(user, table, row, {
       noAutoRelationships: true,
     })
     row = processed.row
     table = processed.table
 
-    for (const [fieldName, schema] of Object.entries(table.schema)) {
-      // check whether the options need to be updated for inclusion as part of the data import
+    // However here we must reference the original table, as we want to mutate
+    // the real schema of the table passed in, not the clone used for
+    // incrementing auto IDs
+    for (const [fieldName, schema] of Object.entries(originalTable.schema)) {
       if (
-        schema.type === FieldTypes.OPTIONS &&
+        (schema.type === FieldTypes.OPTIONS ||
+          schema.type === FieldTypes.ARRAY) &&
         row[fieldName] &&
         (!schema.constraints!.inclusion ||
           schema.constraints!.inclusion.indexOf(row[fieldName]) === -1)