@budibase/server 0.9.188 → 0.9.190-alpha.2

package/src/api/controllers/datasource.js

@@ -7,8 +7,9 @@ const {
   BudibaseInternalDB,
   getTableParams,
 } = require("../../db/utils")
+const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
 const { integrations } = require("../../integrations")
-const { makeExternalQuery } = require("./row/utils")
+const { getDatasourceAndQuery } = require("./row/utils")
 
 exports.fetch = async function (ctx) {
   const database = new CouchDB(ctx.appId)
@@ -43,13 +44,17 @@ exports.buildSchemaFromDb = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasource = await db.get(ctx.params.datasourceId)
 
-  const tables = await buildSchemaHelper(datasource)
+  const { tables, error } = await buildSchemaHelper(datasource)
   datasource.entities = tables
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
-  ctx.body = datasource
+  const response = { datasource }
+  if (error) {
+    response.error = error
+  }
+  ctx.body = response
 }
 
 exports.update = async function (ctx) {
@@ -71,7 +76,7 @@ exports.update = async function (ctx) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  ctx.body = { datasource }
 }
 
 exports.save = async function (ctx) {
@@ -85,13 +90,15 @@ exports.save = async function (ctx) {
     ...ctx.request.body.datasource,
   }
 
+  let schemaError = null
   if (fetchSchema) {
-    let tables = await buildSchemaHelper(datasource)
+    const { tables, error } = await buildSchemaHelper(datasource)
+    schemaError = error
     datasource.entities = tables
   }
 
-  const response = await db.put(datasource)
-  datasource._rev = response.rev
+  const dbResp = await db.put(datasource)
+  datasource._rev = dbResp.rev
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
@@ -101,17 +108,27 @@ exports.save = async function (ctx) {
     }
   }
 
-  ctx.status = 200
-  ctx.message = "Datasource saved successfully."
-  ctx.body = datasource
+  const response = { datasource }
+  if (schemaError) {
+    response.error = schemaError
+  }
+  ctx.body = response
 }
 
 exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   // Delete all queries for the datasource
-  const rows = await db.allDocs(getQueryParams(ctx.params.datasourceId, null))
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  const queries = await db.allDocs(
+    getQueryParams(ctx.params.datasourceId, null)
+  )
+  await db.bulkDocs(
+    queries.rows.map(row => ({
+      _id: row.id,
+      _rev: row.value.rev,
+      _deleted: true,
+    }))
+  )
 
   // delete the datasource
   await db.remove(ctx.params.datasourceId, ctx.params.revId)
@@ -129,12 +146,29 @@ exports.find = async function (ctx) {
 exports.query = async function (ctx) {
   const queryJson = ctx.request.body
   try {
-    ctx.body = await makeExternalQuery(ctx.appId, queryJson)
+    ctx.body = await getDatasourceAndQuery(ctx.appId, queryJson)
   } catch (err) {
     ctx.throw(400, err)
   }
 }
 
+function getErrorTables(errors, errorType) {
+  return Object.entries(errors)
+    .filter(entry => entry[1] === errorType)
+    .map(([name]) => name)
+}
+
+function updateError(error, newError, tables) {
+  if (!error) {
+    error = ""
+  }
+  if (error.length > 0) {
+    error += "\n"
+  }
+  error += `${newError} ${tables.join(", ")}`
+  return error
+}
+
 const buildSchemaHelper = async datasource => {
   const Connector = integrations[datasource.source]
 
@@ -143,5 +177,39 @@ const buildSchemaHelper = async datasource => {
   await connector.buildSchema(datasource._id, datasource.entities)
   datasource.entities = connector.tables
 
-  return connector.tables
+  // make sure they all have a display name selected
+  for (let entity of Object.values(datasource.entities)) {
+    if (entity.primaryDisplay) {
+      continue
+    }
+    const notAutoColumn = Object.values(entity.schema).find(
+      schema => !schema.autocolumn
+    )
+    if (notAutoColumn) {
+      entity.primaryDisplay = notAutoColumn.name
+    }
+  }
+
+  const errors = connector.schemaErrors
+  let error = null
+  if (errors && Object.keys(errors).length > 0) {
+    const noKey = getErrorTables(errors, BuildSchemaErrors.NO_KEY)
+    const invalidCol = getErrorTables(errors, BuildSchemaErrors.INVALID_COLUMN)
+    if (noKey.length) {
+      error = updateError(
+        error,
+        "No primary key constraint found for the following:",
+        noKey
+      )
+    }
+    if (invalidCol.length) {
+      const invalidCols = Object.values(InvalidColumns).join(", ")
+      error = updateError(
+        error,
+        `Cannot use columns ${invalidCols} found in following:`,
+        invalidCol
+      )
+    }
+  }
+  return { tables: connector.tables, error }
 }

package/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

package/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -24,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {
@@ -106,6 +108,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }

package/src/api/controllers/query.js

@@ -1,10 +1,11 @@
 const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
 const { generateQueryID, getQueryParams } = require("../../db/utils")
-const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
-const ScriptRunner = require("../../utilities/scriptRunner")
+const { Thread, ThreadType } = require("../../threads")
+
+const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -18,47 +19,6 @@ function enrichQueries(input) {
   return wasArray ? queries : queries[0]
 }
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
-  }
-  return resp
-}
-
-async function runAndTransform(
-  integration,
-  queryVerb,
-  enrichedQuery,
-  transformer
-) {
-  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
-
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
-
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
-
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
-  }
-
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
-
-  return { rows, keys }
-}
-
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -98,7 +58,9 @@ async function enrichQueryFields(fields, parameters = {}) {
       enrichedQuery[key] = await enrichQueryFields(fields[key], parameters)
     } else if (typeof fields[key] === "string") {
       // enrich string value as normal
-      enrichedQuery[key] = await processString(fields[key], parameters)
+      enrichedQuery[key] = await processString(fields[key], parameters, {
+        noHelpers: true,
+      })
     } else {
       enrichedQuery[key] = fields[key]
     }
@@ -141,30 +103,23 @@ exports.preview = async function (ctx) {
 
   const datasource = await db.get(ctx.request.body.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-  const integration = new Integration(datasource.config)
 
-  const { rows, keys } = await runAndTransform(
-    integration,
-    queryVerb,
-    enrichedQuery,
-    transformer
-  )
+  try {
+    const { rows, keys } = await Runner.run({
+      datasource,
+      queryVerb,
+      query: enrichedQuery,
+      transformer,
+    })
 
-  ctx.body = {
-    rows,
-    schemaFields: [...new Set(keys)],
-  }
-  // cleanup
-  if (integration.end) {
-    integration.end()
+    ctx.body = {
+      rows,
+      schemaFields: [...new Set(keys)],
+    }
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 
@@ -174,29 +129,22 @@ exports.execute = async function (ctx) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const enrichedQuery = await enrichQueryFields(
     query.fields,
     ctx.request.body.parameters
   )
-  const integration = new Integration(datasource.config)
 
   // call the relevant CRUD method on the integration class
-  const { rows } = await runAndTransform(
-    integration,
-    query.queryVerb,
-    enrichedQuery,
-    query.transformer
-  )
-  ctx.body = rows
-  // cleanup
-  if (integration.end) {
-    integration.end()
+  try {
+    const { rows } = await Runner.run({
+      datasource,
+      queryVerb: query.queryVerb,
+      query: enrichedQuery,
+      transformer: query.transformer,
+    })
+    ctx.body = rows
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 

package/src/api/controllers/row/ExternalRequest.ts

@@ -33,10 +33,11 @@ interface RunConfig {
   sort?: SortJson
   paginate?: PaginationJson
   row?: Row
+  rows?: Row[]
 }
 
 module External {
-  const { makeExternalQuery } = require("./utils")
+  const { getDatasourceAndQuery } = require("./utils")
   const {
     DataSourceOperation,
     FieldTypes,
@@ -46,6 +47,7 @@ module External {
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
   const CouchDB = require("../../../db")
+  const { processFormulas } = require("../../../utilities/rowProcessor/utils")
 
   function buildFilters(
     id: string | undefined,
@@ -162,8 +164,8 @@ module External {
     }
   }
 
-  function basicProcessing(row: Row, table: Table) {
-    const thisRow: { [key: string]: any } = {}
+  function basicProcessing(row: Row, table: Table): Row {
+    const thisRow: Row = {}
     // filter the row down to what is actually the row (not joined)
     for (let fieldName of Object.keys(table.schema)) {
       const value = row[`${table.name}.${fieldName}`] || row[fieldName]
@@ -178,9 +180,26 @@ module External {
     return thisRow
   }
 
-  function isMany(field: FieldSchema) {
+  function fixArrayTypes(row: Row, table: Table) {
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
+      if (
+        schema.type === FieldTypes.ARRAY &&
+        typeof row[fieldName] === "string"
+      ) {
+        try {
+          row[fieldName] = JSON.parse(row[fieldName])
+        } catch (err) {
+          // couldn't convert back to array, ignore
+          delete row[fieldName]
+        }
+      }
+    }
+    return row
+  }
+
+  function isOneSide(field: FieldSchema) {
     return (
-      field.relationshipType && field.relationshipType.split("-")[0] === "many"
+      field.relationshipType && field.relationshipType.split("-")[0] === "one"
     )
   }
 
@@ -225,7 +244,12 @@ module External {
         manyRelationships: ManyRelationship[] = []
       for (let [key, field] of Object.entries(table.schema)) {
         // if set already, or not set just skip it
-        if ((!row[key] && row[key] !== "") || newRow[key] || field.autocolumn) {
+        if (
+          row[key] == null ||
+          newRow[key] ||
+          field.autocolumn ||
+          field.type === FieldTypes.FORMULA
+        ) {
           continue
         }
         // if its an empty string then it means return the column to null (if possible)
@@ -250,25 +274,37 @@ module External {
         const linkTable = this.tables[linkTableName]
         // @ts-ignore
         const linkTablePrimary = linkTable.primary[0]
-        if (!isMany(field)) {
+        // one to many
+        if (isOneSide(field)) {
           newRow[field.foreignKey || linkTablePrimary] = breakRowIdField(
             row[key][0]
           )[0]
-        } else {
+        }
+        // many to many
+        else if (field.through) {
           // we're not inserting a doc, will be a bunch of update calls
-          const isUpdate = !field.through
-          const thisKey: string = isUpdate
-            ? "id"
-            : field.throughTo || linkTablePrimary
-          // @ts-ignore
-          const otherKey: string = isUpdate
-            ? field.fieldName
-            : field.throughFrom || tablePrimary
+          const otherKey: string = field.throughFrom || linkTablePrimary
+          const thisKey: string = field.throughTo || tablePrimary
           row[key].map((relationship: any) => {
-            // we don't really support composite keys for relationships, this is why [0] is used
             manyRelationships.push({
               tableId: field.through || field.tableId,
-              isUpdate,
+              isUpdate: false,
+              key: otherKey,
+              [otherKey]: breakRowIdField(relationship)[0],
+              // leave the ID for enrichment later
+              [thisKey]: `{{ literal ${tablePrimary} }}`,
+            })
+          })
+        }
+        // many to one
+        else {
+          const thisKey: string = "id"
+          // @ts-ignore
+          const otherKey: string = field.fieldName
+          row[key].map((relationship: any) => {
+            manyRelationships.push({
+              tableId: field.tableId,
+              isUpdate: true,
               key: otherKey,
               [thisKey]: breakRowIdField(relationship)[0],
               // leave the ID for enrichment later
@@ -336,7 +372,7 @@ module External {
       table: Table,
       relationships: RelationshipsJson[]
     ) {
-      if (rows[0].read === true) {
+      if (!rows || rows.length === 0 || rows[0].read === true) {
         return []
       }
       let finalRows: { [key: string]: Row } = {}
@@ -352,7 +388,10 @@ module External {
           )
           continue
         }
-        const thisRow = basicProcessing(row, table)
+        const thisRow = fixArrayTypes(basicProcessing(row, table), table)
+        if (thisRow._id == null) {
+          throw "Unable to generate row ID for SQL rows"
+        }
         finalRows[thisRow._id] = thisRow
         // do this at end once its been added to the final rows
         finalRows = this.updateRelationshipColumns(
@@ -361,7 +400,7 @@ module External {
           relationships
         )
       }
-      return Object.values(finalRows)
+      return processFormulas(table, Object.values(finalRows))
     }
 
     /**
@@ -398,8 +437,8 @@ module External {
           )
           definition.through = throughTableName
           // don't support composite keys for relationships
-          definition.from = field.throughFrom || table.primary[0]
-          definition.to = field.throughTo || linkTable.primary[0]
+          definition.from = field.throughTo || table.primary[0]
+          definition.to = field.throughFrom || linkTable.primary[0]
           definition.fromPrimary = table.primary[0]
           definition.toPrimary = linkTable.primary[0]
         }
@@ -421,24 +460,36 @@ module External {
       // make a new request to get the row with all its relationships
       // we need this to work out if any relationships need removed
       for (let field of Object.values(table.schema)) {
-        if (field.type !== FieldTypes.LINK || !field.fieldName) {
+        if (
+          field.type !== FieldTypes.LINK ||
+          !field.fieldName ||
+          isOneSide(field)
+        ) {
           continue
         }
         const isMany = field.relationshipType === RelationshipTypes.MANY_TO_MANY
         const tableId = isMany ? field.through : field.tableId
-        const manyKey = field.throughFrom || primaryKey
+        const { tableName: relatedTableName } = breakExternalTableId(tableId)
+        // @ts-ignore
+        const linkPrimaryKey = this.tables[relatedTableName].primary[0]
+        const manyKey = field.throughTo || primaryKey
+        const lookupField = isMany ? primaryKey : field.foreignKey
         const fieldName = isMany ? manyKey : field.fieldName
-        const response = await makeExternalQuery(this.appId, {
+        if (!lookupField || !row[lookupField]) {
+          continue
+        }
+        const response = await getDatasourceAndQuery(this.appId, {
           endpoint: getEndpoint(tableId, DataSourceOperation.READ),
           filters: {
             equal: {
-              [fieldName]: row[primaryKey],
+              [fieldName]: row[lookupField],
             },
           },
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        related[fieldName] = { rows, isMany, tableId }
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        related[storeTo] = { rows, isMany, tableId }
       }
       return related
     }
@@ -479,7 +530,7 @@ module External {
           : DataSourceOperation.CREATE
         if (!found) {
           promises.push(
-            makeExternalQuery(appId, {
+            getDatasourceAndQuery(appId, {
               endpoint: getEndpoint(tableId, operation),
               // if we're doing many relationships then we're writing, only one response
               body,
@@ -509,7 +560,7 @@ module External {
               : DataSourceOperation.UPDATE
             const body = isMany ? null : { [colName]: null }
             promises.push(
-              makeExternalQuery(this.appId, {
+              getDatasourceAndQuery(this.appId, {
                 endpoint: getEndpoint(tableId, op),
                 body,
                 filters,
@@ -532,16 +583,17 @@ module External {
       table: Table,
       includeRelations: IncludeRelationships = IncludeRelationships.INCLUDE
     ) {
-      function extractNonLinkFieldNames(table: Table, existing: string[] = []) {
+      function extractRealFields(table: Table, existing: string[] = []) {
         return Object.entries(table.schema)
           .filter(
             column =>
               column[1].type !== FieldTypes.LINK &&
+              column[1].type !== FieldTypes.FORMULA &&
               !existing.find((field: string) => field === column[0])
           )
           .map(column => `${table.name}.${column[0]}`)
       }
-      let fields = extractNonLinkFieldNames(table)
+      let fields = extractRealFields(table)
       for (let field of Object.values(table.schema)) {
         if (field.type !== FieldTypes.LINK || !includeRelations) {
           continue
@@ -549,7 +601,7 @@ module External {
         const { tableName: linkTableName } = breakExternalTableId(field.tableId)
         const linkTable = this.tables[linkTableName]
         if (linkTable) {
-          const linkedFields = extractNonLinkFieldNames(linkTable, fields)
+          const linkedFields = extractRealFields(linkTable, fields)
           fields = fields.concat(linkedFields)
         }
       }
@@ -573,7 +625,10 @@ module External {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
       // look for specific components of config which may not be considered acceptable
-      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
+      let { id, row, filters, sort, paginate, rows } = cleanupConfig(
+        config,
+        table
+      )
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
       // clean up row on ingress using schema
@@ -599,7 +654,7 @@ module External {
         sort,
         paginate,
         relationships,
-        body: row,
+        body: row || rows,
         // pass an id filter into extra, purely for mysql/returning
         extra: {
           idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),
@@ -609,7 +664,7 @@ module External {
         },
       }
       // can't really use response right now
-      const response = await makeExternalQuery(appId, json)
+      const response = await getDatasourceAndQuery(appId, json)
       // handle many to many relationships now if we know the ID (could be auto increment)
       if (
         operation !== DataSourceOperation.READ &&

package/src/api/controllers/row/external.js

@@ -30,6 +30,8 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
   )
 }
 
+exports.handleRequest = handleRequest
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const inputs = ctx.request.body
@@ -55,7 +57,8 @@ exports.save = async ctx => {
 exports.fetchView = async ctx => {
   // there are no views in external data sources, shouldn't ever be called
   // for now just fetch
-  ctx.params.tableId = ctx.params.viewName.split("all_")[1]
+  const split = ctx.params.viewName.split("all_")
+  ctx.params.tableId = split[1] ? split[1] : split[0]
   return exports.fetch(ctx)
 }