@budibase/server 0.9.188 → 0.9.190-alpha.2

package/src/api/controllers/table/internal.js

@@ -0,0 +1,155 @@
+const CouchDB = require("../../../db")
+const linkRows = require("../../../db/linkedRows")
+const { getRowParams, generateTableID } = require("../../../db/utils")
+const { FieldTypes } = require("../../../constants")
+const {
+  TableSaveFunctions,
+  hasTypeChanged,
+  getTable,
+  handleDataImport,
+} = require("./utils")
+
+exports.save = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const { dataImport, ...rest } = ctx.request.body
+  let tableToSave = {
+    type: "table",
+    _id: generateTableID(),
+    views: {},
+    ...rest,
+  }
+
+  // if the table obj had an _id then it will have been retrieved
+  let oldTable
+  if (ctx.request.body && ctx.request.body._id) {
+    oldTable = await db.get(ctx.request.body._id)
+  }
+
+  if (hasTypeChanged(tableToSave, oldTable)) {
+    ctx.throw(400, "A column type has changed.")
+  }
+
+  // saving a table is a complex operation, involving many different steps, this
+  // has been broken out into a utility to make it more obvious/easier to manipulate
+  const tableSaveFunctions = new TableSaveFunctions({
+    db,
+    ctx,
+    oldTable,
+    dataImport,
+  })
+  tableToSave = await tableSaveFunctions.before(tableToSave)
+
+  // make sure that types don't change of a column, have to remove
+  // the column if you want to change the type
+  if (oldTable && oldTable.schema) {
+    for (let propKey of Object.keys(tableToSave.schema)) {
+      let column = tableToSave.schema[propKey]
+      let oldColumn = oldTable.schema[propKey]
+      if (oldColumn && oldColumn.type === "internal") {
+        oldColumn.type = "auto"
+      }
+      if (oldColumn && oldColumn.type !== column.type) {
+        ctx.throw(400, "Cannot change the type of a column")
+      }
+    }
+  }
+
+  // Don't rename if the name is the same
+  let { _rename } = tableToSave
+  /* istanbul ignore next */
+  if (_rename && _rename.old === _rename.updated) {
+    _rename = null
+    delete tableToSave._rename
+  }
+
+  // rename row fields when table column is renamed
+  /* istanbul ignore next */
+  if (_rename && tableToSave.schema[_rename.updated].type === FieldTypes.LINK) {
+    ctx.throw(400, "Cannot rename a linked column.")
+  }
+
+  tableToSave = await tableSaveFunctions.mid(tableToSave)
+
+  // update schema of non-statistics views when new columns are added
+  for (let view in tableToSave.views) {
+    const tableView = tableToSave.views[view]
+    if (!tableView) continue
+
+    if (tableView.schema.group || tableView.schema.field) continue
+    tableView.schema = tableToSave.schema
+  }
+
+  // update linked rows
+  try {
+    const linkResp = await linkRows.updateLinks({
+      appId,
+      eventType: oldTable
+        ? linkRows.EventType.TABLE_UPDATED
+        : linkRows.EventType.TABLE_SAVE,
+      table: tableToSave,
+      oldTable: oldTable,
+    })
+    if (linkResp != null && linkResp._rev) {
+      tableToSave._rev = linkResp._rev
+    }
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+
+  // don't perform any updates until relationships have been
+  // checked by the updateLinks function
+  const updatedRows = tableSaveFunctions.getUpdatedRows()
+  if (updatedRows && updatedRows.length !== 0) {
+    await db.bulkDocs(updatedRows)
+  }
+  const result = await db.put(tableToSave)
+  tableToSave._rev = result.rev
+
+  tableToSave = await tableSaveFunctions.after(tableToSave)
+
+  return tableToSave
+}
+
+exports.destroy = async function (ctx) {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const tableToDelete = await db.get(ctx.params.tableId)
+
+  // Delete all rows for that table
+  const rows = await db.allDocs(
+    getRowParams(ctx.params.tableId, null, {
+      include_docs: true,
+    })
+  )
+  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+
+  // update linked rows
+  await linkRows.updateLinks({
+    appId,
+    eventType: linkRows.EventType.TABLE_DELETE,
+    table: tableToDelete,
+  })
+
+  // don't remove the table itself until very end
+  await db.remove(tableToDelete)
+
+  // remove table search index
+  const currentIndexes = await db.getIndexes()
+  const existingIndex = currentIndexes.indexes.find(
+    existing => existing.name === `search:${ctx.params.tableId}`
+  )
+  if (existingIndex) {
+    await db.deleteIndex(existingIndex)
+  }
+
+  return tableToDelete
+}
+
+exports.bulkImport = async function (ctx) {
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
+  const { dataImport } = ctx.request.body
+  await handleDataImport(appId, ctx.user, table, dataImport)
+  return table
+}

package/src/api/controllers/table/utils.js

@@ -8,7 +8,7 @@ const {
 const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
 const { inputProcessing } = require("../../../utilities/rowProcessor")
-const { USERS_TABLE_SCHEMA } = require("../../../constants")
+const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
@@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
 }
 
 exports.handleDataImport = async (appId, user, table, dataImport) => {
+  if (!dataImport || !dataImport.csvString) {
+    return table
+  }
   const db = new CouchDB(appId)
-  if (dataImport && dataImport.csvString) {
-    // Populate the table with rows imported from CSV in a bulk update
-    const data = await csvParser.transform(dataImport)
-
-    let finalData = []
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      row._id = generateRowID(table._id)
-      row.tableId = table._id
-      const processed = inputProcessing(user, table, row, {
-        noAutoRelationships: true,
-      })
-      table = processed.table
-      row = processed.row
-
-      for (let [fieldName, schema] of Object.entries(table.schema)) {
-        // check whether the options need to be updated for inclusion as part of the data import
-        if (
-          schema.type === FieldTypes.OPTIONS &&
-          (!schema.constraints.inclusion ||
-            schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
-        ) {
-          schema.constraints.inclusion = [
-            ...schema.constraints.inclusion,
-            row[fieldName],
-          ]
-        }
-      }
+  // Populate the table with rows imported from CSV in a bulk update
+  const data = await csvParser.transform({
+    ...dataImport,
+    existingTable: table,
+  })
+
+  let finalData = []
+  for (let i = 0; i < data.length; i++) {
+    let row = data[i]
+    row._id = generateRowID(table._id)
+    row.tableId = table._id
+    const processed = inputProcessing(user, table, row, {
+      noAutoRelationships: true,
+    })
+    table = processed.table
+    row = processed.row
 
-      finalData.push(row)
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
+      // check whether the options need to be updated for inclusion as part of the data import
+      if (
+        schema.type === FieldTypes.OPTIONS &&
+        (!schema.constraints.inclusion ||
+          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
+      ) {
+        schema.constraints.inclusion = [
+          ...schema.constraints.inclusion,
+          row[fieldName],
+        ]
+      }
     }
 
-    await db.bulkDocs(finalData)
-    let response = await db.put(table)
-    table._rev = response._rev
+    finalData.push(row)
   }
+
+  await db.bulkDocs(finalData)
+  let response = await db.put(table)
+  table._rev = response._rev
   return table
 }
 
@@ -315,4 +319,41 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
   }
 }
 
+exports.generateForeignKey = (column, relatedTable) => {
+  return `fk_${relatedTable.name}_${column.fieldName}`
+}
+
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
+}
+
+exports.foreignKeyStructure = (keyName, meta = null) => {
+  const structure = {
+    type: FieldTypes.NUMBER,
+    constraints: {},
+    name: keyName,
+  }
+  if (meta) {
+    structure.meta = meta
+  }
+  return structure
+}
+
+exports.hasTypeChanged = (table, oldTable) => {
+  if (!oldTable) {
+    return false
+  }
+  for (let [key, field] of Object.entries(oldTable.schema)) {
+    const oldType = field.type
+    if (!table.schema[key]) {
+      continue
+    }
+    const newType = table.schema[key].type
+    if (oldType !== newType && SwitchableTypes.indexOf(oldType) === -1) {
+      return true
+    }
+  }
+  return false
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

package/src/api/controllers/user.js

@@ -4,27 +4,144 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getGlobalUsers } = require("../../utilities/global")
+const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
 const { getFullUser } = require("../../utilities/users")
+const { isEqual } = require("lodash")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db")
+const { doesDatabaseExist } = require("../../utilities")
+const { UserStatus } = require("@budibase/auth/constants")
 
-function removeGlobalProps(user) {
-  // make sure to always remove some of the global user props
+async function rawMetadata(db) {
+  return (
+    await db.allDocs(
+      getUserMetadataParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+}
+
+function combineMetadataAndUser(user, metadata) {
+  // skip users with no access
+  if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
+    return null
+  }
+  delete user._rev
+  const metadataId = generateUserMetadataID(user._id)
+  const newDoc = {
+    ...user,
+    _id: metadataId,
+    tableId: InternalTables.USER_METADATA,
+  }
+  const found = Array.isArray(metadata)
+    ? metadata.find(doc => doc._id === metadataId)
+    : metadata
+  // copy rev over for the purposes of equality check
+  if (found) {
+    newDoc._rev = found._rev
+  }
+  if (found == null || !isEqual(newDoc, found)) {
+    return {
+      ...found,
+      ...newDoc,
+    }
+  }
+  return null
+}
+
+exports.syncGlobalUsers = async appId => {
+  // sync user metadata
+  const db = new CouchDB(appId)
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(appId),
+    rawMetadata(db),
+  ])
+  const toWrite = []
+  for (let user of users) {
+    const combined = await combineMetadataAndUser(user, metadata)
+    if (combined) {
+      toWrite.push(combined)
+    }
+  }
+  await db.bulkDocs(toWrite)
+}
+
+exports.syncUser = async function (ctx) {
+  let deleting = false,
+    user
+  const userId = ctx.params.id
+  try {
+    user = await getRawGlobalUser(userId)
+  } catch (err) {
+    if (err && err.status === 404) {
+      user = {}
+      deleting = true
+    } else {
+      throw err
+    }
+  }
+  const roles = user.roles
+  // remove props which aren't useful to metadata
   delete user.password
+  delete user.forceResetPassword
   delete user.roles
-  delete user.builder
-  return user
+  // run through all production appIDs in the users roles
+  let prodAppIds
+  // if they are a builder then get all production app IDs
+  if ((user.builder && user.builder.global) || deleting) {
+    prodAppIds = await getDeployedAppIDs(CouchDB)
+  } else {
+    prodAppIds = Object.entries(roles)
+      .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
+      .map(([appId]) => appId)
+  }
+  for (let prodAppId of prodAppIds) {
+    const roleId = roles[prodAppId]
+    const devAppId = getDevelopmentAppID(prodAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      if (!(await doesDatabaseExist(appId))) {
+        continue
+      }
+      const db = new CouchDB(appId)
+      const metadataId = generateUserMetadataID(userId)
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err) {
+        if (deleting) {
+          continue
+        }
+        metadata = {
+          tableId: InternalTables.USER_METADATA,
+        }
+      }
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+      let combined = !deleting
+        ? combineMetadataAndUser(user, metadata)
+        : {
+            ...metadata,
+            status: UserStatus.INACTIVE,
+            metadata: BUILTIN_ROLE_IDS.PUBLIC,
+          }
+      // if its null then there was no updates required
+      if (combined) {
+        await db.put(combined)
+      }
+    }
+  }
+  ctx.body = {
+    message: "User synced.",
+  }
 }
 
 exports.fetchMetadata = async function (ctx) {
   const database = new CouchDB(ctx.appId)
   const global = await getGlobalUsers(ctx.appId)
-  const metadata = (
-    await database.allDocs(
-      getUserMetadataParams(null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc)
+  const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
     // find the metadata that matches up to the global ID
@@ -52,7 +169,9 @@ exports.updateSelfMetadata = async function (ctx) {
 exports.updateMetadata = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
-  const user = removeGlobalProps(ctx.request.body)
+  const user = ctx.request.body
+  // this isn't applicable to the user
+  delete user.roles
   const metadata = {
     tableId: InternalTables.USER_METADATA,
     ...user,

package/src/api/controllers/view/exporters.js

@@ -16,3 +16,8 @@ exports.csv = function (headers, rows) {
 exports.json = function (headers, rows) {
   return JSON.stringify(rows, undefined, 2)
 }
+
+exports.ExportFormats = {
+  CSV: "csv",
+  JSON: "json",
+}

package/src/api/controllers/view/index.js

@@ -4,6 +4,7 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
+const { getTable } = require("../table/utils")
 
 exports.fetch = async ctx => {
   const db = new CouchDB(ctx.appId)
@@ -56,7 +57,7 @@ exports.exportView = async ctx => {
   const view = await getView(db, viewName)
 
   const format = ctx.query.format
-  if (!format) {
+  if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
     ctx.throw(400, "Format must be specified, either csv or json")
   }
 
@@ -80,10 +81,22 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await db.get(tableId)
+    const table = await getTable(ctx.appId, tableId)
     schema = table.schema
   }
 
+  // make sure no "undefined" entries appear in the CSV
+  if (format === exporters.ExportFormats.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      for (let row of ctx.body) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
   // Export part
   let headers = Object.keys(schema)
   const exporter = exporters[format]

package/src/api/routes/query.js

@@ -66,6 +66,7 @@ router
   )
   .get(
     "/api/queries/:queryId",
+    paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )

package/src/api/routes/table.js

@@ -53,5 +53,16 @@ router
     authorized(BUILDER),
     tableController.destroy
   )
+  // this is currently builder only, but in the future
+  // it could be carried out by an end user in app,
+  // however some thought will need to be had about
+  // implications for automations (triggers)
+  // new trigger type, bulk rows created
+  .post(
+    "/api/tables/:tableId/import",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.bulkImport
+  )
 
 module.exports = router

package/src/api/routes/tests/datasource.spec.js

@@ -26,8 +26,8 @@ describe("/datasources", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual("Datasource saved successfully.")
-      expect(res.body.name).toEqual("Test")
+      expect(res.body.datasource.name).toEqual("Test")
+      expect(res.body.errors).toBeUndefined()
     })
   })
 

package/src/api/routes/tests/misc.spec.js

@@ -75,7 +75,11 @@ describe("run misc tests", () => {
         },
       })
       const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
+        schema: {},
+      }
+      for (let col of ["a", "b", "c", "d"]) {
+        dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
         config.getAppId(),

package/src/api/routes/tests/query.spec.js

@@ -1,7 +1,6 @@
 // mock out postgres for this
 jest.mock("pg")
 
-const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures

package/src/api/routes/user.js

@@ -34,5 +34,10 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.syncUser
+  )
 
 module.exports = router

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {