@budibase/server 0.9.186 → 0.9.190-alpha.0

package/src/api/controllers/table/utils.js

@@ -8,7 +8,7 @@ const {
 const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
 const { inputProcessing } = require("../../../utilities/rowProcessor")
-const { USERS_TABLE_SCHEMA } = require("../../../constants")
+const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
@@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
 }
 
 exports.handleDataImport = async (appId, user, table, dataImport) => {
+  if (!dataImport || !dataImport.csvString) {
+    return table
+  }
   const db = new CouchDB(appId)
-  if (dataImport && dataImport.csvString) {
-    // Populate the table with rows imported from CSV in a bulk update
-    const data = await csvParser.transform(dataImport)
-
-    let finalData = []
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      row._id = generateRowID(table._id)
-      row.tableId = table._id
-      const processed = inputProcessing(user, table, row, {
-        noAutoRelationships: true,
-      })
-      table = processed.table
-      row = processed.row
-
-      for (let [fieldName, schema] of Object.entries(table.schema)) {
-        // check whether the options need to be updated for inclusion as part of the data import
-        if (
-          schema.type === FieldTypes.OPTIONS &&
-          (!schema.constraints.inclusion ||
-            schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
-        ) {
-          schema.constraints.inclusion = [
-            ...schema.constraints.inclusion,
-            row[fieldName],
-          ]
-        }
-      }
+  // Populate the table with rows imported from CSV in a bulk update
+  const data = await csvParser.transform({
+    ...dataImport,
+    existingTable: table,
+  })
+
+  let finalData = []
+  for (let i = 0; i < data.length; i++) {
+    let row = data[i]
+    row._id = generateRowID(table._id)
+    row.tableId = table._id
+    const processed = inputProcessing(user, table, row, {
+      noAutoRelationships: true,
+    })
+    table = processed.table
+    row = processed.row
 
-      finalData.push(row)
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
+      // check whether the options need to be updated for inclusion as part of the data import
+      if (
+        schema.type === FieldTypes.OPTIONS &&
+        (!schema.constraints.inclusion ||
+          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
+      ) {
+        schema.constraints.inclusion = [
+          ...schema.constraints.inclusion,
+          row[fieldName],
+        ]
+      }
     }
 
-    await db.bulkDocs(finalData)
-    let response = await db.put(table)
-    table._rev = response._rev
+    finalData.push(row)
   }
+
+  await db.bulkDocs(finalData)
+  let response = await db.put(table)
+  table._rev = response._rev
   return table
 }
 
@@ -315,4 +319,41 @@ exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
   }
 }
 
+exports.generateForeignKey = (column, relatedTable) => {
+  return `fk_${relatedTable.name}_${column.fieldName}`
+}
+
+exports.generateJunctionTableName = (column, table, relatedTable) => {
+  return `jt_${table.name}_${relatedTable.name}_${column.name}_${column.fieldName}`
+}
+
+exports.foreignKeyStructure = (keyName, meta = null) => {
+  const structure = {
+    type: FieldTypes.NUMBER,
+    constraints: {},
+    name: keyName,
+  }
+  if (meta) {
+    structure.meta = meta
+  }
+  return structure
+}
+
+exports.hasTypeChanged = (table, oldTable) => {
+  if (!oldTable) {
+    return false
+  }
+  for (let [key, field] of Object.entries(oldTable.schema)) {
+    const oldType = field.type
+    if (!table.schema[key]) {
+      continue
+    }
+    const newType = table.schema[key].type
+    if (oldType !== newType && SwitchableTypes.indexOf(oldType) === -1) {
+      return true
+    }
+  }
+  return false
+}
+
 exports.TableSaveFunctions = TableSaveFunctions

package/src/api/controllers/user.js

@@ -4,27 +4,144 @@ const {
   getUserMetadataParams,
 } = require("../../db/utils")
 const { InternalTables } = require("../../db/utils")
-const { getGlobalUsers } = require("../../utilities/global")
+const { getGlobalUsers, getRawGlobalUser } = require("../../utilities/global")
 const { getFullUser } = require("../../utilities/users")
+const { isEqual } = require("lodash")
+const { BUILTIN_ROLE_IDS } = require("@budibase/auth/roles")
+const { getDevelopmentAppID, getDeployedAppIDs } = require("@budibase/auth/db")
+const { doesDatabaseExist } = require("../../utilities")
+const { UserStatus } = require("@budibase/auth/constants")
 
-function removeGlobalProps(user) {
-  // make sure to always remove some of the global user props
+async function rawMetadata(db) {
+  return (
+    await db.allDocs(
+      getUserMetadataParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc)
+}
+
+function combineMetadataAndUser(user, metadata) {
+  // skip users with no access
+  if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
+    return null
+  }
+  delete user._rev
+  const metadataId = generateUserMetadataID(user._id)
+  const newDoc = {
+    ...user,
+    _id: metadataId,
+    tableId: InternalTables.USER_METADATA,
+  }
+  const found = Array.isArray(metadata)
+    ? metadata.find(doc => doc._id === metadataId)
+    : metadata
+  // copy rev over for the purposes of equality check
+  if (found) {
+    newDoc._rev = found._rev
+  }
+  if (found == null || !isEqual(newDoc, found)) {
+    return {
+      ...found,
+      ...newDoc,
+    }
+  }
+  return null
+}
+
+exports.syncGlobalUsers = async appId => {
+  // sync user metadata
+  const db = new CouchDB(appId)
+  const [users, metadata] = await Promise.all([
+    getGlobalUsers(appId),
+    rawMetadata(db),
+  ])
+  const toWrite = []
+  for (let user of users) {
+    const combined = await combineMetadataAndUser(user, metadata)
+    if (combined) {
+      toWrite.push(combined)
+    }
+  }
+  await db.bulkDocs(toWrite)
+}
+
+exports.syncUser = async function (ctx) {
+  let deleting = false,
+    user
+  const userId = ctx.params.id
+  try {
+    user = await getRawGlobalUser(userId)
+  } catch (err) {
+    if (err && err.status === 404) {
+      user = {}
+      deleting = true
+    } else {
+      throw err
+    }
+  }
+  const roles = user.roles
+  // remove props which aren't useful to metadata
   delete user.password
+  delete user.forceResetPassword
   delete user.roles
-  delete user.builder
-  return user
+  // run through all production appIDs in the users roles
+  let prodAppIds
+  // if they are a builder then get all production app IDs
+  if ((user.builder && user.builder.global) || deleting) {
+    prodAppIds = await getDeployedAppIDs(CouchDB)
+  } else {
+    prodAppIds = Object.entries(roles)
+      .filter(entry => entry[1] !== BUILTIN_ROLE_IDS.PUBLIC)
+      .map(([appId]) => appId)
+  }
+  for (let prodAppId of prodAppIds) {
+    const roleId = roles[prodAppId]
+    const devAppId = getDevelopmentAppID(prodAppId)
+    for (let appId of [prodAppId, devAppId]) {
+      if (!(await doesDatabaseExist(appId))) {
+        continue
+      }
+      const db = new CouchDB(appId)
+      const metadataId = generateUserMetadataID(userId)
+      let metadata
+      try {
+        metadata = await db.get(metadataId)
+      } catch (err) {
+        if (deleting) {
+          continue
+        }
+        metadata = {
+          tableId: InternalTables.USER_METADATA,
+        }
+      }
+      // assign the roleId for the metadata doc
+      if (roleId) {
+        metadata.roleId = roleId
+      }
+      let combined = !deleting
+        ? combineMetadataAndUser(user, metadata)
+        : {
+            ...metadata,
+            status: UserStatus.INACTIVE,
+            metadata: BUILTIN_ROLE_IDS.PUBLIC,
+          }
+      // if its null then there was no updates required
+      if (combined) {
+        await db.put(combined)
+      }
+    }
+  }
+  ctx.body = {
+    message: "User synced.",
+  }
 }
 
 exports.fetchMetadata = async function (ctx) {
   const database = new CouchDB(ctx.appId)
   const global = await getGlobalUsers(ctx.appId)
-  const metadata = (
-    await database.allDocs(
-      getUserMetadataParams(null, {
-        include_docs: true,
-      })
-    )
-  ).rows.map(row => row.doc)
+  const metadata = await rawMetadata(database)
   const users = []
   for (let user of global) {
     // find the metadata that matches up to the global ID
@@ -52,7 +169,9 @@ exports.updateSelfMetadata = async function (ctx) {
 exports.updateMetadata = async function (ctx) {
   const appId = ctx.appId
   const db = new CouchDB(appId)
-  const user = removeGlobalProps(ctx.request.body)
+  const user = ctx.request.body
+  // this isn't applicable to the user
+  delete user.roles
   const metadata = {
     tableId: InternalTables.USER_METADATA,
     ...user,

package/src/api/controllers/view/exporters.js

@@ -16,3 +16,8 @@ exports.csv = function (headers, rows) {
 exports.json = function (headers, rows) {
   return JSON.stringify(rows, undefined, 2)
 }
+
+exports.ExportFormats = {
+  CSV: "csv",
+  JSON: "json",
+}

package/src/api/controllers/view/index.js

@@ -4,6 +4,7 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
+const { getTable } = require("../table/utils")
 
 exports.fetch = async ctx => {
   const db = new CouchDB(ctx.appId)
@@ -56,7 +57,7 @@ exports.exportView = async ctx => {
   const view = await getView(db, viewName)
 
   const format = ctx.query.format
-  if (!format) {
+  if (!format || !Object.values(exporters.ExportFormats).includes(format)) {
     ctx.throw(400, "Format must be specified, either csv or json")
   }
 
@@ -80,10 +81,22 @@ exports.exportView = async ctx => {
   let schema = view && view.meta && view.meta.schema
   if (!schema) {
     const tableId = ctx.params.tableId || view.meta.tableId
-    const table = await db.get(tableId)
+    const table = await getTable(ctx.appId, tableId)
     schema = table.schema
   }
 
+  // make sure no "undefined" entries appear in the CSV
+  if (format === exporters.ExportFormats.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      for (let row of ctx.body) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
   // Export part
   let headers = Object.keys(schema)
   const exporter = exporters[format]

package/src/api/routes/query.js

@@ -66,6 +66,7 @@ router
   )
   .get(
     "/api/queries/:queryId",
+    paramResource("queryId"),
     authorized(PermissionTypes.QUERY, PermissionLevels.READ),
     queryController.find
   )

package/src/api/routes/table.js

@@ -53,5 +53,16 @@ router
     authorized(BUILDER),
     tableController.destroy
   )
+  // this is currently builder only, but in the future
+  // it could be carried out by an end user in app,
+  // however some thought will need to be had about
+  // implications for automations (triggers)
+  // new trigger type, bulk rows created
+  .post(
+    "/api/tables/:tableId/import",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.bulkImport
+  )
 
 module.exports = router

package/src/api/routes/tests/datasource.spec.js

@@ -26,8 +26,8 @@ describe("/datasources", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual("Datasource saved successfully.")
-      expect(res.body.name).toEqual("Test")
+      expect(res.body.datasource.name).toEqual("Test")
+      expect(res.body.errors).toBeUndefined()
     })
   })
 

package/src/api/routes/tests/misc.spec.js

@@ -75,7 +75,11 @@ describe("run misc tests", () => {
         },
       })
       const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
+        schema: {},
+      }
+      for (let col of ["a", "b", "c", "d"]) {
+        dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
         config.getAppId(),

package/src/api/routes/tests/query.spec.js

@@ -1,7 +1,6 @@
 // mock out postgres for this
 jest.mock("pg")
 
-const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures

package/src/api/routes/user.js

@@ -34,5 +34,10 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.syncUser
+  )
 
 module.exports = router

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {

package/src/automations/triggers.js

@@ -11,6 +11,10 @@ const utils = require("./utils")
 const env = require("../environment")
 
 const TRIGGER_DEFINITIONS = definitions
+const JOB_OPTS = {
+  removeOnComplete: true,
+  removeOnFail: true,
+}
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
@@ -47,7 +51,7 @@ async function queueRelevantRowAutomations(event, eventType) {
       automationTrigger.inputs &&
       automationTrigger.inputs.tableId === event.row.tableId
     ) {
-      await queue.add({ automation, event })
+      await queue.add({ automation, event }, JOB_OPTS)
     }
   }
 }
@@ -86,7 +90,7 @@ exports.externalTrigger = async function (
     automation.definition.trigger != null &&
     automation.definition.trigger.stepId === definitions.APP.stepId &&
     automation.definition.trigger.stepId === "APP" &&
-    !checkTestFlag(automation._id)
+    !(await checkTestFlag(automation._id))
   ) {
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
@@ -100,7 +104,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data)
+    return queue.add(data, JOB_OPTS)
   }
 }
 

package/src/automations/utils.js

@@ -1,4 +1,4 @@
-const runner = require("./thread")
+const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
 const CouchDB = require("../db")
@@ -10,11 +10,12 @@ const { getDeployedAppID } = require("@budibase/auth/db")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
+const Runner = new Thread(ThreadType.AUTOMATION)
 
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
-    return await runner(job)
+    return await Runner.run(job)
   } catch (err) {
     console.error(
       `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@@ -162,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

package/src/constants/index.js

@@ -45,6 +45,13 @@ exports.FieldTypes = {
   INTERNAL: "internal",
 }
 
+exports.SwitchableTypes = [
+  exports.FieldTypes.STRING,
+  exports.FieldTypes.OPTIONS,
+  exports.FieldTypes.NUMBER,
+  exports.FieldTypes.BOOLEAN,
+]
+
 exports.RelationshipTypes = {
   ONE_TO_MANY: "one-to-many",
   MANY_TO_ONE: "many-to-one",
@@ -62,6 +69,10 @@ exports.DataSourceOperation = {
   READ: "READ",
   UPDATE: "UPDATE",
   DELETE: "DELETE",
+  BULK_CREATE: "BULK_CREATE",
+  CREATE_TABLE: "CREATE_TABLE",
+  UPDATE_TABLE: "UPDATE_TABLE",
+  DELETE_TABLE: "DELETE_TABLE",
 }
 
 exports.SortDirection = {
@@ -152,5 +163,16 @@ exports.MetadataTypes = {
   AUTOMATION_TEST_HISTORY: "automationTestHistory",
 }
 
+exports.InvalidColumns = {
+  ID: "_id",
+  REV: "_rev",
+  TABLE_ID: "tableId",
+}
+
+exports.BuildSchemaErrors = {
+  NO_KEY: "no_key",
+  INVALID_COLUMN: "invalid_column",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/definitions/common.ts

@@ -17,6 +17,11 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  main?: boolean
+  meta?: {
+    toTable: string
+    toKey: string
+  }
   constraints?: {
     type?: string
     email?: boolean
@@ -36,11 +41,12 @@ export interface TableSchema {
 export interface Table extends Base {
   type?: string
   views?: {}
-  name?: string
+  name: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  constrained?: string[]
 }
 
 export interface Row extends Base {