@budibase/server 0.9.184 → 0.9.185-alpha.11

package/src/api/routes/tests/misc.spec.js

@@ -75,7 +75,11 @@ describe("run misc tests", () => {
         },
       })
       const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
+        schema: {},
+      }
+      for (let col of ["a", "b", "c", "d"]) {
+        dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
         config.getAppId(),

package/src/api/routes/tests/query.spec.js

@@ -1,7 +1,6 @@
 // mock out postgres for this
 jest.mock("pg")
 
-const { findLastKey } = require("lodash/fp")
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures

package/src/api/routes/user.js

@@ -34,5 +34,10 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.syncUser
+  )
 
 module.exports = router

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {

package/src/automations/triggers.js

@@ -11,6 +11,10 @@ const utils = require("./utils")
 const env = require("../environment")
 
 const TRIGGER_DEFINITIONS = definitions
+const JOB_OPTS = {
+  removeOnComplete: true,
+  removeOnFail: true,
+}
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
@@ -47,7 +51,7 @@ async function queueRelevantRowAutomations(event, eventType) {
       automationTrigger.inputs &&
       automationTrigger.inputs.tableId === event.row.tableId
     ) {
-      await queue.add({ automation, event })
+      await queue.add({ automation, event }, JOB_OPTS)
     }
   }
 }
@@ -86,7 +90,7 @@ exports.externalTrigger = async function (
     automation.definition.trigger != null &&
     automation.definition.trigger.stepId === definitions.APP.stepId &&
     automation.definition.trigger.stepId === "APP" &&
-    !checkTestFlag(automation._id)
+    !(await checkTestFlag(automation._id))
   ) {
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
@@ -100,7 +104,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data)
+    return queue.add(data, JOB_OPTS)
   }
 }
 

package/src/automations/utils.js

@@ -1,4 +1,4 @@
-const runner = require("./thread")
+const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
 const CouchDB = require("../db")
@@ -10,11 +10,12 @@ const { getDeployedAppID } = require("@budibase/auth/db")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
+const Runner = new Thread(ThreadType.AUTOMATION)
 
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
-    return await runner(job)
+    return await Runner.run(job)
   } catch (err) {
     console.error(
       `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@@ -162,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

package/src/constants/index.js

@@ -45,6 +45,13 @@ exports.FieldTypes = {
   INTERNAL: "internal",
 }
 
+exports.SwitchableTypes = [
+  exports.FieldTypes.STRING,
+  exports.FieldTypes.OPTIONS,
+  exports.FieldTypes.NUMBER,
+  exports.FieldTypes.BOOLEAN,
+]
+
 exports.RelationshipTypes = {
   ONE_TO_MANY: "one-to-many",
   MANY_TO_ONE: "many-to-one",
@@ -62,6 +69,10 @@ exports.DataSourceOperation = {
   READ: "READ",
   UPDATE: "UPDATE",
   DELETE: "DELETE",
+  BULK_CREATE: "BULK_CREATE",
+  CREATE_TABLE: "CREATE_TABLE",
+  UPDATE_TABLE: "UPDATE_TABLE",
+  DELETE_TABLE: "DELETE_TABLE",
 }
 
 exports.SortDirection = {
@@ -152,5 +163,9 @@ exports.MetadataTypes = {
   AUTOMATION_TEST_HISTORY: "automationTestHistory",
 }
 
+exports.BuildSchemaErrors = {
+  NO_KEY: "no_key",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/definitions/common.ts

@@ -17,6 +17,11 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  main?: boolean
+  meta?: {
+    toTable: string
+    toKey: string
+  }
   constraints?: {
     type?: string
     email?: boolean
@@ -36,11 +41,12 @@ export interface TableSchema {
 export interface Table extends Base {
   type?: string
   views?: {}
-  name?: string
+  name: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  constrained?: string[]
 }
 
 export interface Row extends Base {

package/src/definitions/datasource.ts

@@ -1,10 +1,14 @@
-import { Table } from "./common"
+import { Row, Table } from "./common"
 
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
   UPDATE = "UPDATE",
   DELETE = "DELETE",
+  BULK_CREATE = "BULK_CREATE",
+  CREATE_TABLE = "CREATE_TABLE",
+  UPDATE_TABLE = "UPDATE_TABLE",
+  DELETE_TABLE = "DELETE_TABLE",
 }
 
 export enum SortDirection {
@@ -116,7 +120,7 @@ export interface SortJson {
 
 export interface PaginationJson {
   limit: number
-  page: string | number
+  page?: string | number
 }
 
 export interface RelationshipsJson {
@@ -141,9 +145,11 @@ export interface QueryJson {
   filters?: SearchFilters
   sort?: SortJson
   paginate?: PaginationJson
-  body?: object
+  body?: Row | Row[]
+  table?: Table
   meta?: {
     table?: Table
+    tables?: Record<string, Table>
   }
   extra?: {
     idFilter?: SearchFilters

package/src/integrations/airtable.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module AirtableModule {
   const Airtable = require("airtable")
@@ -73,7 +74,7 @@ module AirtableModule {
     },
   }
 
-  class AirtableIntegration {
+  class AirtableIntegration implements IntegrationBase {
     private config: AirtableConfig
     private client: any
 

package/src/integrations/arangodb.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module ArangoModule {
   const { Database, aql } = require("arangojs")
@@ -55,7 +56,7 @@ module ArangoModule {
     },
   }
 
-  class ArangoDBIntegration {
+  class ArangoDBIntegration implements IntegrationBase {
     private config: ArangodbConfig
     private client: any
 

package/src/integrations/base/IntegrationBase.ts

@@ -0,0 +1,6 @@
+export interface IntegrationBase {
+  create?(query: any): Promise<[any]>
+  read?(query: any): Promise<[any]>
+  update?(query: any): Promise<[any]>
+  delete?(query: any): Promise<[any]>
+}

package/src/integrations/base/datasourcePlus.ts

@@ -0,0 +1,9 @@
+import { Table } from "../../definitions/common"
+import { IntegrationBase } from "./IntegrationBase"
+
+export interface DatasourcePlus extends IntegrationBase {
+  tables: Record<string, Table>
+  schemaErrors: Record<string, string>
+
+  buildSchema(datasourceId: string, entities: Record<string, Table>): any
+}