@budibase/server 0.9.189 → 0.9.190-alpha.11

package/src/api/routes/table.js

@@ -27,13 +27,110 @@ function generateSaveValidator() {
 }
 
 router
+  /**
+   * @api {get} /api/tables Fetch all tables
+   * @apiName Fetch all tables
+   * @apiGroup tables
+   * @apiPermission table read access
+   * @apiDescription This endpoint retrieves all of the tables which have been created in
+   * an app. This includes all of the external and internal tables; to tell the difference
+   * between these look for the "type" property on each table, either being "internal" or "external".
+   *
+   * @apiSuccess {object[]} body The response body will be the list of tables that was found - as
+   * this does not take any parameters the only error scenario is no access.
+   */
   .get("/api/tables", authorized(BUILDER), tableController.fetch)
+  /**
+   * @api {get} /api/tables/:id Fetch a single table
+   * @apiName Fetch a single table
+   * @apiGroup tables
+   * @apiPermission table read access
+   * @apiDescription Retrieves a single table this could be be internal or external based on
+   * the provided table ID.
+   *
+   * @apiParam {string} id The ID of the table which is to be retrieved.
+   *
+   * @apiSuccess {object[]} body The response body will be the table that was found.
+   */
   .get(
     "/api/tables/:id",
     paramResource("id"),
     authorized(PermissionTypes.TABLE, PermissionLevels.READ),
     tableController.find
   )
+  /**
+   * @api {post} /api/tables Save a table
+   * @apiName Save a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription Create or update a table with this endpoint, this will function for both internal
+   * external tables.
+   *
+   * @apiParam (Body) {string} [_id] If updating an existing table then the ID of the table must be specified.
+   * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
+   * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
+   * this will default to internal.
+   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the data source ID. If
+   * building an internal table this does not need to be set, although it will be returned as "bb_internal".
+   * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
+   * supply the table structure to this endpoint with the name changed.
+   * @apiParam (Body) {object} schema A key value object which has all of the columns in the table as the keys in this
+   * object. For each column a "type" and "constraints" must be specified, with some types requiring further information.
+   * More information about the schema structure can be found in the Typescript definitions.
+   * @apiParam (Body) {string} [primaryDisplay] The name of the column which should be used when displaying rows
+   * from this table as relationships.
+   * @apiParam (Body) {object[]} [indexes] Specifies the search indexes - this is deprecated behaviour with the introduction
+   * of lucene indexes. This functionality is only available for internal tables.
+   * @apiParam (Body) {object} [_rename] If a column is to be renamed then the "old" column name should be set in this
+   * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
+   * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
+   * the rows in the table. This functionality is only available for internal tables.
+   * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
+   * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
+   * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
+   * or for external tables.
+   *
+   * @apiParamExample {json} Example:
+   * {
+   *   "_id": "ta_05541307fa0f4044abee071ca2a82119",
+   *   "_rev": "10-0fbe4e78f69b255d79f1017e2eeef807",
+   *   "type": "internal",
+   *   "views": {},
+   *   "name": "tableName",
+   *   "schema": {
+   *     "column": {
+   *       "type": "string",
+   *       "constraints": {
+   *         "type": "string",
+   *         "length": {
+   *           "maximum": null
+   *         },
+   *         "presence": false
+   *       },
+   *       "name": "column"
+   *     },
+   *   },
+   *   "primaryDisplay": "column",
+   *   "indexes": [],
+   *   "sourceId": "bb_internal",
+   *   "_rename": {
+   *     "old": "columnName",
+   *     "updated": "newColumnName",
+   *   },
+   *   "dataImport": {
+   *     "csvString": "column\nvalue",
+   *     "primaryDisplay": "column",
+   *     "schema": {
+   *       "column": {
+   *         "type": "string"
+   *       }
+   *     }
+   *   }
+   * }
+   *
+   * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
+   * saved to the database.
+   */
   .post(
     "/api/tables",
     // allows control over updating a table
@@ -42,16 +139,73 @@ router
     generateSaveValidator(),
     tableController.save
   )
+  /**
+   * @api {post} /api/tables/csv/validate Validate a CSV for a table
+   * @apiName Validate a CSV for a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
+   * converted into a Budibase schema; this endpoint does this.
+   *
+   * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
+   * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
+   * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
+   * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
+   * using a number instead of strings.
+   * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
+   * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
+   * importing a CSV to an existing table only fields that are present on the table can be imported.
+   *
+   * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
+   * the CSV - this will be in the same format used for table schema.s
+   */
   .post(
     "/api/tables/csv/validate",
     authorized(BUILDER),
     tableController.validateCSVSchema
   )
+  /**
+   * @api {post} /api/tables/:tableId/:revId Delete a table
+   * @apiName Delete a table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription This endpoint will delete a table and all of its associated data, for this reason it is
+   * quite dangerous - it will work for internal and external tables.
+   *
+   * @apiParam {string} tableId The ID of the table which is to be deleted.
+   * @apiParam {string} [revId] If deleting an internal table then the revision must also be supplied (_rev), for
+   * external tables this can simply be set to anything, e.g. "external".
+   *
+   * @apiSuccess {string} message A message stating that the table was deleted successfully.
+   */
   .delete(
     "/api/tables/:tableId/:revId",
     paramResource("tableId"),
     authorized(BUILDER),
     tableController.destroy
   )
+  /**
+   * @api {post} /api/tables/:tableId/:revId Import CSV to existing table
+   * @apiName Import CSV to existing table
+   * @apiGroup tables
+   * @apiPermission builder
+   * @apiDescription This endpoint will import data to existing tables, internal or external. It is used in combination
+   * with the CSV validation endpoint. Take the output of the CSV validation endpoint and pass it to this endpoint to
+   * import the data; please note this will only import fields that already exist on the table/match the type.
+   *
+   * @apiParam {string} tableId The ID of the table which the data should be imported to.
+   *
+   * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
+   * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
+   * turned into rows.
+   *
+   * @apiSuccess {string} message A message stating that the data was imported successfully.
+   */
+  .post(
+    "/api/tables/:tableId/import",
+    paramResource("tableId"),
+    authorized(BUILDER),
+    tableController.bulkImport
+  )
 
 module.exports = router

package/src/api/routes/tests/application.spec.js

@@ -75,7 +75,6 @@ describe("/applications", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       // should have empty packages
-      expect(res.body.screens.length).toEqual(1)
       expect(res.body.layouts.length).toEqual(2)
     })
   })
@@ -88,7 +87,6 @@ describe("/applications", () => {
         .expect("Content-Type", /json/)
         .expect(200)
       expect(res.body.application).toBeDefined()
-      expect(res.body.screens.length).toEqual(1)
       expect(res.body.layouts.length).toEqual(2)
     })
   })

package/src/api/routes/tests/datasource.spec.js

@@ -26,8 +26,8 @@ describe("/datasources", () => {
         .expect('Content-Type', /json/)
         .expect(200)
 
-      expect(res.res.statusMessage).toEqual("Datasource saved successfully.")
-      expect(res.body.name).toEqual("Test")
+      expect(res.body.datasource.name).toEqual("Test")
+      expect(res.body.errors).toBeUndefined()
     })
   })
 

package/src/api/routes/tests/misc.spec.js

@@ -75,7 +75,11 @@ describe("run misc tests", () => {
         },
       })
       const dataImport = {
-        csvString: "a,b,c,d\n1,2,3,4"
+        csvString: "a,b,c,d\n1,2,3,4",
+        schema: {},
+      }
+      for (let col of ["a", "b", "c", "d"]) {
+        dataImport.schema[col] = { type: "string" }
       }
       await tableUtils.handleDataImport(
         config.getAppId(),

package/src/api/routes/tests/query.spec.js

@@ -1,7 +1,13 @@
-// mock out postgres for this
+// Mock out postgres for this
 jest.mock("pg")
 
-const { findLastKey } = require("lodash/fp")
+// Mock isProdAppID to we can later mock the implementation and pretend we are
+// using prod app IDs
+const authDb = require("@budibase/auth/db")
+const { isProdAppID } = authDb
+const mockIsProdAppID = jest.fn(isProdAppID)
+authDb.isProdAppID = mockIsProdAppID
+
 const setup = require("./utilities")
 const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
 const { basicQuery, basicDatasource } = setup.structures
@@ -20,10 +26,12 @@ describe("/queries", () => {
   })
 
   async function createInvalidIntegration() {
-    const datasource = await config.createDatasource({datasource: {
-      ...basicDatasource().datasource,
-      source: "INVALID_INTEGRATION",
-    }})
+    const datasource = await config.createDatasource({
+      datasource: {
+        ...basicDatasource().datasource,
+        source: "INVALID_INTEGRATION",
+      },
+    })
     const query = await config.createQuery()
     return { datasource, query }
   }
@@ -97,11 +105,32 @@ describe("/queries", () => {
           .set(await config.defaultHeaders())
           .expect(200)
           .expect("Content-Type", /json/)
-        expect(res.body.fields).toBeUndefined()
-        expect(res.body.parameters).toBeUndefined()
-        expect(res.body.schema).toBeUndefined()
+        expect(res.body.fields).toBeDefined()
+        expect(res.body.parameters).toBeDefined()
+        expect(res.body.schema).toBeDefined()
       })
     })
+
+    it("should remove sensitive info for prod apps", async () => {
+      // Mock isProdAppID to pretend we are using a prod app
+      mockIsProdAppID.mockClear()
+      mockIsProdAppID.mockImplementation(() => true)
+
+      const query = await config.createQuery()
+      const res = await request
+        .get(`/api/queries/${query._id}`)
+        .set(await config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body._id).toEqual(query._id)
+      expect(res.body.fields).toBeUndefined()
+      expect(res.body.parameters).toBeUndefined()
+      expect(res.body.schema).toBeDefined()
+
+      // Reset isProdAppID mock
+      expect(mockIsProdAppID).toHaveBeenCalledTimes(1)
+      mockIsProdAppID.mockImplementation(isProdAppID)
+    })
   })
 
   describe("destroy", () => {

package/src/api/routes/tests/screen.spec.js

@@ -21,7 +21,7 @@ describe("/screens", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(res.body.length).toEqual(2)
+      expect(res.body.length).toEqual(1)
       expect(res.body.some(s => s._id === screen._id)).toEqual(true)
     })
 

package/src/api/routes/user.js

@@ -34,5 +34,10 @@ router
     authorized(PermissionTypes.USER, PermissionLevels.WRITE),
     controller.destroyMetadata
   )
+  .post(
+    "/api/users/metadata/sync/:id",
+    authorized(PermissionTypes.USER, PermissionLevels.WRITE),
+    controller.syncUser
+  )
 
 module.exports = router

package/src/app.ts

@@ -41,16 +41,8 @@ app.use(
 )
 
 if (!env.isTest()) {
-  const bullApp = bullboard.init()
-  app.use(async (ctx: ExtendableContext, next: () => any) => {
-    if (ctx.path.startsWith(bullboard.pathPrefix)) {
-      ctx.status = 200
-      ctx.respond = false
-      bullApp(ctx.req, ctx.res)
-    } else {
-      await next()
-    }
-  })
+  const plugin = bullboard.init()
+  app.use(plugin)
 }
 
 app.context.eventEmitter = eventEmitter

package/src/automations/bullboard.js

@@ -1,6 +1,6 @@
-const { createBullBoard } = require("bull-board")
-const { BullAdapter } = require("bull-board/bullAdapter")
-const express = require("express")
+const { createBullBoard } = require("@bull-board/api")
+const { BullAdapter } = require("@bull-board/api/bullAdapter")
+const { KoaAdapter } = require("@bull-board/koa")
 const env = require("../environment")
 const Queue = env.isTest()
   ? require("../utilities/queue/inMemoryQueue")
@@ -9,23 +9,40 @@ const { JobQueues } = require("../constants")
 const { utils } = require("@budibase/auth/redis")
 const { opts, redisProtocolUrl } = utils.getRedisOptions()
 
-const redisConfig = redisProtocolUrl || { redis: opts }
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, redisConfig)
+const CLEANUP_PERIOD_MS = 60 * 1000
+const queueConfig = redisProtocolUrl || { redis: opts }
+let cleanupInternal = null
 
-exports.pathPrefix = "/bulladmin"
+let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
+
+async function cleanup() {
+  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
+}
+
+const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  const expressApp = express()
+  // cleanup the events every 5 minutes
+  if (!cleanupInternal) {
+    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
+    // fire off an initial cleanup
+    cleanup().catch(err => {
+      console.error(`Unable to cleanup automation queue initially - ${err}`)
+    })
+  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
+  const serverAdapter = new KoaAdapter()
   for (let queue of queues) {
     adapters.push(new BullAdapter(queue))
   }
-  const { router } = createBullBoard(adapters)
-
-  expressApp.use(exports.pathPrefix, router)
-  return expressApp
+  createBullBoard({
+    queues: adapters,
+    serverAdapter,
+  })
+  serverAdapter.setBasePath(PATH_PREFIX)
+  return serverAdapter.registerPlugin()
 }
 
 exports.queue = automationQueue

package/src/automations/tests/automation.spec.js

@@ -1,5 +1,5 @@
 jest.mock("../../utilities/usageQuota")
-jest.mock("../thread")
+jest.mock("../../threads/automation")
 jest.mock("../../utilities/redis", () => ({
   init: jest.fn(),
   checkTestFlag: () => {
@@ -11,8 +11,7 @@ jest.spyOn(global.console, "error")
 
 require("../../environment")
 const automation = require("../index")
-const usageQuota = require("../../utilities/usageQuota")
-const thread = require("../thread")
+const thread = require("../../threads/automation")
 const triggers = require("../triggers")
 const { basicAutomation } = require("../../tests/utilities/structures")
 const { wait } = require("../../utilities")
@@ -62,7 +61,7 @@ describe("Run through some parts of the automations system", () => {
           }
         }
       }
-    }))
+    }), expect.any(Function))
   })
 
   it("should be able to clean inputs with the utilities", () => {

package/src/automations/triggers.js

@@ -11,6 +11,10 @@ const utils = require("./utils")
 const env = require("../environment")
 
 const TRIGGER_DEFINITIONS = definitions
+const JOB_OPTS = {
+  removeOnComplete: true,
+  removeOnFail: true,
+}
 
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
@@ -47,7 +51,7 @@ async function queueRelevantRowAutomations(event, eventType) {
       automationTrigger.inputs &&
       automationTrigger.inputs.tableId === event.row.tableId
     ) {
-      await queue.add({ automation, event })
+      await queue.add({ automation, event }, JOB_OPTS)
     }
   }
 }
@@ -86,7 +90,7 @@ exports.externalTrigger = async function (
     automation.definition.trigger != null &&
     automation.definition.trigger.stepId === definitions.APP.stepId &&
     automation.definition.trigger.stepId === "APP" &&
-    !checkTestFlag(automation._id)
+    !(await checkTestFlag(automation._id))
   ) {
     // values are likely to be submitted as strings, so we shall convert to correct type
     const coercedFields = {}
@@ -100,7 +104,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data)
+    return queue.add(data, JOB_OPTS)
   }
 }
 

package/src/automations/utils.js

@@ -1,4 +1,4 @@
-const runner = require("./thread")
+const { Thread, ThreadType } = require("../threads")
 const { definitions } = require("./triggerInfo")
 const webhooks = require("../api/controllers/webhook")
 const CouchDB = require("../db")
@@ -10,11 +10,12 @@ const { getDeployedAppID } = require("@budibase/auth/db")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
+const Runner = new Thread(ThreadType.AUTOMATION)
 
 exports.processEvent = async job => {
   try {
     // need to actually await these so that an error can be captured properly
-    return await runner(job)
+    return await Runner.run(job)
   } catch (err) {
     console.error(
       `${job.data.automation.appId} automation ${job.data.automation._id} was unable to run - ${err}`
@@ -162,3 +163,12 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
   }
   return newAuto
 }
+
+/**
+ * When removing an app/unpublishing it need to make sure automations are cleaned up (cron).
+ * @param appId {string} the app that is being removed.
+ * @return {Promise<void>} clean is complete if this succeeds.
+ */
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
+}

package/src/constants/index.js

@@ -45,6 +45,13 @@ exports.FieldTypes = {
   INTERNAL: "internal",
 }
 
+exports.SwitchableTypes = [
+  exports.FieldTypes.STRING,
+  exports.FieldTypes.OPTIONS,
+  exports.FieldTypes.NUMBER,
+  exports.FieldTypes.BOOLEAN,
+]
+
 exports.RelationshipTypes = {
   ONE_TO_MANY: "one-to-many",
   MANY_TO_ONE: "many-to-one",
@@ -62,6 +69,10 @@ exports.DataSourceOperation = {
   READ: "READ",
   UPDATE: "UPDATE",
   DELETE: "DELETE",
+  BULK_CREATE: "BULK_CREATE",
+  CREATE_TABLE: "CREATE_TABLE",
+  UPDATE_TABLE: "UPDATE_TABLE",
+  DELETE_TABLE: "DELETE_TABLE",
 }
 
 exports.SortDirection = {
@@ -152,5 +163,16 @@ exports.MetadataTypes = {
   AUTOMATION_TEST_HISTORY: "automationTestHistory",
 }
 
+exports.InvalidColumns = {
+  ID: "_id",
+  REV: "_rev",
+  TABLE_ID: "tableId",
+}
+
+exports.BuildSchemaErrors = {
+  NO_KEY: "no_key",
+  INVALID_COLUMN: "invalid_column",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/definitions/common.ts

@@ -17,6 +17,11 @@ export interface FieldSchema {
   autocolumn?: boolean
   throughFrom?: string
   throughTo?: string
+  main?: boolean
+  meta?: {
+    toTable: string
+    toKey: string
+  }
   constraints?: {
     type?: string
     email?: boolean
@@ -36,11 +41,12 @@ export interface TableSchema {
 export interface Table extends Base {
   type?: string
   views?: {}
-  name?: string
+  name: string
   primary?: string[]
   schema: TableSchema
   primaryDisplay?: string
   sourceId?: string
+  constrained?: string[]
 }
 
 export interface Row extends Base {

package/src/definitions/datasource.ts

@@ -1,10 +1,14 @@
-import { Table } from "./common"
+import { Row, Table } from "./common"
 
 export enum Operation {
   CREATE = "CREATE",
   READ = "READ",
   UPDATE = "UPDATE",
   DELETE = "DELETE",
+  BULK_CREATE = "BULK_CREATE",
+  CREATE_TABLE = "CREATE_TABLE",
+  UPDATE_TABLE = "UPDATE_TABLE",
+  DELETE_TABLE = "DELETE_TABLE",
 }
 
 export enum SortDirection {
@@ -42,6 +46,7 @@ export enum SourceNames {
   MYSQL = "MYSQL",
   ARANGODB = "ARANGODB",
   REST = "REST",
+  ORACLE = "ORACLE",
 }
 
 export enum IncludeRelationships {
@@ -49,6 +54,17 @@ export enum IncludeRelationships {
   EXCLUDE = 0,
 }
 
+export enum FilterTypes {
+  STRING = "string",
+  FUZZY = "fuzzy",
+  RANGE = "range",
+  EQUAL = "equal",
+  NOT_EQUAL = "notEqual",
+  EMPTY = "empty",
+  NOT_EMPTY = "notEmpty",
+  ONE_OF = "oneOf",
+}
+
 export interface QueryDefinition {
   type: QueryTypes
   displayName?: string
@@ -116,7 +132,7 @@ export interface SortJson {
 
 export interface PaginationJson {
   limit: number
-  page: string | number
+  page?: string | number
 }
 
 export interface RelationshipsJson {
@@ -141,9 +157,11 @@ export interface QueryJson {
   filters?: SearchFilters
   sort?: SortJson
   paginate?: PaginationJson
-  body?: object
+  body?: Row | Row[]
+  table?: Table
   meta?: {
     table?: Table
+    tables?: Record<string, Table>
   }
   extra?: {
     idFilter?: SearchFilters

package/src/environment.js

@@ -23,6 +23,8 @@ if (!LOADED && isDev() && !isTest()) {
   LOADED = true
 }
 
+let inThread = false
+
 module.exports = {
   // important
   PORT: process.env.PORT,
@@ -62,6 +64,7 @@ module.exports = {
   USERID_API_KEY: process.env.USERID_API_KEY,
   DEPLOYMENT_CREDENTIALS_URL: process.env.DEPLOYMENT_CREDENTIALS_URL,
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
+  DISABLE_THREADING: process.env.DISABLE_THREADING,
   _set(key, value) {
     process.env[key] = value
     module.exports[key] = value
@@ -72,6 +75,18 @@ module.exports = {
   isProd: () => {
     return !isDev()
   },
+  // used to check if already in a thread, don't thread further
+  setInThread: () => {
+    inThread = true
+  },
+  isInThread: () => {
+    return inThread
+  },
+}
+
+// threading can cause memory issues with node-ts in development
+if (isDev() && module.exports.DISABLE_THREADING == null) {
+  module.exports._set("DISABLE_THREADING", "1")
 }
 
 // clean up any environment variable edge cases

package/src/integrations/airtable.ts

@@ -3,6 +3,7 @@ import {
   DatasourceFieldTypes,
   QueryTypes,
 } from "../definitions/datasource"
+import { IntegrationBase } from "./base/IntegrationBase"
 
 module AirtableModule {
   const Airtable = require("airtable")
@@ -73,7 +74,7 @@ module AirtableModule {
     },
   }
 
-  class AirtableIntegration {
+  class AirtableIntegration implements IntegrationBase {
     private config: AirtableConfig
     private client: any