@budibase/server 2.0.30-alpha.14 → 2.0.30-alpha.16

package/src/api/controllers/table/utils.ts

@@ -1,11 +1,5 @@
 import { transform } from "../../../utilities/csvParser"
-import {
-  getRowParams,
-  generateRowID,
-  InternalTables,
-  getTableParams,
-  BudibaseInternalDB,
-} from "../../../db/utils"
+import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
 import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import {
@@ -17,11 +11,6 @@ import {
   SwitchableTypes,
   CanSwitchTypes,
 } from "../../../constants"
-import {
-  isExternalTable,
-  breakExternalTableId,
-  isSQL,
-} from "../../../integrations/utils"
 import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -256,46 +245,6 @@ class TableSaveFunctions {
   }
 }
 
-export async function getAllInternalTables() {
-  const db = getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
-}
-
-export async function getAllExternalTables(datasourceId: any) {
-  const db = getAppDB()
-  const datasource = await db.get(datasourceId)
-  if (!datasource || !datasource.entities) {
-    throw "Datasource is not configured fully."
-  }
-  return datasource.entities
-}
-
-export async function getExternalTable(datasourceId: any, tableName: any) {
-  const entities = await getAllExternalTables(datasourceId)
-  return entities[tableName]
-}
-
-export async function getTable(tableId: any) {
-  const db = getAppDB()
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
-  } else {
-    return db.get(tableId)
-  }
-}
-
 export async function checkForViewUpdates(
   table: any,
   rename: any,

package/src/api/controllers/view/index.js

@@ -3,12 +3,12 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
-const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
 const { DocumentType } = require("../../../db/utils")
 const { cloneDeep, isEqual } = require("lodash")
+const sdk = require("../../../sdk")
 
 exports.fetch = async ctx => {
   ctx.body = await getViews()
@@ -144,7 +144,7 @@ exports.exportView = async ctx => {
 
   let schema = view && view.meta && view.meta.schema
   const tableId = ctx.params.tableId || view.meta.tableId
-  const table = await getTable(tableId)
+  const table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema
   }

package/src/api/routes/backup.ts

@@ -0,0 +1,10 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/backup"
+import authorized from "../../middleware/authorized"
+import { BUILDER } from "@budibase/backend-core/permissions"
+
+const router = new Router()
+
+router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump)
+
+export default router

package/src/api/routes/index.ts

@@ -25,11 +25,17 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import Router from "@koa/router"
+import { api } from "@budibase/pro"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
-export const mainRoutes = [
+const appBackupRoutes = api.appBackups
+const scheduleRoutes = api.schedules
+export const mainRoutes: Router[] = [
+  appBackupRoutes,
+  backupRoutes,
   authRoutes,
   deployRoutes,
   layoutRoutes,
@@ -49,14 +55,14 @@ export const mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  backupRoutes,
   metadataRoutes,
   devRoutes,
   cloudRoutes,
-  // these need to be handled last as they still use /api/:tableId
-  // this could be breaking as koa may recognise other routes as this
-  tableRoutes,
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  scheduleRoutes,
+  // these need to be handled last as they still use /api/:tableId
+  // this could be breaking as koa may recognise other routes as this
+  tableRoutes,
 ]

package/src/api/routes/tests/backup.spec.js

@@ -21,7 +21,7 @@ describe("/backups", () => {
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()
-      expect(res.text.includes(`"db_name":"${config.getAppId()}"`)).toEqual(true)
+      expect(res.headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported.mock.calls.length).toBe(1)
     })
 

package/src/api/routes/tests/{deployment.spec.js → deployment.spec.ts}

@@ -1,5 +1,5 @@
-const setup = require("./utilities")
-const { events } = require("@budibase/backend-core")
+import setup from "./utilities"
+import { events } from "@budibase/backend-core"
 
 describe("/deployments", () => {
   let request = setup.getRequest()
@@ -19,7 +19,7 @@ describe("/deployments", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(events.app.published.mock.calls.length).toBe(1)
+      expect((events.app.published as jest.Mock).mock.calls.length).toBe(1)
     })
   })
-})
+})

package/src/app.ts

@@ -37,6 +37,8 @@ import {
 } from "./utilities/workerRequests"
 import { watch } from "./watch"
 import { initialise as initialiseWebsockets } from "./websocket"
+import sdk from "./sdk"
+import * as pro from "@budibase/pro"
 
 const app = new Koa()
 
@@ -102,12 +104,25 @@ server.on("close", async () => {
   }
 })
 
+const initPro = async () => {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
 module.exports = server.listen(env.PORT || 0, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   env._set("PORT", server.address().port)
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  await initPro()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/automations/automationUtils.js

@@ -1,10 +1,10 @@
-const { getTable } = require("../api/controllers/table/utils")
 const {
   findHBSBlocks,
   decodeJSBinding,
   isJSBinding,
   encodeJSBinding,
 } = require("@budibase/string-templates")
+const sdk = require("../sdk")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -64,7 +64,7 @@ exports.cleanInputValues = (inputs, schema) => {
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
 exports.cleanUpRow = async (tableId, row) => {
-  let table = await getTable(tableId)
+  let table = await sdk.tables.getTable(tableId)
   return exports.cleanInputValues(row, { properties: table.schema })
 }
 

package/src/automations/bullboard.js

@@ -1,37 +1,17 @@
 const { createBullBoard } = require("@bull-board/api")
 const { BullAdapter } = require("@bull-board/api/bullAdapter")
 const { KoaAdapter } = require("@bull-board/koa")
-const env = require("../environment")
-const Queue = env.isTest()
-  ? require("../utilities/queue/inMemoryQueue")
-  : require("bull")
-const { JobQueues } = require("../constants")
-const { utils } = require("@budibase/backend-core/redis")
-const { opts, redisProtocolUrl } = utils.getRedisOptions()
-const listeners = require("./listeners")
+const { queue } = require("@budibase/backend-core")
+const automation = require("../threads/automation")
 
-const CLEANUP_PERIOD_MS = 60 * 1000
-const queueConfig = redisProtocolUrl || { redis: opts }
-let cleanupInternal = null
-
-let automationQueue = new Queue(JobQueues.AUTOMATIONS, queueConfig)
-listeners.addListeners(automationQueue)
-
-async function cleanup() {
-  await automationQueue.clean(CLEANUP_PERIOD_MS, "completed")
-}
+let automationQueue = queue.createQueue(
+  queue.JobQueue.AUTOMATION,
+  automation.removeStalled
+)
 
 const PATH_PREFIX = "/bulladmin"
 
 exports.init = () => {
-  // cleanup the events every 5 minutes
-  if (!cleanupInternal) {
-    cleanupInternal = setInterval(cleanup, CLEANUP_PERIOD_MS)
-    // fire off an initial cleanup
-    cleanup().catch(err => {
-      console.error(`Unable to cleanup automation queue initially - ${err}`)
-    })
-  }
   // Set up queues for bull board admin
   const queues = [automationQueue]
   const adapters = []
@@ -48,12 +28,7 @@ exports.init = () => {
 }
 
 exports.shutdown = async () => {
-  if (automationQueue) {
-    clearInterval(cleanupInternal)
-    await automationQueue.close()
-    automationQueue = null
-  }
-  console.log("Bull shutdown")
+  await queue.shutdown()
 }
 
-exports.queue = automationQueue
+exports.automationQueue = automationQueue

package/src/automations/index.js

@@ -1,5 +1,5 @@
 const { processEvent } = require("./utils")
-const { queue, shutdown } = require("./bullboard")
+const { automationQueue, shutdown } = require("./bullboard")
 const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
@@ -8,7 +8,7 @@ const { ACTION_DEFINITIONS } = require("./actions")
  */
 exports.init = async function () {
   // this promise will not complete
-  const promise = queue.process(async job => {
+  const promise = automationQueue.process(async job => {
     await processEvent(job)
   })
   // on init we need to trigger any reboot automations
@@ -17,13 +17,13 @@ exports.init = async function () {
 }
 
 exports.getQueues = () => {
-  return [queue]
+  return [automationQueue]
 }
 
 exports.shutdown = () => {
   return shutdown()
 }
 
-exports.queue = queue
+exports.automationQueue = automationQueue
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS
 exports.ACTION_DEFINITIONS = ACTION_DEFINITIONS

package/src/automations/triggers.js

@@ -4,7 +4,7 @@ const { coerce } = require("../utilities/rowProcessor")
 const { definitions } = require("./triggerInfo")
 const { isDevAppID } = require("../db/utils")
 // need this to call directly, so we can get a response
-const { queue } = require("./bullboard")
+const { automationQueue } = require("./bullboard")
 const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
@@ -56,7 +56,7 @@ async function queueRelevantRowAutomations(event, eventType) {
         automationTrigger.inputs &&
         automationTrigger.inputs.tableId === event.row.tableId
       ) {
-        await queue.add({ automation, event }, JOB_OPTS)
+        await automationQueue.add({ automation, event }, JOB_OPTS)
       }
     }
   })
@@ -110,7 +110,7 @@ exports.externalTrigger = async function (
   if (getResponses) {
     return utils.processEvent({ data })
   } else {
-    return queue.add(data, JOB_OPTS)
+    return automationQueue.add(data, JOB_OPTS)
   }
 }
 
@@ -136,7 +136,7 @@ exports.rebootTrigger = async () => {
               timestamp: Date.now(),
             },
           }
-          rebootEvents.push(queue.add(job, JOB_OPTS))
+          rebootEvents.push(automationQueue.add(job, JOB_OPTS))
         }
       }
       await Promise.all(rebootEvents)

package/src/automations/utils.ts

@@ -1,7 +1,7 @@
 import { Thread, ThreadType } from "../threads"
 import { definitions } from "./triggerInfo"
 import * as webhooks from "../api/controllers/webhook"
-import { queue } from "./bullboard"
+import { automationQueue } from "./bullboard"
 import newid from "../db/newid"
 import { updateEntityMetadata } from "../utilities"
 import { MetadataTypes, WebhookType } from "../constants"
@@ -79,21 +79,25 @@ export function removeDeprecated(definitions: any) {
 // end the repetition and the job itself
 export async function disableAllCrons(appId: any) {
   const promises = []
-  const jobs = await queue.getRepeatableJobs()
+  const jobs = await automationQueue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
-      promises.push(queue.removeRepeatableByKey(job.key))
+      promises.push(automationQueue.removeRepeatableByKey(job.key))
       if (job.id) {
-        promises.push(queue.removeJobs(job.id))
+        promises.push(automationQueue.removeJobs(job.id))
       }
     }
   }
   return Promise.all(promises)
 }
 
-export async function disableCron(jobId: string, jobKey: string) {
-  await queue.removeRepeatableByKey(jobKey)
-  await queue.removeJobs(jobId)
+export async function disableCronById(jobId: number | string) {
+  const repeatJobs = await automationQueue.getRepeatableJobs()
+  for (let repeatJob of repeatJobs) {
+    if (repeatJob.id === jobId) {
+      await automationQueue.removeRepeatableByKey(repeatJob.key)
+    }
+  }
   console.log(`jobId=${jobId} disabled`)
 }
 
@@ -141,7 +145,7 @@ export async function enableCronTrigger(appId: any, automation: Automation) {
   ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job: any = await queue.add(
+    const job: any = await automationQueue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },

package/src/constants/index.js

@@ -1,10 +1,6 @@
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { UserStatus } = require("@budibase/backend-core/constants")
-const { ObjectStoreBuckets } = require("@budibase/backend-core/objectStore")
-
-exports.JobQueues = {
-  AUTOMATIONS: "automationQueue",
-}
+const { objectStore } = require("@budibase/backend-core")
 
 const FilterTypes = {
   STRING: "string",
@@ -211,6 +207,6 @@ exports.AutomationErrors = {
 }
 
 // pass through the list from the auth/core lib
-exports.ObjectStoreBuckets = ObjectStoreBuckets
+exports.ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 
 exports.MAX_AUTOMATION_RECURRING_ERRORS = 5

package/src/db/utils.js

@@ -1,6 +1,7 @@
 const newid = require("./newid")
 const {
-  DocumentType: CoreDocTypes,
+  DocumentType: CoreDocType,
+  InternalTable,
   getRoleParams,
   generateRoleID,
   APP_DEV_PREFIX,
@@ -13,6 +14,12 @@ const {
   generateAppID,
   getQueryIndex,
   ViewName,
+  getDocParams,
+  getRowParams,
+  generateRowID,
+  getUserMetadataParams,
+  generateUserMetadataID,
+  getGlobalIDFromUserMetadataID,
 } = require("@budibase/backend-core/db")
 
 const UNICODE_MAX = "\ufff0"
@@ -23,28 +30,7 @@ const AppStatus = {
   DEPLOYED: "published",
 }
 
-const DocumentType = {
-  ...CoreDocTypes,
-  TABLE: "ta",
-  ROW: "ro",
-  USER: "us",
-  AUTOMATION: "au",
-  LINK: "li",
-  WEBHOOK: "wh",
-  INSTANCE: "inst",
-  LAYOUT: "layout",
-  SCREEN: "screen",
-  QUERY: "query",
-  DEPLOYMENTS: "deployments",
-  METADATA: "metadata",
-  MEM_VIEW: "view",
-  USER_FLAG: "flag",
-  AUTOMATION_METADATA: "meta_au",
-}
-
-const InternalTables = {
-  USER_METADATA: "ta_users",
-}
+const DocumentType = CoreDocType
 
 const SearchIndexes = {
   ROWS: "rows",
@@ -64,11 +50,11 @@ exports.APP_PREFIX = APP_PREFIX
 exports.APP_DEV_PREFIX = APP_DEV_PREFIX
 exports.isDevAppID = isDevAppID
 exports.isProdAppID = isProdAppID
-exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
+exports.USER_METDATA_PREFIX = `${DocumentType.ROW}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
+exports.LINK_USER_METADATA_PREFIX = `${DocumentType.LINK}${SEPARATOR}${InternalTable.USER_METADATA}${SEPARATOR}`
 exports.TABLE_ROW_PREFIX = `${DocumentType.ROW}${SEPARATOR}${DocumentType.TABLE}`
 exports.ViewName = ViewName
-exports.InternalTables = InternalTables
+exports.InternalTables = InternalTable
 exports.DocumentType = DocumentType
 exports.SEPARATOR = SEPARATOR
 exports.UNICODE_MAX = UNICODE_MAX
@@ -77,36 +63,15 @@ exports.AppStatus = AppStatus
 exports.BudibaseInternalDB = BudibaseInternalDB
 exports.generateAppID = generateAppID
 exports.generateDevAppID = getDevelopmentAppID
-
 exports.generateRoleID = generateRoleID
 exports.getRoleParams = getRoleParams
-
 exports.getQueryIndex = getQueryIndex
-
-/**
- * If creating DB allDocs/query params with only a single top level ID this can be used, this
- * is usually the case as most of our docs are top level e.g. tables, automations, users and so on.
- * More complex cases such as link docs and rows which have multiple levels of IDs that their
- * ID consists of need their own functions to build the allDocs parameters.
- * @param {string} docType The type of document which input params are being built for, e.g. user,
- * link, app, table and so on.
- * @param {string|null} docId The ID of the document minus its type - this is only needed if looking
- * for a singular document.
- * @param {object} otherProps Add any other properties onto the request, e.g. include_docs.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-function getDocParams(docType, docId = null, otherProps = {}) {
-  if (docId == null) {
-    docId = ""
-  }
-  return {
-    ...otherProps,
-    startkey: `${docType}${SEPARATOR}${docId}`,
-    endkey: `${docType}${SEPARATOR}${docId}${UNICODE_MAX}`,
-  }
-}
-
 exports.getDocParams = getDocParams
+exports.getRowParams = getRowParams
+exports.generateRowID = generateRowID
+exports.getUserMetadataParams = getUserMetadataParams
+exports.generateUserMetadataID = generateUserMetadataID
+exports.getGlobalIDFromUserMetadataID = getGlobalIDFromUserMetadataID
 
 /**
  * Gets parameters for retrieving tables, this is a utility function for the getDocParams function.
@@ -123,24 +88,6 @@ exports.generateTableID = () => {
   return `${DocumentType.TABLE}${SEPARATOR}${newid()}`
 }
 
-/**
- * Gets the DB allDocs/query params for retrieving a row.
- * @param {string|null} tableId The table in which the rows have been stored.
- * @param {string|null} rowId The ID of the row which is being specifically queried for. This can be
- * left null to get all the rows in the table.
- * @param {object} otherProps Any other properties to add to the request.
- * @returns {object} Parameters which can then be used with an allDocs request.
- */
-exports.getRowParams = (tableId = null, rowId = null, otherProps = {}) => {
-  if (tableId == null) {
-    return getDocParams(DocumentType.ROW, null, otherProps)
-  }
-
-  const endOfKey = rowId == null ? `${tableId}${SEPARATOR}` : rowId
-
-  return getDocParams(DocumentType.ROW, endOfKey, otherProps)
-}
-
 /**
  * Given a row ID this will find the table ID within it (only works for internal tables).
  * @param {string} rowId The ID of the row.
@@ -153,44 +100,6 @@ exports.getTableIDFromRowID = rowId => {
   return `${DocumentType.TABLE}${SEPARATOR}${components[0]}`
 }
 
-/**
- * Gets a new row ID for the specified table.
- * @param {string} tableId The table which the row is being created for.
- * @param {string|null} id If an ID is to be used then the UUID can be substituted for this.
- * @returns {string} The new ID which a row doc can be stored under.
- */
-exports.generateRowID = (tableId, id = null) => {
-  id = id || newid()
-  return `${DocumentType.ROW}${SEPARATOR}${tableId}${SEPARATOR}${id}`
-}
-
-/**
- * Gets parameters for retrieving users, this is a utility function for the getDocParams function.
- */
-exports.getUserMetadataParams = (userId = null, otherProps = {}) => {
-  return exports.getRowParams(InternalTables.USER_METADATA, userId, otherProps)
-}
-
-/**
- * Generates a new user ID based on the passed in global ID.
- * @param {string} globalId The ID of the global user.
- * @returns {string} The new user ID which the user doc can be stored under.
- */
-exports.generateUserMetadataID = globalId => {
-  return exports.generateRowID(InternalTables.USER_METADATA, globalId)
-}
-
-/**
- * Breaks up the ID to get the global ID.
- */
-exports.getGlobalIDFromUserMetadataID = id => {
-  const prefix = `${DocumentType.ROW}${SEPARATOR}${InternalTables.USER_METADATA}${SEPARATOR}`
-  if (!id || !id.includes(prefix)) {
-    return id
-  }
-  return id.split(prefix)[1]
-}
-
 /**
  * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
  */

package/src/definitions/automations.ts

@@ -27,18 +27,6 @@ export interface TriggerOutput {
   timestamp?: number
 }
 
-export interface AutomationEvent {
-  data: {
-    automation: Automation
-    event: any
-  }
-  opts?: {
-    repeat?: {
-      jobId: string
-    }
-  }
-}
-
 export interface AutomationContext extends AutomationResults {
   steps: any[]
   trigger: any

package/src/migrations/functions/backfill/app/tables.ts

@@ -1,18 +1,11 @@
 import { events } from "@budibase/backend-core"
-import { getTableParams } from "../../../../db/utils"
-import { Table } from "@budibase/types"
+import sdk from "../../../../sdk"
 
-const getTables = async (appDb: any): Promise<Table[]> => {
-  const response = await appDb.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return response.rows.map((row: any) => row.doc)
-}
-
-export const backfill = async (appDb: any, timestamp: string | number) => {
-  const tables = await getTables(appDb)
+export const backfill = async (
+  appDb: PouchDB.Database,
+  timestamp: string | number
+) => {
+  const tables = await sdk.tables.getAllInternalTables(appDb)
 
   for (const table of tables) {
     await events.table.created(table, timestamp)

package/src/sdk/app/backups/constants.ts

@@ -0,0 +1,2 @@
+export const DB_EXPORT_FILE = "db.txt"
+export const GLOBAL_DB_EXPORT_FILE = "global.txt"