@budibase/server 2.1.44 → 2.1.46-alpha.0

package/src/api/controllers/{automation.js → automation.ts}

@@ -1,26 +1,21 @@
-const actions = require("../../automations/actions")
-const triggers = require("../../automations/triggers")
-const {
+import * as actions from "../../automations/actions"
+import * as triggers from "../../automations/triggers"
+import {
   getAutomationParams,
   generateAutomationID,
   DocumentType,
-} = require("../../db/utils")
-const {
+} from "../../db/utils"
+import {
   checkForWebhooks,
   updateTestHistory,
   removeDeprecated,
-} = require("../../automations/utils")
-const { deleteEntityMetadata } = require("../../utilities")
-const { MetadataTypes } = require("../../constants")
-const { setTestFlag, clearTestFlag } = require("../../utilities/redis")
-const {
-  getAppDB,
-  getProdAppDB,
-  doInAppContext,
-} = require("@budibase/backend-core/context")
-const { events } = require("@budibase/backend-core")
-const { app } = require("@budibase/backend-core/cache")
-const { automations } = require("@budibase/pro")
+} from "../../automations/utils"
+import { deleteEntityMetadata } from "../../utilities"
+import { MetadataTypes } from "../../constants"
+import { setTestFlag, clearTestFlag } from "../../utilities/redis"
+import { context, cache, events } from "@budibase/backend-core"
+import { automations } from "@budibase/pro"
+import { Automation, BBContext } from "@budibase/types"
 
 const ACTION_DEFS = removeDeprecated(actions.ACTION_DEFINITIONS)
 const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
@@ -31,7 +26,7 @@ const TRIGGER_DEFS = removeDeprecated(triggers.TRIGGER_DEFINITIONS)
  *                       *
  *************************/
 
-async function cleanupAutomationMetadata(automationId) {
+async function cleanupAutomationMetadata(automationId: string) {
   await deleteEntityMetadata(MetadataTypes.AUTOMATION_TEST_INPUT, automationId)
   await deleteEntityMetadata(
     MetadataTypes.AUTOMATION_TEST_HISTORY,
@@ -39,7 +34,7 @@ async function cleanupAutomationMetadata(automationId) {
   )
 }
 
-function cleanAutomationInputs(automation) {
+function cleanAutomationInputs(automation: Automation) {
   if (automation == null) {
     return automation
   }
@@ -63,14 +58,14 @@ function cleanAutomationInputs(automation) {
   return automation
 }
 
-exports.create = async function (ctx) {
-  const db = getAppDB()
+export async function create(ctx: BBContext) {
+  const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
 
   // call through to update if already exists
   if (automation._id && automation._rev) {
-    return exports.update(ctx)
+    return update(ctx)
   }
 
   automation._id = generateAutomationID()
@@ -97,17 +92,23 @@ exports.create = async function (ctx) {
   }
 }
 
-const getNewSteps = (oldAutomation, automation) => {
+export function getNewSteps(oldAutomation: Automation, automation: Automation) {
   const oldStepIds = oldAutomation.definition.steps.map(s => s.id)
   return automation.definition.steps.filter(s => !oldStepIds.includes(s.id))
 }
 
-const getDeletedSteps = (oldAutomation, automation) => {
+export function getDeletedSteps(
+  oldAutomation: Automation,
+  automation: Automation
+) {
   const stepIds = automation.definition.steps.map(s => s.id)
   return oldAutomation.definition.steps.filter(s => !stepIds.includes(s.id))
 }
 
-const handleStepEvents = async (oldAutomation, automation) => {
+export async function handleStepEvents(
+  oldAutomation: Automation,
+  automation: Automation
+) {
   // new steps
   const newSteps = getNewSteps(oldAutomation, automation)
   for (let step of newSteps) {
@@ -121,8 +122,8 @@ const handleStepEvents = async (oldAutomation, automation) => {
   }
 }
 
-exports.update = async function (ctx) {
-  const db = getAppDB()
+export async function update(ctx: BBContext) {
+  const db = context.getAppDB()
   let automation = ctx.request.body
   automation.appId = ctx.appId
   const oldAutomation = await db.get(automation._id)
@@ -146,9 +147,8 @@ exports.update = async function (ctx) {
   if (oldAutoTrigger && oldAutoTrigger.id !== newAutoTrigger.id) {
     await events.automation.triggerUpdated(automation)
     await deleteEntityMetadata(
-      ctx.appId,
       MetadataTypes.AUTOMATION_TEST_INPUT,
-      automation._id
+      automation._id!
     )
   }
 
@@ -165,8 +165,8 @@ exports.update = async function (ctx) {
   }
 }
 
-exports.fetch = async function (ctx) {
-  const db = getAppDB()
+export async function fetch(ctx: BBContext) {
+  const db = context.getAppDB()
   const response = await db.allDocs(
     getAutomationParams(null, {
       include_docs: true,
@@ -175,13 +175,13 @@ exports.fetch = async function (ctx) {
   ctx.body = response.rows.map(row => row.doc)
 }
 
-exports.find = async function (ctx) {
-  const db = getAppDB()
+export async function find(ctx: BBContext) {
+  const db = context.getAppDB()
   ctx.body = await db.get(ctx.params.id)
 }
 
-exports.destroy = async function (ctx) {
-  const db = getAppDB()
+export async function destroy(ctx: BBContext) {
+  const db = context.getAppDB()
   const automationId = ctx.params.id
   const oldAutomation = await db.get(automationId)
   await checkForWebhooks({
@@ -193,14 +193,14 @@ exports.destroy = async function (ctx) {
   await events.automation.deleted(oldAutomation)
 }
 
-exports.logSearch = async function (ctx) {
+export async function logSearch(ctx: BBContext) {
   ctx.body = await automations.logs.logSearch(ctx.request.body)
 }
 
-exports.clearLogError = async function (ctx) {
+export async function clearLogError(ctx: BBContext) {
   const { automationId, appId } = ctx.request.body
-  await doInAppContext(appId, async () => {
-    const db = getProdAppDB()
+  await context.doInAppContext(appId, async () => {
+    const db = context.getProdAppDB()
     const metadata = await db.get(DocumentType.APP_METADATA)
     if (!automationId) {
       delete metadata.automationErrors
@@ -211,20 +211,20 @@ exports.clearLogError = async function (ctx) {
       delete metadata.automationErrors[automationId]
     }
     await db.put(metadata)
-    await app.invalidateAppMetadata(metadata.appId, metadata)
+    await cache.app.invalidateAppMetadata(metadata.appId, metadata)
     ctx.body = { message: `Error logs cleared.` }
   })
 }
 
-exports.getActionList = async function (ctx) {
+export async function getActionList(ctx: BBContext) {
   ctx.body = ACTION_DEFS
 }
 
-exports.getTriggerList = async function (ctx) {
+export async function getTriggerList(ctx: BBContext) {
   ctx.body = TRIGGER_DEFS
 }
 
-module.exports.getDefinitionList = async function (ctx) {
+export async function getDefinitionList(ctx: BBContext) {
   ctx.body = {
     trigger: TRIGGER_DEFS,
     action: ACTION_DEFS,
@@ -237,8 +237,8 @@ module.exports.getDefinitionList = async function (ctx) {
  *                   *
  *********************/
 
-exports.trigger = async function (ctx) {
-  const db = getAppDB()
+export async function trigger(ctx: BBContext) {
+  const db = context.getAppDB()
   let automation = await db.get(ctx.params.id)
   await triggers.externalTrigger(automation, {
     ...ctx.request.body,
@@ -250,7 +250,7 @@ exports.trigger = async function (ctx) {
   }
 }
 
-function prepareTestInput(input) {
+function prepareTestInput(input: any) {
   // prepare the test parameters
   if (input.id && input.row) {
     input.row._id = input.id
@@ -261,8 +261,8 @@ function prepareTestInput(input) {
   return input
 }
 
-exports.test = async function (ctx) {
-  const db = getAppDB()
+export async function test(ctx: BBContext) {
+  const db = context.getAppDB()
   let automation = await db.get(ctx.params.id)
   await setTestFlag(automation._id)
   const testInput = prepareTestInput(ctx.request.body)

package/src/api/controllers/{cloud.js → cloud.ts}

@@ -1,14 +1,14 @@
-const env = require("../../environment")
-const { getAllApps, getGlobalDBName } = require("@budibase/backend-core/db")
-const { getGlobalDB } = require("@budibase/backend-core/tenancy")
-const { streamFile } = require("../../utilities/fileSystem")
-const { stringToReadStream } = require("../../utilities")
-const { getDocParams, DocumentType, isDevAppID } = require("../../db/utils")
-const { create } = require("./application")
-const { join } = require("path")
-const sdk = require("../../sdk")
+import env from "../../environment"
+import { db as dbCore, tenancy } from "@budibase/backend-core"
+import { streamFile } from "../../utilities/fileSystem"
+import { stringToReadStream } from "../../utilities"
+import { getDocParams, DocumentType, isDevAppID } from "../../db/utils"
+import { create } from "./application"
+import { join } from "path"
+import { App, BBContext, Database } from "@budibase/types"
+import sdk from "../../sdk"
 
-async function createApp(appName, appDirectory) {
+async function createApp(appName: string, appDirectory: string) {
   const ctx = {
     request: {
       body: {
@@ -22,10 +22,11 @@ async function createApp(appName, appDirectory) {
       },
     },
   }
+  // @ts-ignore
   return create(ctx)
 }
 
-async function getAllDocType(db, docType) {
+async function getAllDocType(db: Database, docType: string) {
   const response = await db.allDocs(
     getDocParams(docType, null, {
       include_docs: true,
@@ -34,19 +35,19 @@ async function getAllDocType(db, docType) {
   return response.rows.map(row => row.doc)
 }
 
-exports.exportApps = async ctx => {
+export async function exportApps(ctx: BBContext) {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
-  const apps = await getAllApps({ all: true })
-  const globalDBString = await sdk.backups.exportDB(getGlobalDBName(), {
-    filter: doc => !doc._id.startsWith(DocumentType.USER),
+  const apps = (await dbCore.getAllApps({ all: true })) as App[]
+  const globalDBString = await sdk.backups.exportDB(dbCore.getGlobalDBName(), {
+    filter: (doc: any) => !doc._id.startsWith(DocumentType.USER),
   })
   // only export the dev apps as they will be the latest, the user can republish the apps
   // in their self-hosted environment
   let appMetadata = apps
-    .filter(app => isDevAppID(app.appId || app._id))
-    .map(app => ({ appId: app.appId || app._id, name: app.name }))
+    .filter((app: App) => isDevAppID(app.appId || app._id))
+    .map((app: App) => ({ appId: (app.appId || app._id)!, name: app.name }))
   const tmpPath = await sdk.backups.exportMultipleApps(
     appMetadata,
     globalDBString
@@ -56,25 +57,25 @@ exports.exportApps = async ctx => {
   ctx.body = streamFile(tmpPath)
 }
 
-async function hasBeenImported() {
+async function checkHasBeenImported() {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     return true
   }
-  const apps = await getAllApps({ all: true })
+  const apps = await dbCore.getAllApps({ all: true })
   return apps.length !== 0
 }
 
-exports.hasBeenImported = async ctx => {
+export async function hasBeenImported(ctx: BBContext) {
   ctx.body = {
-    imported: await hasBeenImported(),
+    imported: await checkHasBeenImported(),
   }
 }
 
-exports.importApps = async ctx => {
+export async function importApps(ctx: BBContext) {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     ctx.throw(400, "Importing only allowed in self hosted environments.")
   }
-  const beenImported = await hasBeenImported()
+  const beenImported = await checkHasBeenImported()
   if (beenImported || !ctx.request.files || !ctx.request.files.importFile) {
     ctx.throw(
       400,
@@ -90,7 +91,7 @@ exports.importApps = async ctx => {
   const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
   const appNames = sdk.backups.getListOfAppsInMulti(tmpPath)
 
-  const globalDb = getGlobalDB()
+  const globalDb = tenancy.getGlobalDB()
   // load the global db first
   await globalDb.load(stringToReadStream(globalDbImport))
   for (let appName of appNames) {

package/src/api/controllers/component.ts

@@ -2,8 +2,9 @@ import { DocumentType } from "../../db/utils"
 import { Plugin } from "@budibase/types"
 import { db as dbCore, context, tenancy } from "@budibase/backend-core"
 import { getComponentLibraryManifest } from "../../utilities/fileSystem"
+import { BBContext } from "@budibase/types"
 
-exports.fetchAppComponentDefinitions = async function (ctx: any) {
+export async function fetchAppComponentDefinitions(ctx: BBContext) {
   try {
     const db = context.getAppDB()
     const app = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/{datasource.js → datasource.ts}

@@ -1,31 +1,37 @@
-const {
+import {
   generateDatasourceID,
   getDatasourceParams,
   getQueryParams,
   DocumentType,
   BudibaseInternalDB,
   getTableParams,
-} = require("../../db/utils")
-const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { getIntegration } = require("../../integrations")
-const { getDatasourceAndQuery } = require("./row/utils")
-const { invalidateDynamicVariables } = require("../../threads/utils")
-const { getAppDB } = require("@budibase/backend-core/context")
-const { events } = require("@budibase/backend-core")
-
-exports.fetch = async function (ctx) {
+} from "../../db/utils"
+import { destroy as tableDestroy } from "./table/internal"
+import { BuildSchemaErrors, InvalidColumns } from "../../constants"
+import { getIntegration } from "../../integrations"
+import { getDatasourceAndQuery } from "./row/utils"
+import { invalidateDynamicVariables } from "../../threads/utils"
+import { db as dbCore, context, events } from "@budibase/backend-core"
+import { BBContext, Datasource, Row } from "@budibase/types"
+
+export async function fetch(ctx: BBContext) {
   // Get internal tables
-  const db = getAppDB()
+  const db = context.getAppDB()
   const internalTables = await db.allDocs(
     getTableParams(null, {
       include_docs: true,
     })
   )
-  const internal = internalTables.rows.map(row => row.doc)
+
+  const internal = internalTables.rows.reduce((acc: any, row: Row) => {
+    const sourceId = row.doc.sourceId || "bb_internal"
+    acc[sourceId] = acc[sourceId] || []
+    acc[sourceId].push(row.doc)
+    return acc
+  }, {})
 
   const bbInternalDb = {
     ...BudibaseInternalDB,
-    entities: internal,
   }
 
   // Get external datasources
@@ -37,18 +43,24 @@ exports.fetch = async function (ctx) {
     )
   ).rows.map(row => row.doc)
 
-  for (let datasource of datasources) {
+  const allDatasources = [bbInternalDb, ...datasources]
+
+  for (let datasource of allDatasources) {
     if (datasource.config && datasource.config.auth) {
       // strip secrets from response so they don't show in the network request
       delete datasource.config.auth
     }
+
+    if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
+      datasource.entities = internal[datasource._id]
+    }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-exports.buildSchemaFromDb = async function (ctx) {
-  const db = getAppDB()
+export async function buildSchemaFromDb(ctx: BBContext) {
+  const db = context.getAppDB()
   const datasource = await db.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
@@ -59,7 +71,9 @@ exports.buildSchemaFromDb = async function (ctx) {
     }
     for (let key in tables) {
       if (
-        tablesFilter.some(filter => filter.toLowerCase() === key.toLowerCase())
+        tablesFilter.some(
+          (filter: any) => filter.toLowerCase() === key.toLowerCase()
+        )
       ) {
         datasource.entities[key] = tables[key]
       }
@@ -72,7 +86,7 @@ exports.buildSchemaFromDb = async function (ctx) {
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
 
-  const response = { datasource }
+  const response: any = { datasource }
   if (error) {
     response.error = error
   }
@@ -82,9 +96,9 @@ exports.buildSchemaFromDb = async function (ctx) {
 /**
  * Make sure all datasource entities have a display name selected
  */
-const setDefaultDisplayColumns = datasource => {
+function setDefaultDisplayColumns(datasource: Datasource) {
   //
-  for (let entity of Object.values(datasource.entities)) {
+  for (let entity of Object.values(datasource.entities || {})) {
     if (entity.primaryDisplay) {
       continue
     }
@@ -100,9 +114,12 @@ const setDefaultDisplayColumns = datasource => {
 /**
  * Check for variables that have been updated or removed and invalidate them.
  */
-const invalidateVariables = async (existingDatasource, updatedDatasource) => {
-  const existingVariables = existingDatasource.config.dynamicVariables
-  const updatedVariables = updatedDatasource.config.dynamicVariables
+async function invalidateVariables(
+  existingDatasource: Datasource,
+  updatedDatasource: Datasource
+) {
+  const existingVariables: any = existingDatasource.config?.dynamicVariables
+  const updatedVariables: any = updatedDatasource.config?.dynamicVariables
   const toInvalidate = []
 
   if (!existingVariables) {
@@ -114,9 +131,9 @@ const invalidateVariables = async (existingDatasource, updatedDatasource) => {
     toInvalidate.push(...existingVariables)
   } else {
     // invaldate changed / removed
-    existingVariables.forEach(existing => {
+    existingVariables.forEach((existing: any) => {
       const unchanged = updatedVariables.find(
-        updated =>
+        (updated: any) =>
           existing.name === updated.name &&
           existing.queryId === updated.queryId &&
           existing.value === updated.value
@@ -129,13 +146,20 @@ const invalidateVariables = async (existingDatasource, updatedDatasource) => {
   await invalidateDynamicVariables(toInvalidate)
 }
 
-exports.update = async function (ctx) {
-  const db = getAppDB()
+export async function update(ctx: BBContext) {
+  const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
   const auth = datasource.config.auth
   await invalidateVariables(datasource, ctx.request.body)
-  datasource = { ...datasource, ...ctx.request.body }
+
+  const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
+
+  const dataSourceBody = isBudibaseSource
+    ? { name: ctx.request.body?.name }
+    : ctx.request.body
+
+  datasource = { ...datasource, ...dataSourceBody }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
     datasource.config.auth = auth
@@ -146,7 +170,7 @@ exports.update = async function (ctx) {
   datasource._rev = response.rev
 
   // Drain connection pools when configuration is changed
-  if (datasource.source) {
+  if (datasource.source && !isBudibaseSource) {
     const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
@@ -158,8 +182,8 @@ exports.update = async function (ctx) {
   ctx.body = { datasource }
 }
 
-exports.save = async function (ctx) {
-  const db = getAppDB()
+export async function save(ctx: BBContext) {
+  const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
 
@@ -189,27 +213,63 @@ exports.save = async function (ctx) {
     }
   }
 
-  const response = { datasource }
+  const response: any = { datasource }
   if (schemaError) {
     response.error = schemaError
   }
   ctx.body = response
 }
 
-exports.destroy = async function (ctx) {
-  const db = getAppDB()
+async function destroyInternalTablesBySourceId(datasourceId: string) {
+  const db = context.getAppDB()
+
+  // Get all internal tables
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+
+  // Filter by datasource and return the docs.
+  const datasourceTableDocs = internalTables.rows.reduce(
+    (acc: any, table: any) => {
+      if (table.doc.sourceId == datasourceId) {
+        acc.push(table.doc)
+      }
+      return acc
+    },
+    []
+  )
+
+  // Destroy the tables.
+  for (const table of datasourceTableDocs) {
+    await tableDestroy({
+      params: {
+        tableId: table._id,
+      },
+    })
+  }
+}
+
+export async function destroy(ctx: BBContext) {
+  const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
   const datasource = await db.get(datasourceId)
   // Delete all queries for the datasource
-  const queries = await db.allDocs(getQueryParams(datasourceId, null))
-  await db.bulkDocs(
-    queries.rows.map(row => ({
-      _id: row.id,
-      _rev: row.value.rev,
-      _deleted: true,
-    }))
-  )
+
+  if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
+    await destroyInternalTablesBySourceId(datasourceId)
+  } else {
+    const queries = await db.allDocs(getQueryParams(datasourceId, null))
+    await db.bulkDocs(
+      queries.rows.map((row: any) => ({
+        _id: row.id,
+        _rev: row.value.rev,
+        _deleted: true,
+      }))
+    )
+  }
 
   // delete the datasource
   await db.remove(datasourceId, ctx.params.revId)
@@ -219,28 +279,28 @@ exports.destroy = async function (ctx) {
   ctx.status = 200
 }
 
-exports.find = async function (ctx) {
-  const database = getAppDB()
+export async function find(ctx: BBContext) {
+  const database = context.getAppDB()
   ctx.body = await database.get(ctx.params.datasourceId)
 }
 
 // dynamic query functionality
-exports.query = async function (ctx) {
+export async function query(ctx: BBContext) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
-  } catch (err) {
+  } catch (err: any) {
     ctx.throw(400, err)
   }
 }
 
-function getErrorTables(errors, errorType) {
+function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
     .filter(entry => entry[1] === errorType)
     .map(([name]) => name)
 }
 
-function updateError(error, newError, tables) {
+function updateError(error: any, newError: any, tables: string[]) {
   if (!error) {
     error = ""
   }
@@ -251,7 +311,7 @@ function updateError(error, newError, tables) {
   return error
 }
 
-const buildSchemaHelper = async datasource => {
+async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema