@budibase/server 2.2.26 → 2.2.27-alpha.0

package/specs/openapi.yaml

@@ -411,6 +411,30 @@ components:
               - version
       required:
         - data
+    deploymentOutput:
+      type: object
+      properties:
+        data:
+          type: object
+          properties:
+            _id:
+              description: The ID of the app.
+              type: string
+            status:
+              description: Status of the deployment, whether it succeeded or failed
+              type: string
+              enum:
+                - SUCCESS
+                - FAILURE
+            appUrl:
+              description: The URL of the published app
+              type: string
+          required:
+            - _id
+            - status
+            - appUrl
+      required:
+        - data
     row:
       description: The row to be created/updated, based on the table schema.
       type: object
@@ -1453,6 +1477,35 @@ paths:
               examples:
                 application:
                   $ref: "#/components/examples/application"
+  "/applications/{appId}/unpublish":
+    post:
+      operationId: unpublish
+      summary: Unpublish an application
+      tags:
+        - applications
+      parameters:
+        - $ref: "#/components/parameters/appIdUrl"
+      responses:
+        "204":
+          description: The app was published successfully.
+  "/applications/{appId}/publish":
+    post:
+      operationId: publish
+      summary: Unpublish an application
+      tags:
+        - applications
+      parameters:
+        - $ref: "#/components/parameters/appIdUrl"
+      responses:
+        "200":
+          description: Returns the deployment object.
+          content:
+            application/json:
+              schema:
+                $ref: "#/components/schemas/deploymentOutput"
+              examples:
+                deployment:
+                  $ref: "#/components/examples/deploymentOutput"
   /applications/search:
     post:
       operationId: search

package/specs/resources/application.js

@@ -80,6 +80,22 @@ const applicationOutputSchema = object(
   }
 )
 
+const deploymentOutputSchema = object({
+  _id: {
+    description: "The ID of the app.",
+    type: "string",
+  },
+  status: {
+    description: "Status of the deployment, whether it succeeded or failed",
+    type: "string",
+    enum: ["SUCCESS", "FAILURE"],
+  },
+  appUrl: {
+    description: "The URL of the published app",
+    type: "string",
+  },
+})
+
 module.exports = new Resource()
   .setExamples({
     application: {
@@ -104,4 +120,7 @@ module.exports = new Resource()
         items: applicationOutputSchema,
       },
     }),
+    deploymentOutput: object({
+      data: deploymentOutputSchema,
+    }),
   })

package/src/api/controllers/application.ts

@@ -23,21 +23,18 @@ import {
   errors,
   events,
   migrations,
+  objectStore,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import {
-  clientLibraryPath,
-  stringToReadStream,
-  isQsTrue,
-} from "../../utilities"
+import { stringToReadStream, isQsTrue } from "../../utilities"
 import { getLocksById } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
   revertClientLibrary,
-} from "../../utilities/fileSystem/clientLibrary"
+} from "../../utilities/fileSystem"
 import { cleanupAutomations } from "../../automations/utils"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
@@ -49,9 +46,9 @@ import {
   MigrationType,
   BBContext,
   Database,
+  UserCtx,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
-import { enrichPluginURLs } from "../../utilities/plugins"
 import sdk from "../../sdk"
 
 // utility function, need to do away with this
@@ -115,12 +112,11 @@ function checkAppName(
   }
 }
 
-async function createInstance(template: any, includeSampleData: boolean) {
-  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
-  const baseAppId = generateAppID(tenantId)
-  const appId = generateDevAppID(baseAppId)
-  await context.updateAppId(appId)
-
+async function createInstance(
+  appId: string,
+  template: any,
+  includeSampleData: boolean
+) {
   const db = context.getAppDB()
   await db.put({
     _id: "_design/database",
@@ -204,27 +200,34 @@ export async function fetchAppDefinition(ctx: BBContext) {
   }
 }
 
-export async function fetchAppPackage(ctx: BBContext) {
+export async function fetchAppPackage(ctx: UserCtx) {
   const db = context.getAppDB()
   let application = await db.get(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
 
   // Enrich plugin URLs
-  application.usedPlugins = enrichPluginURLs(application.usedPlugins)
+  application.usedPlugins = objectStore.enrichPluginURLs(
+    application.usedPlugins
+  )
 
   // Only filter screens if the user is not a builder
-  if (!(ctx.user?.builder && ctx.user.builder.global)) {
+  if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
     const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
+  const clientLibPath = objectStore.clientLibraryUrl(
+    ctx.params.appId,
+    application.version
+  )
+
   ctx.body = {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath,
   }
 }
 
@@ -246,82 +249,90 @@ async function performAppCreate(ctx: BBContext) {
     instanceConfig.file = ctx.request.files.templateFile
   }
   const includeSampleData = isQsTrue(ctx.request.body.sampleData)
-  const instance = await createInstance(instanceConfig, includeSampleData)
-  const appId = instance._id
-  const db = context.getAppDB()
+  const tenantId = tenancy.isMultiTenant() ? tenancy.getTenantId() : null
+  const appId = generateDevAppID(generateAppID(tenantId))
 
-  let newApplication: App = {
-    _id: DocumentType.APP_METADATA,
-    _rev: undefined,
-    appId,
-    type: "app",
-    version: packageJson.version,
-    componentLibraries: ["@budibase/standard-components"],
-    name: name,
-    url: url,
-    template: templateKey,
-    instance,
-    tenantId: tenancy.getTenantId(),
-    updatedAt: new Date().toISOString(),
-    createdAt: new Date().toISOString(),
-    status: AppStatus.DEV,
-    navigation: {
-      navigation: "Top",
-      title: name,
-      navWidth: "Large",
-      navBackground: "var(--spectrum-global-color-gray-100)",
-      links: [
-        {
-          url: "/home",
-          text: "Home",
-        },
-      ],
-    },
-    theme: "spectrum--light",
-    customTheme: {
-      buttonBorderRadius: "16px",
-    },
-  }
+  return await context.doInAppContext(appId, async () => {
+    const instance = await createInstance(
+      appId,
+      instanceConfig,
+      includeSampleData
+    )
+    const db = context.getAppDB()
 
-  // If we used a template or imported an app there will be an existing doc.
-  // Fetch and migrate some metadata from the existing app.
-  try {
-    const existing: App = await db.get(DocumentType.APP_METADATA)
-    const keys: (keyof App)[] = [
-      "_rev",
-      "navigation",
-      "theme",
-      "customTheme",
-      "icon",
-    ]
-    keys.forEach(key => {
-      if (existing[key]) {
-        // @ts-ignore
-        newApplication[key] = existing[key]
-      }
-    })
+    let newApplication: App = {
+      _id: DocumentType.APP_METADATA,
+      _rev: undefined,
+      appId,
+      type: "app",
+      version: packageJson.version,
+      componentLibraries: ["@budibase/standard-components"],
+      name: name,
+      url: url,
+      template: templateKey,
+      instance,
+      tenantId: tenancy.getTenantId(),
+      updatedAt: new Date().toISOString(),
+      createdAt: new Date().toISOString(),
+      status: AppStatus.DEV,
+      navigation: {
+        navigation: "Top",
+        title: name,
+        navWidth: "Large",
+        navBackground: "var(--spectrum-global-color-gray-100)",
+        links: [
+          {
+            url: "/home",
+            text: "Home",
+          },
+        ],
+      },
+      theme: "spectrum--light",
+      customTheme: {
+        buttonBorderRadius: "16px",
+      },
+    }
+
+    // If we used a template or imported an app there will be an existing doc.
+    // Fetch and migrate some metadata from the existing app.
+    try {
+      const existing: App = await db.get(DocumentType.APP_METADATA)
+      const keys: (keyof App)[] = [
+        "_rev",
+        "navigation",
+        "theme",
+        "customTheme",
+        "icon",
+      ]
+      keys.forEach(key => {
+        if (existing[key]) {
+          // @ts-ignore
+          newApplication[key] = existing[key]
+        }
+      })
 
-    // Migrate navigation settings and screens if required
-    if (existing) {
-      const navigation = await migrateAppNavigation()
-      if (navigation) {
-        newApplication.navigation = navigation
+      // Migrate navigation settings and screens if required
+      if (existing) {
+        const navigation = await migrateAppNavigation()
+        if (navigation) {
+          newApplication.navigation = navigation
+        }
       }
+    } catch (err) {
+      // Nothing to do
     }
-  } catch (err) {
-    // Nothing to do
-  }
 
-  const response = await db.put(newApplication, { force: true })
-  newApplication._rev = response.rev
+    const response = await db.put(newApplication, { force: true })
+    newApplication._rev = response.rev
 
-  /* istanbul ignore next */
-  if (!env.isTest()) {
-    await createApp(appId)
-  }
+    /* istanbul ignore next */
+    if (!env.isTest()) {
+      await createApp(appId)
+    }
 
-  await cache.app.invalidateAppMetadata(appId, newApplication)
-  return newApplication
+    await cache.app.invalidateAppMetadata(appId, newApplication)
+    return newApplication
+  })
 }
 
 async function creationEvents(request: any, app: App) {
@@ -370,7 +381,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
           // delete the app
-          // skip pre- and post-steps as no rows have been added to quotas yet
+          // skip pre and post-steps as no rows have been added to quotas yet
           ctx.params.appId = app.appId
           await destroyApp(ctx)
         }
@@ -460,41 +471,47 @@ export async function revertClient(ctx: BBContext) {
   ctx.body = app
 }
 
+const unpublishApp = async (ctx: any) => {
+  let appId = ctx.params.appId
+  appId = dbCore.getProdAppID(appId)
+
+  const db = context.getProdAppDB()
+  const result = await db.destroy()
+
+  await events.app.unpublished({ appId } as App)
+
+  // automations only in production
+  await cleanupAutomations(appId)
+
+  await cache.app.invalidateAppMetadata(appId)
+  return result
+}
+
 async function destroyApp(ctx: BBContext) {
   let appId = ctx.params.appId
-  let isUnpublish = ctx.query && ctx.query.unpublish
+  appId = dbCore.getProdAppID(appId)
+  const devAppId = dbCore.getDevAppID(appId)
 
-  if (isUnpublish) {
-    appId = dbCore.getProdAppID(appId)
-    const devAppId = dbCore.getDevAppID(appId)
-    // sync before removing the published app
+  // check if we need to unpublish first
+  if (await dbCore.dbExists(appId)) {
+    // app is deployed, run through unpublish flow
     await sdk.applications.syncApp(devAppId)
+    await unpublishApp(ctx)
   }
 
-  const db = isUnpublish ? context.getProdAppDB() : context.getAppDB()
+  const db = dbCore.getDB(devAppId)
+  // standard app deletion flow
   const app = await db.get(DocumentType.APP_METADATA)
   const result = await db.destroy()
+  await quotas.removeApp()
+  await events.app.deleted(app)
 
-  if (isUnpublish) {
-    await events.app.unpublished(app)
-  } else {
-    await quotas.removeApp()
-    await events.app.deleted(app)
-  }
-
-  /* istanbul ignore next */
-  if (!env.isTest() && !isUnpublish) {
+  if (!env.isTest()) {
     await deleteApp(appId)
   }
-  // automations only in production
-  if (isUnpublish) {
-    await cleanupAutomations(appId)
-  }
-  // remove app role when the dev app is deleted (no trace of app anymore)
-  else {
-    await removeAppFromUserRoles(ctx, appId)
-  }
-  await cache.app.invalidateAppMetadata(appId)
+
+  await removeAppFromUserRoles(ctx, appId)
+  await cache.app.invalidateAppMetadata(devAppId)
   return result
 }
 
@@ -519,6 +536,21 @@ export async function destroy(ctx: BBContext) {
   ctx.body = result
 }
 
+export const unpublish = async (ctx: BBContext) => {
+  const prodAppId = dbCore.getProdAppID(ctx.params.appId)
+  const dbExists = await dbCore.dbExists(prodAppId)
+
+  // check app has been published
+  if (!dbExists) {
+    return ctx.throw(400, "App has not been published.")
+  }
+
+  await preDestroyApp(ctx)
+  await unpublishApp(ctx)
+  await postDestroyApp(ctx)
+  ctx.status = 204
+}
+
 export async function sync(ctx: BBContext) {
   const appId = ctx.params.appId
   try {

package/src/api/controllers/auth.ts

@@ -3,7 +3,7 @@ import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
-import { ContextUser, User, Row } from "@budibase/types"
+import { ContextUser, User, Row, UserCtx } from "@budibase/types"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -16,7 +16,7 @@ const addSessionAttributesToUser = (ctx: any) => {
   }
 }
 
-export async function fetchSelf(ctx: any) {
+export async function fetchSelf(ctx: UserCtx) {
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId || !ctx.isAuthenticated) {

package/src/api/controllers/cloud.ts

@@ -5,8 +5,8 @@ import { stringToReadStream } from "../../utilities"
 import { getDocParams, DocumentType, isDevAppID } from "../../db/utils"
 import { create } from "./application"
 import { join } from "path"
-import { App, BBContext, Database } from "@budibase/types"
 import sdk from "../../sdk"
+import { App, Ctx, Database } from "@budibase/types"
 
 async function createApp(appName: string, appDirectory: string) {
   const ctx = {
@@ -35,7 +35,7 @@ async function getAllDocType(db: Database, docType: string) {
   return response.rows.map(row => row.doc)
 }
 
-export async function exportApps(ctx: BBContext) {
+export async function exportApps(ctx: Ctx) {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
@@ -65,13 +65,13 @@ async function checkHasBeenImported() {
   return apps.length !== 0
 }
 
-export async function hasBeenImported(ctx: BBContext) {
+export async function hasBeenImported(ctx: Ctx) {
   ctx.body = {
     imported: await checkHasBeenImported(),
   }
 }
 
-export async function importApps(ctx: BBContext) {
+export async function importApps(ctx: Ctx) {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     ctx.throw(400, "Importing only allowed in self hosted environments.")
   }
@@ -82,12 +82,16 @@ export async function importApps(ctx: BBContext) {
       "Import file is required and environment must be fresh to import apps."
     )
   }
-  if (ctx.request.files.importFile.type !== "application/gzip") {
+  const file = ctx.request.files.importFile as any
+  if (Array.isArray(file)) {
+    ctx.throw(400, "Single file is required")
+  }
+  if (file.type !== "application/gzip" && file.type !== "application/x-gzip") {
     ctx.throw(400, "Import file must be a gzipped tarball.")
   }
 
   // initially get all the app databases out of the tarball
-  const tmpPath = sdk.backups.untarFile(ctx.request.files.importFile)
+  const tmpPath = sdk.backups.untarFile(file)
   const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
   const appNames = sdk.backups.getListOfAppsInMulti(tmpPath)
 

package/src/api/controllers/datasource.ts

@@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { BBContext, Datasource, Row } from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { mergeConfigs } from "../../sdk/app/datasources/datasources"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id]
+      datasource.entities = internal[datasource._id!]
     }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function buildSchemaFromDb(ctx: BBContext) {
+export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -146,11 +146,11 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
-  const auth = datasource.config.auth
+  let datasource = await sdk.datasources.get(datasourceId)
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -159,10 +159,13 @@ export async function update(ctx: BBContext) {
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = { ...datasource, ...dataSourceBody }
+  datasource = {
+    ...datasource,
+    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+  }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -179,10 +182,12 @@ export async function update(ctx: BBContext) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = { datasource }
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
@@ -213,7 +218,9 @@ export async function save(ctx: BBContext) {
     }
   }
 
-  const response: any = { datasource }
+  const response: any = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
   if (schemaError) {
     response.error = schemaError
   }
@@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) {
   ctx.status = 200
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  ctx.body = await database.get(ctx.params.datasourceId)
+  const datasource = await database.get(ctx.params.datasourceId)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality
-export async function query(ctx: BBContext) {
+export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
@@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
-
+  datasource = await sdk.datasources.enrich(datasource)
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)

package/src/api/controllers/deploy/Deployment.ts

@@ -9,6 +9,7 @@ export default class Deployment {
   verification: any
   status?: string
   err?: any
+  appUrl?: string
 
   constructor(id = null) {
     this._id = id || newid()