@budibase/server 1.3.12 → 1.3.15-alpha.0

package/src/api/controllers/screen.js

@@ -1,7 +1,14 @@
-const { getScreenParams, generateScreenID } = require("../../db/utils")
+const {
+  getScreenParams,
+  generateScreenID,
+  getPluginParams,
+  DocumentType,
+} = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { updateAppPackage } = require("./application")
 
 exports.fetch = async ctx => {
   const db = getAppDB()
@@ -32,6 +39,47 @@ exports.save = async ctx => {
 
   const response = await db.put(screen)
 
+  // Find any custom components being used
+  let pluginNames = []
+  let pluginAdded = false
+  findPlugins(screen.props, pluginNames)
+  if (pluginNames.length) {
+    const globalDB = getGlobalDB()
+    const pluginsResponse = await globalDB.allDocs(
+      getPluginParams(null, {
+        include_docs: true,
+      })
+    )
+    const requiredPlugins = pluginsResponse.rows
+      .map(row => row.doc)
+      .filter(plugin => {
+        return (
+          plugin.schema.type === "component" &&
+          pluginNames.includes(`plugin/${plugin.name}`)
+        )
+      })
+
+    // Update the app metadata
+    const application = await db.get(DocumentType.APP_METADATA)
+    let usedPlugins = application.usedPlugins || []
+
+    requiredPlugins.forEach(plugin => {
+      if (!usedPlugins.find(x => x._id === plugin._id)) {
+        pluginAdded = true
+        usedPlugins.push({
+          _id: plugin._id,
+          name: plugin.name,
+          version: plugin.version,
+          jsUrl: plugin.jsUrl,
+        })
+      }
+    })
+
+    if (pluginAdded) {
+      await updateAppPackage({ usedPlugins }, ctx.appId)
+    }
+  }
+
   if (eventFn) {
     await eventFn(screen)
   }
@@ -40,6 +88,7 @@ exports.save = async ctx => {
     ...screen,
     _id: response.id,
     _rev: response.rev,
+    pluginAdded,
   }
 }
 
@@ -56,3 +105,18 @@ exports.destroy = async ctx => {
   }
   ctx.status = 200
 }
+
+const findPlugins = (component, foundPlugins) => {
+  if (!component) {
+    return
+  }
+  if (component._component.startsWith("plugin")) {
+    if (!foundPlugins.includes(component._component)) {
+      foundPlugins.push(component._component)
+    }
+  }
+  if (!component._children || !component._children.length) {
+    return
+  }
+  component._children.forEach(child => findPlugins(child, foundPlugins))
+}

package/src/api/controllers/static/index.ts

@@ -113,6 +113,7 @@ export const serveApp = async function (ctx: any) {
       production: env.isProd(),
       appId,
       clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+      usedPlugins: appInfo.usedPlugins,
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -3,6 +3,7 @@
   export let favicon = ""
 
   export let clientLibPath
+  export let usedPlugins
 </script>
 
 <svelte:head>
@@ -83,6 +84,15 @@
   </script>
   <script type="application/javascript" src={clientLibPath}>
   </script>
+  <!-- Custom components need inserted after the core client library -->
+  <!-- But before loadBudibase is called -->
+  {#if usedPlugins?.length}
+    {#each usedPlugins as plugin}
+      <script
+        type="application/javascript"
+        src={`/plugins/${plugin.jsUrl}`}></script>
+    {/each}
+  {/if}
   <script type="application/javascript">
     if (window.loadBudibase) {
       window.loadBudibase()

package/src/api/controllers/static/templates/preview.hbs

@@ -66,7 +66,8 @@
         customTheme,
         previewDevice,
         navigation,
-        hiddenComponentIds
+        hiddenComponentIds,
+        usedPlugins
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -81,6 +82,7 @@
       window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
+      window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
 
       // Initialise app
       try {

package/src/api/routes/index.ts

@@ -24,6 +24,7 @@ import metadataRoutes from "./metadata"
 import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
+import pluginRoutes from "./plugin"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
@@ -57,4 +58,5 @@ export const mainRoutes = [
   tableRoutes,
   rowRoutes,
   migrationRoutes,
+  pluginRoutes,
 ]

package/src/api/routes/plugin.ts

@@ -0,0 +1,13 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/plugin"
+import authorized from "../../middleware/authorized"
+import { BUILDER } from "@budibase/backend-core/permissions"
+
+const router = new Router()
+
+router
+  .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
+  .get("/api/plugin", authorized(BUILDER), controller.fetch)
+  .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
+
+export default router

package/src/api/routes/tests/query.spec.js

@@ -231,20 +231,6 @@ describe("/queries", () => {
         url: `/api/queries/preview`,
       })
     })
-
-    it("should fail with invalid integration type", async () => {
-      const { datasource } = await createInvalidIntegration()
-      await request
-        .post(`/api/queries/preview`)
-        .send({
-          datasourceId: datasource._id,
-          parameters: {},
-          fields: {},
-          queryVerb: "read",
-        })
-        .set(config.defaultHeaders())
-        .expect(400)
-    })
   })
 
   describe("execute", () => {
@@ -261,17 +247,14 @@ describe("/queries", () => {
     })
 
     it("should fail with invalid integration type", async () => {
-      const { query, datasource } = await createInvalidIntegration()
-      await request
-        .post(`/api/queries/${query._id}`)
-        .send({
-          datasourceId: datasource._id,
-          parameters: {},
-          fields: {},
-          queryVerb: "read",
-        })
-        .set(config.defaultHeaders())
-        .expect(400)
+      let error
+      try {
+        await createInvalidIntegration()
+      } catch (err) {
+        error = err
+      }
+      expect(error).toBeDefined()
+      expect(error.message).toBe("No datasource implementation found.")
     })
   })
 
@@ -311,7 +294,7 @@ describe("/queries", () => {
         "url": "string",
         "value": "string"
       })
-      expect(res.body.rows[0].url).toContain("doctype html")
+      expect(res.body.rows[0].url).toContain("doctype%20html")
     })
 
     it("check that it automatically retries on fail with cached dynamics", async () => {
@@ -396,7 +379,7 @@ describe("/queries", () => {
         "queryHdr": userDetails.firstName,
         "secondHdr" : "1234"
       })
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email)
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?email=" + userDetails.email.replace("@", "%40"))
     })
 
     it("should bind the current user to query parameters", async () => {
@@ -413,7 +396,7 @@ describe("/queries", () => {
         "testParam" : "1234"
       })
   
-      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email + 
+      expect(res.body.rows[0].url).toEqual("http://www.google.com?test=" + userDetails.email.replace("@", "%40") +
         "&testName=" + userDetails.firstName + "&testParam=1234")
     })
 

package/src/app.ts

@@ -26,10 +26,12 @@ const bullboard = require("./automations/bullboard")
 const { logAlert } = require("@budibase/backend-core/logging")
 const { pinoSettings } = require("@budibase/backend-core")
 const { Thread } = require("./threads")
+const fs = require("fs")
 import redis from "./utilities/redis"
 import * as migrations from "./migrations"
 import { events, installation, tenancy } from "@budibase/backend-core"
 import { createAdminUser, getChecklist } from "./utilities/workerRequests"
+import { watch } from "./watch"
 
 const app = new Koa()
 
@@ -139,6 +141,16 @@ module.exports = server.listen(env.PORT || 0, async () => {
     }
   }
 
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
   // check for version updates
   await installation.checkInstallVersion()
 

package/src/automations/index.js

@@ -1,16 +1,19 @@
 const { processEvent } = require("./utils")
 const { queue, shutdown } = require("./bullboard")
-const { TRIGGER_DEFINITIONS } = require("./triggers")
+const { TRIGGER_DEFINITIONS, rebootTrigger } = require("./triggers")
 const { ACTION_DEFINITIONS } = require("./actions")
 
 /**
  * This module is built purely to kick off the worker farm and manage the inputs/outputs
  */
-exports.init = function () {
+exports.init = async function () {
   // this promise will not complete
-  return queue.process(async job => {
+  const promise = queue.process(async job => {
     await processEvent(job)
   })
+  // on init we need to trigger any reboot automations
+  await rebootTrigger()
+  return promise
 }
 
 exports.getQueues = () => {

package/src/automations/triggers.js

@@ -9,6 +9,7 @@ const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
+const { getAllApps } = require("@budibase/backend-core/db")
 
 const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -16,24 +17,27 @@ const JOB_OPTS = {
   removeOnFail: true,
 }
 
+async function getAllAutomations() {
+  const db = getAppDB()
+  let automations = await db.allDocs(
+    getAutomationParams(null, { include_docs: true })
+  )
+  return automations.rows.map(row => row.doc)
+}
+
 async function queueRelevantRowAutomations(event, eventType) {
   if (event.appId == null) {
     throw `No appId specified for ${eventType} - check event emitters.`
   }
 
   doInAppContext(event.appId, async () => {
-    const db = getAppDB()
-    let automations = await db.allDocs(
-      getAutomationParams(null, { include_docs: true })
-    )
+    let automations = await getAllAutomations()
 
     // filter down to the correct event type
-    automations = automations.rows
-      .map(automation => automation.doc)
-      .filter(automation => {
-        const trigger = automation.definition.trigger
-        return trigger && trigger.event === eventType
-      })
+    automations = automations.filter(automation => {
+      const trigger = automation.definition.trigger
+      return trigger && trigger.event === eventType
+    })
 
     for (let automation of automations) {
       let automationDef = automation.definition
@@ -110,4 +114,34 @@ exports.externalTrigger = async function (
   }
 }
 
+exports.rebootTrigger = async () => {
+  // reboot cron option is only available on the main thread at
+  // startup and only usable in self host
+  if (env.isInThread() || !env.SELF_HOSTED) {
+    return
+  }
+  // iterate through all production apps, find the reboot crons
+  // and trigger events for them
+  const appIds = await getAllApps({ dev: false, idsOnly: true })
+  for (let prodAppId of appIds) {
+    await doInAppContext(prodAppId, async () => {
+      let automations = await getAllAutomations()
+      let rebootEvents = []
+      for (let automation of automations) {
+        if (utils.isRebootTrigger(automation)) {
+          const job = {
+            automation,
+            event: {
+              appId: prodAppId,
+              timestamp: Date.now(),
+            },
+          }
+          rebootEvents.push(queue.add(job, JOB_OPTS))
+        }
+      }
+      await Promise.all(rebootEvents)
+    })
+  }
+}
+
 exports.TRIGGER_DEFINITIONS = TRIGGER_DEFINITIONS

package/src/automations/utils.ts

@@ -17,6 +17,7 @@ import { tenancy } from "@budibase/backend-core"
 import { quotas } from "@budibase/pro"
 import { Automation } from "@budibase/types"
 
+const REBOOT_CRON = "@reboot"
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
@@ -109,22 +110,33 @@ export async function clearMetadata() {
   await db.bulkDocs(automationMetadata)
 }
 
+export function isCronTrigger(auto: Automation) {
+  return (
+    auto &&
+    auto.definition.trigger &&
+    auto.definition.trigger.stepId === CRON_STEP_ID
+  )
+}
+
+export function isRebootTrigger(auto: Automation) {
+  const trigger = auto ? auto.definition.trigger : null
+  return isCronTrigger(auto) && trigger?.inputs.cron === REBOOT_CRON
+}
+
 /**
  * This function handles checking of any cron jobs that need to be enabled/updated.
  * @param {string} appId The ID of the app in which we are checking for webhooks
  * @param {object|undefined} automation The automation object to be updated.
  */
-export async function enableCronTrigger(appId: any, automation: any) {
+export async function enableCronTrigger(appId: any, automation: Automation) {
   const trigger = automation ? automation.definition.trigger : null
-  function isCronTrigger(auto: any) {
-    return (
-      auto &&
-      auto.definition.trigger &&
-      auto.definition.trigger.stepId === CRON_STEP_ID
-    )
-  }
+
   // need to create cron job
-  if (isCronTrigger(automation) && trigger?.inputs.cron) {
+  if (
+    isCronTrigger(automation) &&
+    !isRebootTrigger(automation) &&
+    trigger?.inputs.cron
+  ) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
     const job: any = await queue.add(

package/src/db/utils.js

@@ -42,6 +42,7 @@ const DocumentType = {
   MEM_VIEW: "view",
   USER_FLAG: "flag",
   AUTOMATION_METADATA: "meta_au",
+  PLUGIN: "plg",
 }
 
 const InternalTables = {
@@ -370,6 +371,10 @@ exports.getMemoryViewParams = (otherProps = {}) => {
   return getDocParams(DocumentType.MEM_VIEW, null, otherProps)
 }
 
+exports.generatePluginID = name => {
+  return `${DocumentType.PLUGIN}${SEPARATOR}${name}`
+}
+
 /**
  * This can be used with the db.allDocs to get a list of IDs
  */
@@ -379,3 +384,10 @@ exports.getMultiIDParams = ids => {
     include_docs: true,
   }
 }
+
+/**
+ * Gets parameters for retrieving automations, this is a utility function for the getDocParams function.
+ */
+exports.getPluginParams = (pluginId = null, otherProps = {}) => {
+  return getDocParams(DocumentType.PLUGIN, pluginId, otherProps)
+}

package/src/definitions/datasource.ts

@@ -34,6 +34,7 @@ export interface RestConfig {
   defaultHeaders: {
     [key: string]: any
   }
+  legacyHttpParser: boolean
   authConfigs: AuthConfig[]
   staticVariables: {
     [key: string]: string

package/src/environment.js

@@ -74,6 +74,7 @@ module.exports = {
   SQL_MAX_ROWS: process.env.SQL_MAX_ROWS,
   BB_ADMIN_USER_EMAIL: process.env.BB_ADMIN_USER_EMAIL,
   BB_ADMIN_USER_PASSWORD: process.env.BB_ADMIN_USER_PASSWORD,
+  PLUGINS_DIR: process.env.PLUGINS_DIR || "/plugins",
   // flags
   ALLOW_DEV_AUTOMATIONS: process.env.ALLOW_DEV_AUTOMATIONS,
   DISABLE_THREADING: process.env.DISABLE_THREADING,