@budibase/server 1.3.20 → 1.3.21-alpha.0

package/src/api/controllers/plugin/index.ts

@@ -0,0 +1,215 @@
+import { ObjectStoreBuckets } from "../../../constants"
+import { loadJSFile } from "../../../utilities/fileSystem"
+import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
+import { validate } from "@budibase/backend-core/plugins"
+import { generatePluginID, getPluginParams } from "../../../db/utils"
+import {
+  uploadDirectory,
+  deleteFolder,
+} from "@budibase/backend-core/objectStore"
+import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types"
+import env from "../../../environment"
+import { ClientAppSocket } from "../../../websocket"
+import { events } from "@budibase/backend-core"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function upload(ctx: any) {
+  const plugins: FileType[] =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
+  try {
+    let docs = []
+    // can do single or multiple plugins
+    for (let plugin of plugins) {
+      const doc = await processPlugin(plugin, PluginSource.FILE)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function create(ctx: any) {
+  const { source, url, headers, githubToken } = ctx.request.body
+
+  try {
+    let metadata
+    let directory
+    // Generating random name as a backup and needed for url
+    let name = "PLUGIN_" + Math.floor(100000 + Math.random() * 900000)
+
+    switch (source) {
+      case PluginSource.NPM:
+        const { metadata: metadataNpm, directory: directoryNpm } =
+          await npmUpload(url, name)
+        metadata = metadataNpm
+        directory = directoryNpm
+        break
+      case PluginSource.GITHUB:
+        const { metadata: metadataGithub, directory: directoryGithub } =
+          await githubUpload(url, name, githubToken)
+        metadata = metadataGithub
+        directory = directoryGithub
+        break
+      case PluginSource.URL:
+        const headersObj = headers || {}
+        const { metadata: metadataUrl, directory: directoryUrl } =
+          await urlUpload(url, name, headersObj)
+        metadata = metadataUrl
+        directory = directoryUrl
+        break
+    }
+
+    validate(metadata?.schema)
+
+    // Only allow components in cloud
+    if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+      throw new Error(
+        "Only component plugins are supported outside of self-host"
+      )
+    }
+
+    const doc = await storePlugin(metadata, directory, source)
+
+    ctx.body = {
+      message: "Plugin uploaded successfully",
+      plugins: [doc],
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+  ctx.status = 200
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {
+  const db = getGlobalDB()
+  const { pluginId } = ctx.params
+
+  try {
+    const plugin: Plugin = await db.get(pluginId)
+    const bucketPath = `${plugin.name}/`
+    await deleteFolder(ObjectStoreBuckets.PLUGINS, bucketPath)
+
+    await db.remove(pluginId, plugin._rev)
+    await events.plugin.deleted(plugin)
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+
+    ctx.throw(400, `Failed to delete plugin: ${errMsg}`)
+  }
+
+  ctx.message = `Plugin ${ctx.params.pluginId} deleted.`
+  ctx.status = 200
+}
+
+export async function storePlugin(
+  metadata: any,
+  directory: any,
+  source?: PluginSource
+) {
+  const db = getGlobalDB()
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description,
+    hash = metadata.schema.hash
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  let doc: Plugin = {
+    _id: pluginId,
+    _rev: rev,
+    ...metadata,
+    name,
+    version,
+    hash,
+    description,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+
+  if (source) {
+    doc = {
+      ...doc,
+      source,
+    }
+  }
+
+  const response = await db.put(doc)
+  await events.plugin.imported(doc)
+  ClientAppSocket.emit("plugin-update", { name, hash })
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}
+
+export async function processPlugin(plugin: FileType, source?: PluginSource) {
+  const { metadata, directory } = await fileUpload(plugin)
+  validate(metadata?.schema)
+
+  // Only allow components in cloud
+  if (!env.SELF_HOSTED && metadata?.schema?.type !== PluginType.COMPONENT) {
+    throw new Error("Only component plugins are supported outside of self-host")
+  }
+
+  return await storePlugin(metadata, directory, source)
+}

package/src/api/controllers/plugin/npm.ts

@@ -0,0 +1,56 @@
+import {
+  getPluginMetadata,
+  findFileRec,
+  extractTarball,
+  deleteFolderFileSystem,
+} from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { join } from "path"
+import { downloadUnzipTarball } from "./utils"
+
+export async function npmUpload(url: string, name: string, headers = {}) {
+  let npmTarballUrl = url
+  let pluginName = name
+
+  if (
+    !npmTarballUrl.includes("https://www.npmjs.com") &&
+    !npmTarballUrl.includes("https://registry.npmjs.org")
+  ) {
+    throw new Error("The plugin origin must be from NPM")
+  }
+
+  if (!npmTarballUrl.includes(".tgz")) {
+    const npmPackageURl = url.replace(
+      "https://www.npmjs.com/package/",
+      "https://registry.npmjs.org/"
+    )
+    const response = await fetch(npmPackageURl)
+    if (response.status !== 200) {
+      throw new Error("NPM Package not found")
+    }
+
+    let npmDetails = await response.json()
+    pluginName = npmDetails.name
+    const npmVersion = npmDetails["dist-tags"].latest
+    npmTarballUrl = npmDetails?.versions?.[npmVersion]?.dist?.tarball
+
+    if (!npmTarballUrl) {
+      throw new Error("NPM tarball url not found")
+    }
+  }
+
+  const path = await downloadUnzipTarball(npmTarballUrl, pluginName, headers)
+  const tarballPluginFile = findFileRec(path, ".tar.gz")
+  if (!tarballPluginFile) {
+    throw new Error("Tarball plugin file not found")
+  }
+
+  try {
+    await extractTarball(tarballPluginFile, path)
+    deleteFolderFileSystem(join(path, "package"))
+  } catch (err: any) {
+    throw new Error(err)
+  }
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/uploaders.ts

@@ -0,0 +1,4 @@
+export { fileUpload } from "./file"
+export { githubUpload } from "./github"
+export { npmUpload } from "./npm"
+export { urlUpload } from "./url"

package/src/api/controllers/plugin/url.ts

@@ -0,0 +1,12 @@
+import { downloadUnzipTarball } from "./utils"
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+
+export async function urlUpload(url: string, name = "", headers = {}) {
+  if (!url.includes(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+
+  const path = await downloadUnzipTarball(url, name, headers)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/utils.ts

@@ -0,0 +1,19 @@
+import {
+  createTempFolder,
+  downloadTarballDirect,
+} from "../../../utilities/fileSystem"
+
+export async function downloadUnzipTarball(
+  url: string,
+  name: string,
+  headers = {}
+) {
+  try {
+    const path = createTempFolder(name)
+    await downloadTarballDirect(url, path, headers)
+
+    return path
+  } catch (e: any) {
+    throw new Error(e.message)
+  }
+}

package/src/api/controllers/public/mapping/types.ts

@@ -4,10 +4,18 @@ export type Query = components["schemas"]["query"]
 export type ExecuteQuery = components["schemas"]["executeQueryOutput"]
 
 export type Application = components["schemas"]["applicationOutput"]["data"]
+export type CreateApplicationParams = components["schemas"]["application"]
 
 export type Table = components["schemas"]["tableOutput"]["data"]
+export type CreateTableParams = components["schemas"]["table"]
 
 export type Row = components["schemas"]["rowOutput"]["data"]
 export type RowSearch = components["schemas"]["searchOutput"]
+export type CreateRowParams = components["schemas"]["row"]
 
 export type User = components["schemas"]["userOutput"]["data"]
+export type CreateUserParams = components["schemas"]["user"]
+
+export type SearchInputParams =
+  | components["schemas"]["nameSearch"]
+  | components["schemas"]["rowSearch"]

package/src/api/controllers/row/external.js

@@ -53,7 +53,7 @@ exports.save = async ctx => {
 }
 
 exports.fetchView = async ctx => {
-  // there are no views in external data sources, shouldn't ever be called
+  // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
   const split = ctx.params.viewName.split("all_")
   ctx.params.tableId = split[1] ? split[1] : split[0]

package/src/api/controllers/screen.js

@@ -1,7 +1,14 @@
-const { getScreenParams, generateScreenID } = require("../../db/utils")
+const {
+  getScreenParams,
+  generateScreenID,
+  getPluginParams,
+  DocumentType,
+} = require("../../db/utils")
 const { AccessController } = require("@budibase/backend-core/roles")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const { updateAppPackage } = require("./application")
 
 exports.fetch = async ctx => {
   const db = getAppDB()
@@ -32,6 +39,48 @@ exports.save = async ctx => {
 
   const response = await db.put(screen)
 
+  // Find any custom components being used
+  let pluginNames = []
+  let pluginAdded = false
+  findPlugins(screen.props, pluginNames)
+  if (pluginNames.length) {
+    const globalDB = getGlobalDB()
+    const pluginsResponse = await globalDB.allDocs(
+      getPluginParams(null, {
+        include_docs: true,
+      })
+    )
+    const requiredPlugins = pluginsResponse.rows
+      .map(row => row.doc)
+      .filter(plugin => {
+        return (
+          plugin.schema.type === "component" &&
+          pluginNames.includes(`plugin/${plugin.name}`)
+        )
+      })
+
+    // Update the app metadata
+    const application = await db.get(DocumentType.APP_METADATA)
+    let usedPlugins = application.usedPlugins || []
+
+    requiredPlugins.forEach(plugin => {
+      if (!usedPlugins.find(x => x._id === plugin._id)) {
+        pluginAdded = true
+        usedPlugins.push({
+          _id: plugin._id,
+          name: plugin.name,
+          version: plugin.version,
+          jsUrl: plugin.jsUrl,
+          hash: plugin.hash,
+        })
+      }
+    })
+
+    if (pluginAdded) {
+      await updateAppPackage({ usedPlugins }, ctx.appId)
+    }
+  }
+
   if (eventFn) {
     await eventFn(screen)
   }
@@ -40,6 +89,7 @@ exports.save = async ctx => {
     ...screen,
     _id: response.id,
     _rev: response.rev,
+    pluginAdded,
   }
 }
 
@@ -56,3 +106,18 @@ exports.destroy = async ctx => {
   }
   ctx.status = 200
 }
+
+const findPlugins = (component, foundPlugins) => {
+  if (!component) {
+    return
+  }
+  if (component._component.startsWith("plugin")) {
+    if (!foundPlugins.includes(component._component)) {
+      foundPlugins.push(component._component)
+    }
+  }
+  if (!component._children || !component._children.length) {
+    return
+  }
+  component._children.forEach(child => findPlugins(child, foundPlugins))
+}

package/src/api/controllers/static/index.ts

@@ -18,7 +18,6 @@ const { DocumentType } = require("../../../db/utils")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 const { setCookie, clearCookie } = require("@budibase/backend-core/utils")
 const AWS = require("aws-sdk")
-
 const fs = require("fs")
 const {
   downloadTarballDirect,
@@ -113,6 +112,7 @@ export const serveApp = async function (ctx: any) {
       production: env.isProd(),
       appId,
       clientLibPath: clientLibraryPath(appId, appInfo.version, ctx),
+      usedPlugins: appInfo.usedPlugins,
     })
 
     const appHbs = loadHandlebarsFile(`${__dirname}/templates/app.hbs`)

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -3,6 +3,7 @@
   export let favicon = ""
 
   export let clientLibPath
+  export let usedPlugins
 </script>
 
 <svelte:head>
@@ -83,6 +84,15 @@
   </script>
   <script type="application/javascript" src={clientLibPath}>
   </script>
+  <!-- Custom components need inserted after the core client library -->
+  <!-- But before loadBudibase is called -->
+  {#if usedPlugins?.length}
+    {#each usedPlugins as plugin}
+      <script
+        type="application/javascript"
+        src={`/plugins/${plugin.jsUrl}`}></script>
+    {/each}
+  {/if}
   <script type="application/javascript">
     if (window.loadBudibase) {
       window.loadBudibase()

package/src/api/controllers/static/templates/preview.hbs

@@ -66,7 +66,9 @@
         customTheme,
         previewDevice,
         navigation,
-        hiddenComponentIds
+        hiddenComponentIds,
+        usedPlugins,
+        location
       } = parsed
 
       // Set some flags so the app knows we're in the builder
@@ -81,6 +83,8 @@
       window["##BUDIBASE_PREVIEW_DEVICE##"] = previewDevice
       window["##BUDIBASE_PREVIEW_NAVIGATION##"] = navigation
       window["##BUDIBASE_HIDDEN_COMPONENT_IDS##"] = hiddenComponentIds
+      window["##BUDIBASE_USED_PLUGINS##"] = usedPlugins
+      window["##BUDIBASE_LOCATION##"] = location
 
       // Initialise app
       try {

package/src/api/index.js

@@ -54,8 +54,8 @@ router
       noTenancyRequired: true,
     })
   )
-  .use(currentApp)
   .use(pro.licensing())
+  .use(currentApp)
   .use(auditLog)
 
 // error handling middleware

package/src/api/routes/application.ts

@@ -2,16 +2,27 @@ import Router from "@koa/router"
 import * as controller from "../controllers/application"
 import authorized from "../../middleware/authorized"
 import { BUILDER } from "@budibase/backend-core/permissions"
+import { applicationValidator } from "./utils/validators"
 
 const router = new Router()
 
 router
   .post("/api/applications/:appId/sync", authorized(BUILDER), controller.sync)
-  .post("/api/applications", authorized(BUILDER), controller.create)
+  .post(
+    "/api/applications",
+    authorized(BUILDER),
+    applicationValidator(),
+    controller.create
+  )
   .get("/api/applications/:appId/definition", controller.fetchAppDefinition)
   .get("/api/applications", controller.fetch)
   .get("/api/applications/:appId/appPackage", controller.fetchAppPackage)
-  .put("/api/applications/:appId", authorized(BUILDER), controller.update)
+  .put(
+    "/api/applications/:appId",
+    authorized(BUILDER),
+    applicationValidator({ isCreate: false }),
+    controller.update
+  )
   .post(
     "/api/applications/:appId/client/update",
     authorized(BUILDER),

package/src/api/routes/index.ts

@@ -24,6 +24,7 @@ import metadataRoutes from "./metadata"
 import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
+import pluginRoutes from "./plugin"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
@@ -57,4 +58,5 @@ export const mainRoutes = [
   tableRoutes,
   rowRoutes,
   migrationRoutes,
+  pluginRoutes,
 ]

package/src/api/routes/plugin.ts

@@ -0,0 +1,14 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/plugin"
+import authorized from "../../middleware/authorized"
+import { BUILDER } from "@budibase/backend-core/permissions"
+
+const router = new Router()
+
+router
+  .post("/api/plugin/upload", authorized(BUILDER), controller.upload)
+  .post("/api/plugin", authorized(BUILDER), controller.create)
+  .get("/api/plugin", authorized(BUILDER), controller.fetch)
+  .delete("/api/plugin/:pluginId", authorized(BUILDER), controller.destroy)
+
+export default router

package/src/api/routes/public/rows.ts

@@ -143,88 +143,7 @@ read.push(new Endpoint("get", "/tables/:tableId/rows/:rowId", controller.read))
  *       content:
  *         application/json:
  *           schema:
- *             type: object
- *             required:
- *               - query
- *             properties:
- *               query:
- *                 type: object
- *                 properties:
- *                   allOr:
- *                     type: boolean
- *                     description: Specifies that a row should be returned if it satisfies
- *                       any of the specified options, rather than requiring it to fulfill all
- *                       the search parameters. This defaults to false, meaning AND logic will be used.
- *                   string:
- *                     type: object
- *                     example:
- *                       columnName1: value
- *                       columnName2: value
- *                     description: A map of field name to the string to search for,
- *                       this will look for rows that have a value starting with the
- *                       string value.
- *                     additionalProperties:
- *                       type: string
- *                       description: The value to search for in the column.
- *                   fuzzy:
- *                     type: object
- *                     description: A fuzzy search, only supported by internal tables.
- *                   range:
- *                     type: object
- *                     description: Searches within a range, the format of this must be
- *                       in the format of an object with a "low" and "high" property.
- *                     example:
- *                       columnName1: { low: 10, high: 20 }
- *                   equal:
- *                     type: object
- *                     description: Searches for rows that have a column value that is
- *                       exactly the value set.
- *                   notEqual:
- *                     type: object
- *                     description: Searches for any row which does not contain the specified
- *                       column value.
- *                   empty:
- *                     type: object
- *                     description: Searches for rows which do not contain the specified column.
- *                       The object should simply contain keys of the column names, these
- *                       can map to any value.
- *                     example:
- *                       columnName1: ""
- *                   notEmpty:
- *                     type: object
- *                     description: Searches for rows which have the specified column.
- *                   oneOf:
- *                     type: object
- *                     description: Searches for rows which have a column value that is any
- *                       of the specified values. The format of this must be columnName -> [value1, value2].
- *               paginate:
- *                 type: boolean
- *                 description: Enables pagination, by default this is disabled.
- *               bookmark:
- *                 oneOf:
- *                   - type: string
- *                   - type: integer
- *                 description: If retrieving another page, the bookmark from the previous request must be supplied.
- *               limit:
- *                 type: integer
- *                 description: The maximum number of rows to return, useful when paginating, for internal tables this
- *                   will be limited to 1000, for SQL tables it will be 5000.
- *               sort:
- *                 type: object
- *                 description: A set of parameters describing the sort behaviour of the search.
- *                 properties:
- *                   order:
- *                     type: string
- *                     enum: [ascending, descending]
- *                     description: The order of the sort, by default this is ascending.
- *                   column:
- *                     type: string
- *                     description: The name of the column by which the rows will be sorted.
- *                   type:
- *                     type: string
- *                     enum: [string, number]
- *                     description: Defines whether the column should be treated as a string
- *                       or as numbers when sorting.
+ *             $ref: '#/components/schemas/rowSearch'
  *     responses:
  *       200:
  *         description: The response will contain an array of rows that match the search parameters.

package/src/api/routes/public/tables.ts

@@ -26,7 +26,7 @@ const read = [],
  *     responses:
  *       200:
  *         description: Returns the created table, including the ID which has been generated for it. This can be
- *           internal or external data sources.
+ *           internal or external datasources.
  *         content:
  *           application/json:
  *             schema:

package/src/api/routes/table.js

@@ -55,7 +55,7 @@ router
    * @apiParam (Body) {string} [_rev] If updating an existing internal table then the revision must also be specified.
    * @apiParam (Body) {string} type] This should either be "internal" or "external" depending on the table type -
    * this will default to internal.
-   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the data source ID. If
+   * @apiParam (Body) {string} [sourceId] If creating an external table then this should be set to the datasource ID. If
    * building an internal table this does not need to be set, although it will be returned as "bb_internal".
    * @apiParam (Body) {string} name The name of the table, this will be used in the UI. To rename the table simply
    * supply the table structure to this endpoint with the name changed.