@budibase/server 1.3.21 → 1.3.22-alpha.1

package/specs/openapi.yaml

@@ -1004,7 +1004,7 @@ components:
           description: The ID of the query.
           type: string
         datasourceId:
-          description: The ID of the data source the query belongs to.
+          description: The ID of the datasource the query belongs to.
           type: string
         parameters:
           description: The bindings which are required to perform this query.
@@ -1051,7 +1051,7 @@ components:
                 description: The ID of the query.
                 type: string
               datasourceId:
-                description: The ID of the data source the query belongs to.
+                description: The ID of the datasource the query belongs to.
                 type: string
               parameters:
                 description: The bindings which are required to perform this query.
@@ -1264,6 +1264,98 @@ components:
               - _id
       required:
         - data
+    rowSearch:
+      type: object
+      properties:
+        query:
+          type: object
+          properties:
+            allOr:
+              type: boolean
+              description: Specifies that a row should be returned if it satisfies any of the
+                specified options, rather than requiring it to fulfill all the
+                search parameters. This defaults to false, meaning AND logic
+                will be used.
+            string:
+              type: object
+              example:
+                columnName1: value
+                columnName2: value
+              description: A map of field name to the string to search for, this will look for
+                rows that have a value starting with the string value.
+              additionalProperties:
+                type: string
+                description: The value to search for in the column.
+            fuzzy:
+              type: object
+              description: A fuzzy search, only supported by internal tables.
+            range:
+              type: object
+              description: Searches within a range, the format of this must be in the format
+                of an object with a "low" and "high" property.
+              example:
+                columnName1:
+                  low: 10
+                  high: 20
+            equal:
+              type: object
+              description: Searches for rows that have a column value that is exactly the
+                value set.
+            notEqual:
+              type: object
+              description: Searches for any row which does not contain the specified column
+                value.
+            empty:
+              type: object
+              description: Searches for rows which do not contain the specified column. The
+                object should simply contain keys of the column names, these can
+                map to any value.
+              example:
+                columnName1: ""
+            notEmpty:
+              type: object
+              description: Searches for rows which have the specified column.
+            oneOf:
+              type: object
+              description: Searches for rows which have a column value that is any of the
+                specified values. The format of this must be columnName ->
+                [value1, value2].
+        paginate:
+          type: boolean
+          description: Enables pagination, by default this is disabled.
+        bookmark:
+          oneOf:
+            - type: string
+            - type: integer
+          description: If retrieving another page, the bookmark from the previous request
+            must be supplied.
+        limit:
+          type: integer
+          description: The maximum number of rows to return, useful when paginating, for
+            internal tables this will be limited to 1000, for SQL tables it will
+            be 5000.
+        sort:
+          type: object
+          description: A set of parameters describing the sort behaviour of the search.
+          properties:
+            order:
+              type: string
+              enum:
+                - ascending
+                - descending
+              description: The order of the sort, by default this is ascending.
+            column:
+              type: string
+              description: The name of the column by which the rows will be sorted.
+            type:
+              type: string
+              enum:
+                - string
+                - number
+              description: Defines whether the column should be treated as a string or as
+                numbers when sorting.
+      required:
+        - query
     nameSearch:
       type: object
       properties:
@@ -1544,97 +1636,7 @@ paths:
         content:
           application/json:
             schema:
-              type: object
-              required:
-                - query
-              properties:
-                query:
-                  type: object
-                  properties:
-                    allOr:
-                      type: boolean
-                      description: Specifies that a row should be returned if it satisfies any of the
-                        specified options, rather than requiring it to fulfill
-                        all the search parameters. This defaults to false,
-                        meaning AND logic will be used.
-                    string:
-                      type: object
-                      example:
-                        columnName1: value
-                        columnName2: value
-                      description: A map of field name to the string to search for, this will look for
-                        rows that have a value starting with the string value.
-                      additionalProperties:
-                        type: string
-                        description: The value to search for in the column.
-                    fuzzy:
-                      type: object
-                      description: A fuzzy search, only supported by internal tables.
-                    range:
-                      type: object
-                      description: Searches within a range, the format of this must be in the format
-                        of an object with a "low" and "high" property.
-                      example:
-                        columnName1:
-                          low: 10
-                          high: 20
-                    equal:
-                      type: object
-                      description: Searches for rows that have a column value that is exactly the
-                        value set.
-                    notEqual:
-                      type: object
-                      description: Searches for any row which does not contain the specified column
-                        value.
-                    empty:
-                      type: object
-                      description: Searches for rows which do not contain the specified column. The
-                        object should simply contain keys of the column names,
-                        these can map to any value.
-                      example:
-                        columnName1: ""
-                    notEmpty:
-                      type: object
-                      description: Searches for rows which have the specified column.
-                    oneOf:
-                      type: object
-                      description: Searches for rows which have a column value that is any of the
-                        specified values. The format of this must be columnName
-                        -> [value1, value2].
-                paginate:
-                  type: boolean
-                  description: Enables pagination, by default this is disabled.
-                bookmark:
-                  oneOf:
-                    - type: string
-                    - type: integer
-                  description: If retrieving another page, the bookmark from the previous request
-                    must be supplied.
-                limit:
-                  type: integer
-                  description: The maximum number of rows to return, useful when paginating, for
-                    internal tables this will be limited to 1000, for SQL tables
-                    it will be 5000.
-                sort:
-                  type: object
-                  description: A set of parameters describing the sort behaviour of the search.
-                  properties:
-                    order:
-                      type: string
-                      enum:
-                        - ascending
-                        - descending
-                      description: The order of the sort, by default this is ascending.
-                    column:
-                      type: string
-                      description: The name of the column by which the rows will be sorted.
-                    type:
-                      type: string
-                      enum:
-                        - string
-                        - number
-                      description: Defines whether the column should be treated as a string or as
-                        numbers when sorting.
+              $ref: "#/components/schemas/rowSearch"
       responses:
         "200":
           description: The response will contain an array of rows that match the search
@@ -1665,7 +1667,7 @@ paths:
       responses:
         "200":
           description: Returns the created table, including the ID which has been
-            generated for it. This can be internal or external data sources.
+            generated for it. This can be internal or external datasources.
           content:
             application/json:
               schema:

package/specs/resources/misc.js

@@ -2,6 +2,122 @@ const { object } = require("./utils")
 const Resource = require("./utils/Resource")
 
 module.exports = new Resource().setSchemas({
+  rowSearch: object(
+    {
+      query: {
+        type: "object",
+        properties: {
+          allOr: {
+            type: "boolean",
+            description:
+              "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used.",
+          },
+          string: {
+            type: "object",
+            example: {
+              columnName1: "value",
+              columnName2: "value",
+            },
+            description:
+              "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
+            additionalProperties: {
+              type: "string",
+              description: "The value to search for in the column.",
+            },
+          },
+          fuzzy: {
+            type: "object",
+            description: "A fuzzy search, only supported by internal tables.",
+          },
+          range: {
+            type: "object",
+            description:
+              'Searches within a range, the format of this must be in the format of an object with a "low" and "high" property.',
+            example: {
+              columnName1: {
+                low: 10,
+                high: 20,
+              },
+            },
+          },
+          equal: {
+            type: "object",
+            description:
+              "Searches for rows that have a column value that is exactly the value set.",
+          },
+          notEqual: {
+            type: "object",
+            description:
+              "Searches for any row which does not contain the specified column value.",
+          },
+          empty: {
+            type: "object",
+            description:
+              "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
+            example: {
+              columnName1: "",
+            },
+          },
+          notEmpty: {
+            type: "object",
+            description: "Searches for rows which have the specified column.",
+          },
+          oneOf: {
+            type: "object",
+            description:
+              "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2].",
+          },
+        },
+      },
+      paginate: {
+        type: "boolean",
+        description: "Enables pagination, by default this is disabled.",
+      },
+      bookmark: {
+        oneOf: [
+          {
+            type: "string",
+          },
+          {
+            type: "integer",
+          },
+        ],
+        description:
+          "If retrieving another page, the bookmark from the previous request must be supplied.",
+      },
+      limit: {
+        type: "integer",
+        description:
+          "The maximum number of rows to return, useful when paginating, for internal tables this will be limited to 1000, for SQL tables it will be 5000.",
+      },
+      sort: {
+        type: "object",
+        description:
+          "A set of parameters describing the sort behaviour of the search.",
+        properties: {
+          order: {
+            type: "string",
+            enum: ["ascending", "descending"],
+            description: "The order of the sort, by default this is ascending.",
+          },
+          column: {
+            type: "string",
+            description:
+              "The name of the column by which the rows will be sorted.",
+          },
+          type: {
+            type: "string",
+            enum: ["string", "number"],
+            description:
+              "Defines whether the column should be treated as a string or as numbers when sorting.",
+          },
+        },
+      },
+    },
+    {
+      required: ["query"],
+    }
+  ),
   nameSearch: object({
     name: {
       type: "string",

package/specs/resources/query.js

@@ -82,7 +82,7 @@ const querySchema = object(
       type: "string",
     },
     datasourceId: {
-      description: "The ID of the data source the query belongs to.",
+      description: "The ID of the datasource the query belongs to.",
       type: "string",
     },
     parameters: {

package/src/api/controllers/application.ts

@@ -472,7 +472,6 @@ const destroyApp = async (ctx: any) => {
   const result = await db.destroy()
 
   if (isUnpublish) {
-    await quotas.removePublishedApp()
     await events.app.unpublished(app)
   } else {
     await quotas.removeApp()
@@ -572,7 +571,7 @@ export const sync = async (ctx: any, next: any) => {
   }
 }
 
-const updateAppPackage = async (appPackage: any, appId: any) => {
+export const updateAppPackage = async (appPackage: any, appId: any) => {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/auth.js

@@ -4,6 +4,15 @@ const { getFullUser } = require("../../utilities/users")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
+/**
+ * Add the attributes that are session based to the current user.
+ */
+const addSessionAttributesToUser = ctx => {
+  if (ctx.user) {
+    ctx.body.license = ctx.user.license
+  }
+}
+
 exports.fetchSelf = async ctx => {
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
@@ -55,4 +64,6 @@ exports.fetchSelf = async ctx => {
   } else {
     ctx.body = user
   }
+
+  addSessionAttributesToUser(ctx)
 }

package/src/api/controllers/component.ts

@@ -0,0 +1,57 @@
+import { DocumentType } from "../../db/utils"
+import { Plugin } from "@budibase/types"
+import { db as dbCore, context, tenancy } from "@budibase/backend-core"
+import { getComponentLibraryManifest } from "../../utilities/fileSystem"
+
+exports.fetchAppComponentDefinitions = async function (ctx: any) {
+  try {
+    const db = context.getAppDB()
+    const app = await db.get(DocumentType.APP_METADATA)
+
+    let componentManifests = await Promise.all(
+      app.componentLibraries.map(async (library: any) => {
+        let manifest = await getComponentLibraryManifest(library)
+        return {
+          manifest,
+          library,
+        }
+      })
+    )
+    const definitions: { [key: string]: any } = {}
+    for (let { manifest, library } of componentManifests) {
+      for (let key of Object.keys(manifest)) {
+        if (key === "features") {
+          definitions[key] = manifest[key]
+        } else {
+          const fullComponentName = `${library}/${key}`.toLowerCase()
+          definitions[fullComponentName] = {
+            component: fullComponentName,
+            ...manifest[key],
+          }
+        }
+      }
+    }
+
+    // Add custom components
+    const globalDB = tenancy.getGlobalDB()
+    const response = await globalDB.allDocs(
+      dbCore.getPluginParams(null, {
+        include_docs: true,
+      })
+    )
+    response.rows
+      .map((row: any) => row.doc)
+      .filter((plugin: Plugin) => plugin.schema.type === "component")
+      .forEach((plugin: Plugin) => {
+        const fullComponentName = `plugin/${plugin.name}`
+        definitions[fullComponentName] = {
+          component: fullComponentName,
+          ...plugin.schema.schema,
+        }
+      })
+
+    ctx.body = definitions
+  } catch (err) {
+    console.error(`component-definitions=failed`, err)
+  }
+}

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -50,9 +50,23 @@ exports.fetch = async function (ctx) {
 exports.buildSchemaFromDb = async function (ctx) {
   const db = getAppDB()
   const datasource = await db.get(ctx.params.datasourceId)
+  const tablesFilter = ctx.request.body.tablesFilter
 
-  const { tables, error } = await buildSchemaHelper(datasource)
-  datasource.entities = tables
+  let { tables, error } = await buildSchemaHelper(datasource)
+  if (tablesFilter) {
+    if (!datasource.entities) {
+      datasource.entities = {}
+    }
+    for (let key in tables) {
+      if (
+        tablesFilter.some(filter => filter.toLowerCase() === key.toLowerCase())
+      ) {
+        datasource.entities[key] = tables[key]
+      }
+    }
+  } else {
+    datasource.entities = tables
+  }
 
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
@@ -114,7 +128,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +163,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,15 +232,14 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = integrations[datasource.source]
+  const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)
-  datasource.entities = connector.tables
 
   // make sure they all have a display name selected
-  for (let entity of Object.values(datasource.entities)) {
+  for (let entity of Object.values(datasource.entities ?? {})) {
     if (entity.primaryDisplay) {
       continue
     }

package/src/api/controllers/deploy/index.ts

@@ -206,12 +206,7 @@ const _deployApp = async function (ctx: any) {
 
   console.log("Deploying app...")
 
-  let app
-  if (await isFirstDeploy()) {
-    app = await quotas.addPublishedApp(() => deployApp(deployment))
-  } else {
-    app = await deployApp(deployment)
-  }
+  let app = await deployApp(deployment)
 
   await events.app.published(app)
   ctx.body = deployment

package/src/api/controllers/integration.js

@@ -1,12 +1,11 @@
-const { cloneDeep } = require("lodash")
-const { definitions } = require("../../integrations")
+const { getDefinitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = cloneDeep(definitions)
+  const defs = await getDefinitions()
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
+  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = definitions[ctx.params.type]
+  ctx.body = defs[ctx.params.type]
 }

package/src/api/controllers/plugin/file.ts

@@ -0,0 +1,15 @@
+import {
+  createTempFolder,
+  getPluginMetadata,
+  extractTarball,
+} from "../../../utilities/fileSystem"
+
+export async function fileUpload(file: { name: string; path: string }) {
+  if (!file.name.endsWith(".tar.gz")) {
+    throw new Error("Plugin must be compressed into a gzipped tarball.")
+  }
+  const path = createTempFolder(file.name.split(".tar.gz")[0])
+  await extractTarball(file.path, path)
+
+  return await getPluginMetadata(path)
+}

package/src/api/controllers/plugin/github.ts

@@ -0,0 +1,75 @@
+import { getPluginMetadata } from "../../../utilities/fileSystem"
+import fetch from "node-fetch"
+import { downloadUnzipTarball } from "./utils"
+
+export async function request(
+  url: string,
+  headers: { [key: string]: string },
+  err: string
+) {
+  const response = await fetch(url, { headers })
+  if (response.status >= 300) {
+    const respErr = await response.text()
+    throw new Error(`Error: ${err} - ${respErr}`)
+  }
+  return response.json()
+}
+
+export async function githubUpload(url: string, name = "", token = "") {
+  let githubUrl = url
+
+  if (!githubUrl.includes("https://github.com/")) {
+    throw new Error("The plugin origin must be from Github")
+  }
+
+  if (url.includes(".git")) {
+    githubUrl = url.replace(".git", "")
+  }
+
+  const githubApiUrl = githubUrl.replace(
+    "https://github.com/",
+    "https://api.github.com/repos/"
+  )
+  const headers: any = token ? { Authorization: `Bearer ${token}` } : {}
+  const pluginDetails = await request(
+    githubApiUrl,
+    headers,
+    "Repository not found"
+  )
+  const pluginName = pluginDetails.name || name
+  const pluginLatestReleaseUrl = pluginDetails?.["releases_url"]
+    ? pluginDetails?.["releases_url"].replace("{/id}", "/latest")
+    : undefined
+  if (!pluginLatestReleaseUrl) {
+    throw new Error("Github release not found")
+  }
+
+  const pluginReleaseDetails = await request(
+    pluginLatestReleaseUrl,
+    headers,
+    "Github latest release not found"
+  )
+  const pluginReleaseTarballAsset = pluginReleaseDetails?.assets?.find(
+    (x: any) => x?.["content_type"] === "application/gzip"
+  )
+  const pluginLastReleaseTarballUrl =
+    pluginReleaseTarballAsset?.["browser_download_url"]
+  if (!pluginLastReleaseTarballUrl) {
+    throw new Error("Github latest release url not found")
+  }
+
+  try {
+    const path = await downloadUnzipTarball(
+      pluginLastReleaseTarballUrl,
+      pluginName,
+      headers
+    )
+    return await getPluginMetadata(path)
+  } catch (err: any) {
+    let errMsg = err?.message || err
+    if (errMsg === "unexpected response Not Found") {
+      errMsg = "Github release tarball not found"
+    }
+    throw new Error(errMsg)
+  }
+}