@budibase/server 1.3.12 → 1.3.15-alpha.0

package/dist/utilities/fileSystem/index.js

@@ -16,12 +16,16 @@ const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/d
 const { ObjectStoreBuckets } = require("../../constants");
 const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
+const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
+const tar = require("tar");
+const fetch = require("node-fetch");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -87,6 +91,12 @@ exports.getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, func
 exports.loadHandlebarsFile = path => {
     return fs.readFileSync(path, "utf8");
 };
+/**
+ * Same as above just with a different name.
+ */
+exports.loadJSFile = (directory, name) => {
+    return fs.readFileSync(join(directory, name), "utf8");
+};
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
@@ -110,11 +120,11 @@ exports.defineFilter = excludeRows => {
  * data or user relationships.
  * @param {string} appId The app to backup
  * @param {object} config Config to send to export DB
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} either a string or a stream of the backup
  */
-const backupAppData = (appId, config, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(includeRows) }));
+const backupAppData = (appId, config, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: exports.defineFilter(excludeRows) }));
 });
 /**
  * Takes a copy of the database state for an app to the object store.
@@ -128,11 +138,11 @@ exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0,
 /**
  * Streams a backup of the database state for an app
  * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} includeRows Flag to state whether the export should include data.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
  * @returns {*} a readable stream of the backup which is written in real time
  */
-exports.streamBackup = (appId, includeRows) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true }, includeRows);
+exports.streamBackup = (appId, excludeRows) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true }, excludeRows);
 });
 /**
  * Exports a DB to either file or a variable (memory).
@@ -285,6 +295,61 @@ exports.cleanup = appIds => {
         }
     }
 };
+exports.extractPluginTarball = (file) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!file.name.endsWith(".tar.gz")) {
+        throw new Error("Plugin must be compressed into a gzipped tarball.");
+    }
+    const path = join(budibaseTempDir(), file.name.split(".tar.gz")[0]);
+    // remove old tmp directories automatically - don't combine
+    if (fs.existsSync(path)) {
+        fs.rmSync(path, { recursive: true, force: true });
+    }
+    fs.mkdirSync(path);
+    yield tar.extract({
+        file: file.path,
+        C: path,
+    });
+    let metadata = {};
+    try {
+        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
+        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
+        metadata.schema = JSON.parse(schema);
+        metadata.package = JSON.parse(pkg);
+    }
+    catch (err) {
+        throw new Error("Unable to process schema.json/package.json in plugin.");
+    }
+    return { metadata, directory: path };
+});
+exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!fs.existsSync(DATASOURCE_PATH)) {
+        fs.mkdirSync(DATASOURCE_PATH);
+    }
+    const filename = join(DATASOURCE_PATH, name);
+    const metadataName = `${filename}.bbmetadata`;
+    if (fs.existsSync(filename)) {
+        const currentHash = fs.readFileSync(metadataName, "utf8");
+        // if hash is the same return the file, otherwise remove it and re-download
+        if (currentHash === hash) {
+            return require(filename);
+        }
+        else {
+            console.log(`Updating plugin: ${name}`);
+            fs.unlinkSync(filename);
+        }
+    }
+    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
+    const response = yield fetch(fullUrl);
+    if (response.status === 200) {
+        const content = yield response.text();
+        fs.writeFileSync(filename, content);
+        fs.writeFileSync(metadataName, hash);
+        require(filename);
+    }
+    else {
+        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
+    }
+});
 /**
  * Full function definition for below can be found in the utilities.
  */

package/dist/utilities/index.js

@@ -150,3 +150,11 @@ exports.convertBookmark = bookmark => {
     }
     return bookmark;
 };
+exports.isQsTrue = param => {
+    if (typeof param === "string") {
+        return param.toLowerCase() === "true";
+    }
+    else {
+        return param === true;
+    }
+};

package/dist/utilities/rowProcessor/utils.js

@@ -64,7 +64,7 @@ exports.processDates = (table, rows) => {
         if (schema.type !== FieldTypes.DATETIME) {
             continue;
         }
-        if (!schema.ignoreTimezones) {
+        if (!schema.timeOnly && !schema.ignoreTimezones) {
             datesWithTZ.push(column);
         }
     }

package/dist/utilities/usageQuota/rows.js

@@ -72,7 +72,8 @@ exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function*
         // ensure uniqueness on a per app pair basis
         // this can't be done on all rows because app import results in
         // duplicate row ids across apps
-        uniqueRows = uniqueRows.concat(...new Set(appRows));
+        // the array pre-concat is important to avoid stack overflow
+        uniqueRows = uniqueRows.concat([...new Set(appRows)]);
     }
     return uniqueRows;
 });

package/dist/watch.js

@@ -0,0 +1,77 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.watch = void 0;
+const path_1 = __importDefault(require("path"));
+const env = __importStar(require("./environment"));
+const chokidar_1 = __importDefault(require("chokidar"));
+const fs_1 = __importDefault(require("fs"));
+const backend_core_1 = require("@budibase/backend-core");
+const constants_1 = require("@budibase/backend-core/constants");
+const plugin_1 = require("./api/controllers/plugin");
+function watch() {
+    const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
+    chokidar_1.default
+        .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: {
+            pollInterval: 100,
+            stabilityThreshold: 250,
+        },
+        usePolling: true,
+        interval: 250,
+    })
+        .on("all", (event, path) => __awaiter(this, void 0, void 0, function* () {
+        // Sanity checks
+        if (!(path === null || path === void 0 ? void 0 : path.endsWith(".tar.gz")) || !fs_1.default.existsSync(path)) {
+            return;
+        }
+        yield backend_core_1.tenancy.doInTenant(constants_1.DEFAULT_TENANT_ID, () => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const split = path.split("/");
+                const name = split[split.length - 1];
+                console.log("Importing plugin:", path);
+                yield (0, plugin_1.processPlugin)({ name, path });
+            }
+            catch (err) {
+                const message = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
+                console.error("Failed to import plugin:", message);
+            }
+        }));
+    }));
+}
+exports.watch = watch;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.12",
+  "version": "1.3.15-alpha.0",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.3.12",
-    "@budibase/client": "^1.3.12",
-    "@budibase/pro": "1.3.11",
-    "@budibase/string-templates": "^1.3.12",
-    "@budibase/types": "^1.3.12",
+    "@budibase/backend-core": "1.3.15-alpha.0",
+    "@budibase/client": "1.3.15-alpha.0",
+    "@budibase/pro": "1.3.13",
+    "@budibase/string-templates": "1.3.15-alpha.0",
+    "@budibase/types": "1.3.15-alpha.0",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
+    "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
@@ -140,6 +141,7 @@
     "snowflake-promise": "^4.5.0",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
+    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -196,5 +198,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "7d458faa8915a9aa7d975a3795d0f76bc929e7be"
+  "gitHead": "38f3c6d1d713f5e1bd18fdd81bdffb9bf7d4f9d3"
 }

package/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {

package/src/api/controllers/application.ts

@@ -553,11 +553,7 @@ export const sync = async (ctx: any, next: any) => {
   })
   let error
   try {
-    await replication.replicate({
-      filter: function (doc: any) {
-        return doc._id !== DocumentType.APP_METADATA
-      },
-    })
+    await replication.replicate(replication.appReplicateOpts())
   } catch (err) {
     error = err
   } finally {
@@ -576,7 +572,7 @@ export const sync = async (ctx: any, next: any) => {
   }
 }
 
-const updateAppPackage = async (appPackage: any, appId: any) => {
+export const updateAppPackage = async (appPackage: any, appId: any) => {
   return context.doInAppContext(appId, async () => {
     const db = context.getAppDB()
     const application = await db.get(DocumentType.APP_METADATA)

package/src/api/controllers/backup.js

@@ -1,11 +1,12 @@
 const { streamBackup } = require("../../utilities/fileSystem")
 const { events, context } = require("@budibase/backend-core")
 const { DocumentType } = require("../../db/utils")
+const { isQsTrue } = require("../../utilities")
 
 exports.exportAppDump = async function (ctx) {
   let { appId, excludeRows } = ctx.query
   const appName = decodeURI(ctx.query.appname)
-  excludeRows = excludeRows === "true"
+  excludeRows = isQsTrue(excludeRows)
   const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
   ctx.body = await streamBackup(appId, excludeRows)

package/src/api/controllers/component.js

@@ -1,6 +1,8 @@
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, getPluginParams } = require("../../db/utils")
 const { getComponentLibraryManifest } = require("../../utilities/fileSystem")
 const { getAppDB } = require("@budibase/backend-core/context")
+const { getGlobalDB } = require("@budibase/backend-core/tenancy")
+const env = require("../../environment")
 
 exports.fetchAppComponentDefinitions = async function (ctx) {
   try {
@@ -10,7 +12,6 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
     let componentManifests = await Promise.all(
       app.componentLibraries.map(async library => {
         let manifest = await getComponentLibraryManifest(library)
-
         return {
           manifest,
           library,
@@ -31,6 +32,28 @@ exports.fetchAppComponentDefinitions = async function (ctx) {
         }
       }
     }
+
+    // for now custom components only supported in self-host
+    if (env.SELF_HOSTED) {
+      // Add custom components
+      const globalDB = getGlobalDB()
+      const response = await globalDB.allDocs(
+        getPluginParams(null, {
+          include_docs: true,
+        })
+      )
+      response.rows
+        .map(row => row.doc)
+        .filter(plugin => plugin.schema.type === "component")
+        .forEach(plugin => {
+          const fullComponentName = `plugin/${plugin.name}`
+          definitions[fullComponentName] = {
+            component: fullComponentName,
+            ...plugin.schema.schema,
+          }
+        })
+    }
+
     ctx.body = definitions
   } catch (err) {
     console.error(`component-definitions=failed`, err)

package/src/api/controllers/datasource.js

@@ -7,7 +7,7 @@ const {
   getTableParams,
 } = require("../../db/utils")
 const { BuildSchemaErrors, InvalidColumns } = require("../../constants")
-const { integrations } = require("../../integrations")
+const { getIntegration } = require("../../integrations")
 const { getDatasourceAndQuery } = require("./row/utils")
 const { invalidateDynamicVariables } = require("../../threads/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -114,7 +114,7 @@ exports.update = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -149,7 +149,7 @@ exports.save = async function (ctx) {
 
   // Drain connection pools when configuration is changed
   if (datasource.source) {
-    const source = integrations[datasource.source]
+    const source = await getIntegration(datasource.source)
     if (source && source.pool) {
       await source.pool.end()
     }
@@ -218,7 +218,7 @@ function updateError(error, newError, tables) {
 }
 
 const buildSchemaHelper = async datasource => {
-  const Connector = integrations[datasource.source]
+  const Connector = await getIntegration(datasource.source)
 
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)

package/src/api/controllers/deploy/index.ts

@@ -15,6 +15,7 @@ import {
   getAppId,
   getAppDB,
   getProdAppDB,
+  getDevAppDB,
 } from "@budibase/backend-core/context"
 import { quotas } from "@budibase/pro"
 import { events } from "@budibase/backend-core"
@@ -110,17 +111,29 @@ async function deployApp(deployment: any) {
       target: productionAppId,
     }
     replication = new Replication(config)
-
+    const devDb = getDevAppDB()
+    console.log("Compacting development DB")
+    await devDb.compact()
     console.log("Replication object created")
-    await replication.replicate()
+    await replication.replicate(replication.appReplicateOpts())
     console.log("replication complete.. replacing app meta doc")
+    // app metadata is excluded as it is likely to be in conflict
+    // replicate the app metadata document manually
     const db = getProdAppDB()
-    const appDoc = await db.get(DocumentType.APP_METADATA)
+    const appDoc = await devDb.get(DocumentType.APP_METADATA)
+    try {
+      const prodAppDoc = await db.get(DocumentType.APP_METADATA)
+      appDoc._rev = prodAppDoc._rev
+    } catch (err) {
+      delete appDoc._rev
+    }
 
+    // switch to production app ID
     deployment.appUrl = appDoc.url
-
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
+    // remove automation errors if they exist
+    delete appDoc.automationErrors
     await db.put(appDoc)
     await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")

package/src/api/controllers/integration.js

@@ -1,12 +1,11 @@
-const { cloneDeep } = require("lodash")
-const { definitions } = require("../../integrations")
+const { getDefinitions } = require("../../integrations")
 const { SourceName } = require("@budibase/types")
 const googlesheets = require("../../integrations/googlesheets")
 const { featureFlags } = require("@budibase/backend-core")
 
 exports.fetch = async function (ctx) {
   ctx.status = 200
-  const defs = cloneDeep(definitions)
+  const defs = await getDefinitions()
 
   // for google sheets integration google verification
   if (featureFlags.isEnabled(featureFlags.FeatureFlag.GOOGLE_SHEETS)) {
@@ -17,6 +16,7 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
+  const defs = await getDefinitions()
   ctx.status = 200
-  ctx.body = definitions[ctx.params.type]
+  ctx.body = defs[ctx.params.type]
 }

package/src/api/controllers/plugin.ts

@@ -0,0 +1,110 @@
+import { ObjectStoreBuckets } from "../../constants"
+import { extractPluginTarball, loadJSFile } from "../../utilities/fileSystem"
+import { getGlobalDB } from "@budibase/backend-core/tenancy"
+import { generatePluginID, getPluginParams } from "../../db/utils"
+import { uploadDirectory } from "@budibase/backend-core/objectStore"
+import { PluginType, FileType } from "@budibase/types"
+import env from "../../environment"
+
+export async function getPlugins(type?: PluginType) {
+  const db = getGlobalDB()
+  const response = await db.allDocs(
+    getPluginParams(null, {
+      include_docs: true,
+    })
+  )
+  const plugins = response.rows.map((row: any) => row.doc)
+  if (type) {
+    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+  } else {
+    return plugins
+  }
+}
+
+export async function upload(ctx: any) {
+  const plugins: FileType[] =
+    ctx.request.files.file.length > 1
+      ? Array.from(ctx.request.files.file)
+      : [ctx.request.files.file]
+  try {
+    let docs = []
+    // can do single or multiple plugins
+    for (let plugin of plugins) {
+      const doc = await processPlugin(plugin)
+      docs.push(doc)
+    }
+    ctx.body = {
+      message: "Plugin(s) uploaded successfully",
+      plugins: docs,
+    }
+  } catch (err: any) {
+    const errMsg = err?.message ? err?.message : err
+    ctx.throw(400, `Failed to import plugin: ${errMsg}`)
+  }
+}
+
+export async function fetch(ctx: any) {
+  ctx.body = await getPlugins()
+}
+
+export async function destroy(ctx: any) {}
+
+export async function processPlugin(plugin: FileType) {
+  if (!env.SELF_HOSTED) {
+    throw new Error("Plugins not supported outside of self-host.")
+  }
+  const db = getGlobalDB()
+  const { metadata, directory } = await extractPluginTarball(plugin)
+  const version = metadata.package.version,
+    name = metadata.package.name,
+    description = metadata.package.description
+
+  // first open the tarball into tmp directory
+  const bucketPath = `${name}/`
+  const files = await uploadDirectory(
+    ObjectStoreBuckets.PLUGINS,
+    directory,
+    bucketPath
+  )
+  const jsFile = files.find((file: any) => file.name.endsWith(".js"))
+  if (!jsFile) {
+    throw new Error(`Plugin missing .js file.`)
+  }
+  // validate the JS for a datasource
+  if (metadata.schema.type === PluginType.DATASOURCE) {
+    const js = loadJSFile(directory, jsFile.name)
+    // TODO: this isn't safe - but we need full node environment
+    // in future we should do this in a thread for safety
+    try {
+      eval(js)
+    } catch (err: any) {
+      const message = err?.message ? err.message : JSON.stringify(err)
+      throw new Error(`JS invalid: ${message}`)
+    }
+  }
+  const jsFileName = jsFile.name
+  const pluginId = generatePluginID(name)
+
+  // overwrite existing docs entirely if they exist
+  let rev
+  try {
+    const existing = await db.get(pluginId)
+    rev = existing._rev
+  } catch (err) {
+    rev = undefined
+  }
+  const doc = {
+    _id: pluginId,
+    _rev: rev,
+    ...metadata,
+    name,
+    version,
+    description,
+    jsUrl: `${bucketPath}${jsFileName}`,
+  }
+  const response = await db.put(doc)
+  return {
+    ...doc,
+    _rev: response.rev,
+  }
+}

package/src/api/controllers/row/ExternalRequest.ts

@@ -534,7 +534,7 @@ module External {
         })
         // this is the response from knex if no rows found
         const rows = !response[0].read ? response : []
-        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : manyKey
+        const storeTo = isMany ? field.throughFrom || linkPrimaryKey : fieldName
         related[storeTo] = { rows, isMany, tableId }
       }
       return related