@budibase/server 0.9.169-alpha.0 → 0.9.169-alpha.4

package/dist/utilities/fileSystem/index.js

@@ -98,53 +98,69 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
+/**
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to backup
+ * @param {object} config Config to send to export DB
+ * @returns {*} either a string or a stream of the backup
+ */
+const backupAppData = (appId, config) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc._id.includes(LINK_USER_METADATA_PREFIX)) }));
+});
 /**
  * Takes a copy of the database state for an app to the object store.
  * @param {string} appId The ID of the app which is to be backed up.
  * @param {string} backupName The name of the backup located in the object store.
- * @return The backup has been completed when this promise completes and returns a file stream
- * to the temporary backup file (to return via API if required).
+ * @return {*} a readable stream to the completed backup file
  */
 exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0, function* () {
-    return exports.exportDB(appId, {
-        exportName: backupName,
-        filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc.includes(LINK_USER_METADATA_PREFIX)),
-    });
+    return yield backupAppData(appId, { exportName: backupName });
 });
 /**
- * exports a DB to either file or a variable (memory).
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true });
+});
+/**
+ * Exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - the file name to export to, if not in memory.
+ * @param {string} exportName optional - provide a filename to write the backup to a file
+ * @param {boolean} stream optional - whether to perform a full backup
  * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return Either the file stream or the variable (if no export name provided).
+ * @return {*} either a readable stream or a string
  */
-exports.exportDB = (dbName, { exportName, filter } = { exportName: undefined, filter: undefined }) => __awaiter(void 0, void 0, void 0, function* () {
-    let stream, appString = "", path = null;
-    if (exportName) {
-        path = join(budibaseTempDir(), exportName);
-        stream = fs.createWriteStream(path);
-    }
-    else {
-        stream = new MemoryStream();
-        stream.on("data", chunk => {
-            appString += chunk.toString();
-        });
-    }
-    // perform couch dump
+exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
     const instanceDb = new CouchDB(dbName);
-    yield instanceDb.dump(stream, {
-        filter,
-    });
-    // just in memory, return the final string
-    if (!exportName) {
-        return appString;
+    // Stream the dump if required
+    if (stream) {
+        const memStream = new MemoryStream();
+        instanceDb.dump(memStream, { filter });
+        return memStream;
     }
-    // write the file to the object store
-    if (env.SELF_HOSTED) {
-        yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+    // Write the dump to file if required
+    if (exportName) {
+        const path = join(budibaseTempDir(), exportName);
+        const writeStream = fs.createWriteStream(path);
+        yield instanceDb.dump(writeStream, { filter });
+        // Upload the dump to the object store if self hosted
+        if (env.SELF_HOSTED) {
+            yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+        }
+        return fs.createReadStream(path);
     }
-    return fs.createReadStream(path);
+    // Stringify the dump in memory if required
+    const memStream = new MemoryStream();
+    let appString = "";
+    memStream.on("data", chunk => {
+        appString += chunk.toString();
+    });
+    yield instanceDb.dump(memStream, { filter });
+    return appString;
 });
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.169-alpha.0",
+  "version": "0.9.169-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,9 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.169-alpha.0",
-    "@budibase/client": "^0.9.169-alpha.0",
-    "@budibase/string-templates": "^0.9.169-alpha.0",
+    "@budibase/auth": "^0.9.169-alpha.4",
+    "@budibase/client": "^0.9.169-alpha.4",
+    "@budibase/string-templates": "^0.9.169-alpha.4",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -149,5 +149,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "1d5494a7f5eb7a6e9d942c4aa5da9cff4bca9648"
+  "gitHead": "65077556b1a5e4bef39d3b6b2631ab9094cdc65f"
 }

package/src/api/controllers/backup.js

@@ -1,10 +1,9 @@
-const { performBackup } = require("../../utilities/fileSystem")
+const { streamBackup } = require("../../utilities/fileSystem")
 
 exports.exportAppDump = async function (ctx) {
   const { appId } = ctx.query
-  const appname = decodeURI(ctx.query.appname)
-  const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
-
+  const appName = decodeURI(ctx.query.appname)
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await performBackup(appId, backupIdentifier)
+  ctx.body = await streamBackup(appId)
 }

package/src/api/controllers/permission.js

@@ -147,6 +147,7 @@ exports.getResourcePerms = async function (ctx) {
       const rolePerms = role.permissions
       if (
         rolePerms &&
+        rolePerms[resourceId] &&
         (rolePerms[resourceId] === level ||
           rolePerms[resourceId].indexOf(level) !== -1)
       ) {

package/src/api/controllers/row/ExternalRequest.ts

@@ -15,8 +15,9 @@ import {
 import {
   breakRowIdField,
   generateRowIdField,
+  isRowId,
+  convertRowId,
 } from "../../../integrations/utils"
-import { RelationshipTypes } from "../../../constants"
 
 interface ManyRelationship {
   tableId?: string
@@ -36,7 +37,7 @@ interface RunConfig {
 
 module External {
   const { makeExternalQuery } = require("./utils")
-  const { DataSourceOperation, FieldTypes } = require("../../../constants")
+  const { DataSourceOperation, FieldTypes, RelationshipTypes } = require("../../../constants")
   const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
@@ -83,6 +84,48 @@ module External {
     }
   }
 
+  /**
+   * This function checks the incoming parameters to make sure all the inputs are
+   * valid based on on the table schema. The main thing this is looking for is when a
+   * user has made use of the _id field of a row for a foreign key or a search parameter.
+   * In these cases the key will be sent up as [1], rather than 1. In these cases we will
+   * simplify it down to the requirements. This function is quite complex as we try to be
+   * relatively restrictive over what types of columns we will perform this action for.
+   */
+  function cleanupConfig(config: RunConfig, table: Table): RunConfig {
+    const primaryOptions = [
+      FieldTypes.STRING,
+      FieldTypes.LONGFORM,
+      FieldTypes.OPTIONS,
+      FieldTypes.NUMBER,
+    ]
+    // filter out fields which cannot be keys
+    const fieldNames = Object.entries(table.schema)
+      .filter(schema => primaryOptions.find(val => val === schema[1].type))
+      .map(([fieldName]) => fieldName)
+    const iterateObject = (obj: { [key: string]: any }) => {
+      for (let [field, value] of Object.entries(obj)) {
+        if (fieldNames.find(name => name === field) && isRowId(value)) {
+          obj[field] = convertRowId(value)
+        }
+      }
+    }
+    // check the row and filters to make sure they aren't a key of some sort
+    if (config.filters) {
+      for (let filter of Object.values(config.filters)) {
+        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
+          continue
+        }
+        iterateObject(filter)
+      }
+    }
+    if (config.row) {
+      iterateObject(config.row)
+    }
+
+    return config
+  }
+
   function generateIdForRow(row: Row | undefined, table: Table): string {
     const primary = table.primary
     if (!row || !primary) {
@@ -509,7 +552,7 @@ module External {
       return fields
     }
 
-    async run({ id, row, filters, sort, paginate }: RunConfig) {
+    async run(config: RunConfig) {
       const { appId, operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
       if (!this.datasource) {
@@ -525,9 +568,11 @@ module External {
       if (!table) {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
-      // clean up row on ingress using schema
+      // look for specific components of config which may not be considered acceptable
+      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
+      // clean up row on ingress using schema
       const processed = this.inputProcessing(row, table)
       row = processed.row
       if (

package/src/integrations/utils.ts

@@ -5,6 +5,7 @@ const { DocumentTypes, SEPARATOR } = require("../db/utils")
 const { FieldTypes } = require("../constants")
 
 const DOUBLE_SEPARATOR = `${SEPARATOR}${SEPARATOR}`
+const ROW_ID_REGEX = /^\[.*]$/g
 
 export function isExternalTable(tableId: string) {
   return tableId.includes(DocumentTypes.DATASOURCE)
@@ -32,6 +33,20 @@ export function generateRowIdField(keyProps: any[] = []) {
   return encodeURIComponent(JSON.stringify(keyProps).replace(/"/g, "'"))
 }
 
+export function isRowId(field: any) {
+  return Array.isArray(field) || (typeof field === "string" && field.match(ROW_ID_REGEX) != null)
+}
+
+export function convertRowId(field: any) {
+  if (Array.isArray(field)) {
+    return field[0]
+  }
+  if (typeof field === "string" && field.match(ROW_ID_REGEX) != null) {
+    return field.substring(1, field.length - 1)
+  }
+  return field
+}
+
 // should always return an array
 export function breakRowIdField(_id: string | { _id: string }): any[] {
   if (!_id) {

package/src/utilities/fileSystem/index.js

@@ -106,64 +106,86 @@ exports.apiFileReturn = contents => {
 }
 
 /**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return The backup has been completed when this promise completes and returns a file stream
- * to the temporary backup file (to return via API if required).
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to backup
+ * @param {object} config Config to send to export DB
+ * @returns {*} either a string or a stream of the backup
  */
-exports.performBackup = async (appId, backupName) => {
-  return exports.exportDB(appId, {
-    exportName: backupName,
+const backupAppData = async (appId, config) => {
+  return await exports.exportDB(appId, {
+    ...config,
     filter: doc =>
       !(
         doc._id.includes(USER_METDATA_PREFIX) ||
-        doc.includes(LINK_USER_METADATA_PREFIX)
+        doc._id.includes(LINK_USER_METADATA_PREFIX)
       ),
   })
 }
 
 /**
- * exports a DB to either file or a variable (memory).
+ * Takes a copy of the database state for an app to the object store.
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {string} backupName The name of the backup located in the object store.
+ * @return {*} a readable stream to the completed backup file
+ */
+exports.performBackup = async (appId, backupName) => {
+  return await backupAppData(appId, { exportName: backupName })
+}
+
+/**
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+exports.streamBackup = async appId => {
+  return await backupAppData(appId, { stream: true })
+}
+
+/**
+ * Exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - the file name to export to, if not in memory.
+ * @param {string} exportName optional - provide a filename to write the backup to a file
+ * @param {boolean} stream optional - whether to perform a full backup
  * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return Either the file stream or the variable (if no export name provided).
+ * @return {*} either a readable stream or a string
  */
-exports.exportDB = async (
-  dbName,
-  { exportName, filter } = { exportName: undefined, filter: undefined }
-) => {
-  let stream,
-    appString = "",
-    path = null
-  if (exportName) {
-    path = join(budibaseTempDir(), exportName)
-    stream = fs.createWriteStream(path)
-  } else {
-    stream = new MemoryStream()
-    stream.on("data", chunk => {
-      appString += chunk.toString()
-    })
-  }
-  // perform couch dump
+exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
   const instanceDb = new CouchDB(dbName)
-  await instanceDb.dump(stream, {
-    filter,
-  })
-  // just in memory, return the final string
-  if (!exportName) {
-    return appString
+
+  // Stream the dump if required
+  if (stream) {
+    const memStream = new MemoryStream()
+    instanceDb.dump(memStream, { filter })
+    return memStream
   }
-  // write the file to the object store
-  if (env.SELF_HOSTED) {
-    await streamUpload(
-      ObjectStoreBuckets.BACKUPS,
-      join(dbName, exportName),
-      fs.createReadStream(path)
-    )
+
+  // Write the dump to file if required
+  if (exportName) {
+    const path = join(budibaseTempDir(), exportName)
+    const writeStream = fs.createWriteStream(path)
+    await instanceDb.dump(writeStream, { filter })
+
+    // Upload the dump to the object store if self hosted
+    if (env.SELF_HOSTED) {
+      await streamUpload(
+        ObjectStoreBuckets.BACKUPS,
+        join(dbName, exportName),
+        fs.createReadStream(path)
+      )
+    }
+
+    return fs.createReadStream(path)
   }
-  return fs.createReadStream(path)
+
+  // Stringify the dump in memory if required
+  const memStream = new MemoryStream()
+  let appString = ""
+  memStream.on("data", chunk => {
+    appString += chunk.toString()
+  })
+  await instanceDb.dump(memStream, { filter })
+  return appString
 }
 
 /**