@budibase/server 0.9.169-alpha.9 → 0.9.172

package/dist/utilities/fileSystem/index.js

@@ -98,69 +98,53 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
-/**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @returns {*} either a string or a stream of the backup
- */
-const backupAppData = (appId, config) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc._id.includes(LINK_USER_METADATA_PREFIX)) }));
-});
 /**
  * Takes a copy of the database state for an app to the object store.
  * @param {string} appId The ID of the app which is to be backed up.
  * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
+ * @return The backup has been completed when this promise completes and returns a file stream
+ * to the temporary backup file (to return via API if required).
  */
 exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { exportName: backupName });
-});
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* () {
-    return yield backupAppData(appId, { stream: true });
+    return exports.exportDB(appId, {
+        exportName: backupName,
+        filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc.includes(LINK_USER_METADATA_PREFIX)),
+    });
 });
 /**
- * Exports a DB to either file or a variable (memory).
+ * exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
+ * @param {string} exportName optional - the file name to export to, if not in memory.
  * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
+ * @return Either the file stream or the variable (if no export name provided).
  */
-exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
-    const instanceDb = new CouchDB(dbName);
-    // Stream the dump if required
-    if (stream) {
-        const memStream = new MemoryStream();
-        instanceDb.dump(memStream, { filter });
-        return memStream;
-    }
-    // Write the dump to file if required
+exports.exportDB = (dbName, { exportName, filter } = { exportName: undefined, filter: undefined }) => __awaiter(void 0, void 0, void 0, function* () {
+    let stream, appString = "", path = null;
     if (exportName) {
-        const path = join(budibaseTempDir(), exportName);
-        const writeStream = fs.createWriteStream(path);
-        yield instanceDb.dump(writeStream, { filter });
-        // Upload the dump to the object store if self hosted
-        if (env.SELF_HOSTED) {
-            yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
-        }
-        return fs.createReadStream(path);
+        path = join(budibaseTempDir(), exportName);
+        stream = fs.createWriteStream(path);
+    }
+    else {
+        stream = new MemoryStream();
+        stream.on("data", chunk => {
+            appString += chunk.toString();
+        });
     }
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream();
-    let appString = "";
-    memStream.on("data", chunk => {
-        appString += chunk.toString();
+    // perform couch dump
+    const instanceDb = new CouchDB(dbName);
+    yield instanceDb.dump(stream, {
+        filter,
     });
-    yield instanceDb.dump(memStream, { filter });
-    return appString;
+    // just in memory, return the final string
+    if (!exportName) {
+        return appString;
+    }
+    // write the file to the object store
+    if (env.SELF_HOSTED) {
+        yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+    }
+    return fs.createReadStream(path);
 });
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.

package/dist/utilities/global.js

@@ -33,7 +33,7 @@ exports.updateAppRole = (appId, user) => {
         user.roleId = BUILTIN_ROLE_IDS.ADMIN;
     }
     else if (!user.roleId) {
-        user.roleId = BUILTIN_ROLE_IDS.PUBLIC;
+        user.roleId = BUILTIN_ROLE_IDS.BASIC;
     }
     delete user.roles;
     return user;

package/dist/utilities/rowProcessor/index.js

@@ -149,10 +149,6 @@ exports.processAutoColumn = processAutoColumn;
  * @returns {object} The coerced value
  */
 exports.coerce = (row, type) => {
-    // no coercion specified for type, skip it
-    if (!TYPE_TRANSFORM_MAP[type]) {
-        return row;
-    }
     // eslint-disable-next-line no-prototype-builtins
     if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
         return TYPE_TRANSFORM_MAP[type][row];
@@ -194,10 +190,6 @@ exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships:
             clonedRow[key] = exports.coerce(value, field.type);
         }
     }
-    if (!clonedRow._id || !clonedRow._rev) {
-        clonedRow._id = row._id;
-        clonedRow._rev = row._rev;
-    }
     // handle auto columns - this returns an object like {table, row}
     return processAutoColumn(user, copiedTable, clonedRow, opts);
 };

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.169-alpha.9",
+  "version": "0.9.172",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,13 +68,13 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.169-alpha.9",
-    "@budibase/client": "^0.9.169-alpha.9",
-    "@budibase/string-templates": "^0.9.169-alpha.9",
+    "@budibase/auth": "^0.9.172",
+    "@budibase/client": "^0.9.172",
+    "@budibase/string-templates": "^0.9.172",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
-    "@sentry/node": "^6.0.0",
+    "@sentry/node": "5.19.2",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "^2.767.0",
@@ -119,7 +119,6 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
-    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -149,5 +148,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "00e2f5b2201a0cf582c38213855284c37fa67863"
+  "gitHead": "f33cc5a9cd747ffb8491af36ac7baf0bf66de290"
 }

package/src/api/controllers/backup.js

@@ -1,9 +1,10 @@
-const { streamBackup } = require("../../utilities/fileSystem")
+const { performBackup } = require("../../utilities/fileSystem")
 
 exports.exportAppDump = async function (ctx) {
   const { appId } = ctx.query
-  const appName = decodeURI(ctx.query.appname)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
+  const appname = decodeURI(ctx.query.appname)
+  const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
+
   ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId)
+  ctx.body = await performBackup(appId, backupIdentifier)
 }

package/src/api/controllers/dev.js

@@ -7,13 +7,11 @@ const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
 
-async function redirect(ctx, method, path = "global") {
+async function redirect(ctx, method) {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
-    checkSlashesInUrl(
-      `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
-    ),
+    checkSlashesInUrl(`${env.WORKER_URL}/api/global/${devPath}?${queryString}`),
     request(
       ctx,
       {
@@ -43,22 +41,16 @@ async function redirect(ctx, method, path = "global") {
   ctx.cookies
 }
 
-exports.buildRedirectGet = path => {
-  return async ctx => {
-    await redirect(ctx, "GET", path)
-  }
+exports.redirectGet = async ctx => {
+  await redirect(ctx, "GET")
 }
 
-exports.buildRedirectPost = path => {
-  return async ctx => {
-    await redirect(ctx, "POST", path)
-  }
+exports.redirectPost = async ctx => {
+  await redirect(ctx, "POST")
 }
 
-exports.buildRedirectDelete = path => {
-  return async ctx => {
-    await redirect(ctx, "DELETE", path)
-  }
+exports.redirectDelete = async ctx => {
+  await redirect(ctx, "DELETE")
 }
 
 exports.clearLock = async ctx => {

package/src/api/controllers/permission.js

@@ -147,7 +147,6 @@ exports.getResourcePerms = async function (ctx) {
       const rolePerms = role.permissions
       if (
         rolePerms &&
-        rolePerms[resourceId] &&
         (rolePerms[resourceId] === level ||
           rolePerms[resourceId].indexOf(level) !== -1)
       ) {

package/src/api/controllers/query.js

@@ -4,7 +4,6 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
-const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -29,37 +28,10 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  return resp
-}
-
-async function runAndTransform(
-  integration,
-  queryVerb,
-  enrichedQuery,
-  transformer
-) {
-  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
-
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
-
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
+  if (!Array.isArray(resp)) {
+    resp = [resp]
   }
-
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
-  }
-
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
-
-  return { rows, keys }
+  return resp
 }
 
 exports.fetch = async function (ctx) {
@@ -150,16 +122,15 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb, transformer } = ctx.request.body
+  const { fields, parameters, queryVerb } = ctx.request.body
+
   const enrichedQuery = await enrichQueryFields(fields, parameters)
+
   const integration = new Integration(datasource.config)
+  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  const { rows, keys } = await runAndTransform(
-    integration,
-    queryVerb,
-    enrichedQuery,
-    transformer
-  )
+  // get all the potential fields in the schema
+  const keys = rows.flatMap(Object.keys)
 
   ctx.body = {
     rows,
@@ -187,16 +158,10 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-  const integration = new Integration(datasource.config)
 
+  const integration = new Integration(datasource.config)
   // call the relevant CRUD method on the integration class
-  const { rows } = await runAndTransform(
-    integration,
-    query.queryVerb,
-    enrichedQuery,
-    query.transformer
-  )
-  ctx.body = rows
+  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
   // cleanup
   if (integration.end) {
     integration.end()

package/src/api/controllers/row/ExternalRequest.ts

@@ -15,9 +15,8 @@ import {
 import {
   breakRowIdField,
   generateRowIdField,
-  isRowId,
-  convertRowId,
 } from "../../../integrations/utils"
+import { RelationshipTypes } from "../../../constants"
 
 interface ManyRelationship {
   tableId?: string
@@ -37,7 +36,7 @@ interface RunConfig {
 
 module External {
   const { makeExternalQuery } = require("./utils")
-  const { DataSourceOperation, FieldTypes, RelationshipTypes } = require("../../../constants")
+  const { DataSourceOperation, FieldTypes } = require("../../../constants")
   const { breakExternalTableId, isSQL } = require("../../../integrations/utils")
   const { processObjectSync } = require("@budibase/string-templates")
   const { cloneDeep } = require("lodash/fp")
@@ -84,48 +83,6 @@ module External {
     }
   }
 
-  /**
-   * This function checks the incoming parameters to make sure all the inputs are
-   * valid based on on the table schema. The main thing this is looking for is when a
-   * user has made use of the _id field of a row for a foreign key or a search parameter.
-   * In these cases the key will be sent up as [1], rather than 1. In these cases we will
-   * simplify it down to the requirements. This function is quite complex as we try to be
-   * relatively restrictive over what types of columns we will perform this action for.
-   */
-  function cleanupConfig(config: RunConfig, table: Table): RunConfig {
-    const primaryOptions = [
-      FieldTypes.STRING,
-      FieldTypes.LONGFORM,
-      FieldTypes.OPTIONS,
-      FieldTypes.NUMBER,
-    ]
-    // filter out fields which cannot be keys
-    const fieldNames = Object.entries(table.schema)
-      .filter(schema => primaryOptions.find(val => val === schema[1].type))
-      .map(([fieldName]) => fieldName)
-    const iterateObject = (obj: { [key: string]: any }) => {
-      for (let [field, value] of Object.entries(obj)) {
-        if (fieldNames.find(name => name === field) && isRowId(value)) {
-          obj[field] = convertRowId(value)
-        }
-      }
-    }
-    // check the row and filters to make sure they aren't a key of some sort
-    if (config.filters) {
-      for (let filter of Object.values(config.filters)) {
-        if (typeof filter !== "object" || Object.keys(filter).length === 0) {
-          continue
-        }
-        iterateObject(filter)
-      }
-    }
-    if (config.row) {
-      iterateObject(config.row)
-    }
-
-    return config
-  }
-
   function generateIdForRow(row: Row | undefined, table: Table): string {
     const primary = table.primary
     if (!row || !primary) {
@@ -552,7 +509,7 @@ module External {
       return fields
     }
 
-    async run(config: RunConfig) {
+    async run({ id, row, filters, sort, paginate }: RunConfig) {
       const { appId, operation, tableId } = this
       let { datasourceId, tableName } = breakExternalTableId(tableId)
       if (!this.datasource) {
@@ -568,11 +525,9 @@ module External {
       if (!table) {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
-      // look for specific components of config which may not be considered acceptable
-      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
+      // clean up row on ingress using schema
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
-      // clean up row on ingress using schema
       const processed = this.inputProcessing(row, table)
       row = processed.row
       if (

package/src/api/controllers/script.js

@@ -1,9 +1,24 @@
-const ScriptRunner = require("../../utilities/scriptRunner")
+const fetch = require("node-fetch")
+const vm = require("vm")
+
+class ScriptExecutor {
+  constructor(body) {
+    const code = `let fn = () => {\n${body.script}\n}; out = fn();`
+    this.script = new vm.Script(code)
+    this.context = vm.createContext(body.context)
+    this.context.fetch = fetch
+  }
+
+  execute() {
+    this.script.runInContext(this.context)
+    return this.context.out
+  }
+}
 
 exports.execute = async function (ctx) {
-  const { script, context } = ctx.request.body
-  const runner = new ScriptRunner(script, context)
-  ctx.body = runner.execute()
+  const executor = new ScriptExecutor(ctx.request.body)
+
+  ctx.body = executor.execute()
 }
 
 exports.save = async function (ctx) {

package/src/api/controllers/table/index.js

@@ -9,7 +9,11 @@ const {
   BudibaseInternalDB,
 } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions, getTable } = require("./utils")
+const { TableSaveFunctions, getExternalTable } = require("./utils")
+const {
+  isExternalTable,
+  breakExternalTableId,
+} = require("../../../integrations/utils")
 
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
@@ -44,8 +48,14 @@ exports.fetch = async function (ctx) {
 }
 
 exports.find = async function (ctx) {
+  const db = new CouchDB(ctx.appId)
   const tableId = ctx.params.id
-  ctx.body = await getTable(ctx.appId, tableId)
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    ctx.body = await getExternalTable(ctx.appId, datasourceId, tableName)
+  } else {
+    ctx.body = await db.get(ctx.params.id)
+  }
 }
 
 exports.save = async function (ctx) {
@@ -81,9 +91,6 @@ exports.save = async function (ctx) {
     for (let propKey of Object.keys(tableToSave.schema)) {
       let column = tableToSave.schema[propKey]
       let oldColumn = oldTable.schema[propKey]
-      if (oldColumn && oldColumn.type === "internal") {
-        oldColumn.type = "auto"
-      }
       if (oldColumn && oldColumn.type !== column.type) {
         ctx.throw(400, "Cannot change the type of a column")
       }

package/src/api/controllers/table/utils.js

@@ -1,4 +1,5 @@
 const CouchDB = require("../../../db")
+const linkRows = require("../../../db/linkedRows")
 const csvParser = require("../../../utilities/csvParser")
 const {
   getRowParams,
@@ -9,12 +10,6 @@ const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
 const { inputProcessing } = require("../../../utilities/rowProcessor")
 const { USERS_TABLE_SCHEMA } = require("../../../constants")
-const {
-  isExternalTable,
-  breakExternalTableId,
-} = require("../../../integrations/utils")
-const { getViews, saveView } = require("../view/utils")
-const viewTemplate = require("../view/viewBuilder")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -27,7 +22,6 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   }
   // check for renaming of columns or deleted columns
   if (rename || deletedColumns.length !== 0) {
-    // Update all rows
     const rows = await db.allDocs(
       getRowParams(updatedTable._id, null, {
         include_docs: true,
@@ -42,9 +36,6 @@ exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
       }
       return doc
     })
-
-    // Update views
-    await exports.checkForViewUpdates(db, updatedTable, rename, deletedColumns)
     delete updatedTable._rename
   }
   return { rows: updatedRows, table: updatedTable }
@@ -102,10 +93,19 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
         }
       }
 
-      finalData.push(row)
+      // make sure link rows are up to date
+      finalData.push(
+        linkRows.updateLinks({
+          appId,
+          eventType: linkRows.EventType.ROW_SAVE,
+          row,
+          tableId: row.tableId,
+          table,
+        })
+      )
     }
 
-    await db.bulkDocs(finalData)
+    await db.bulkDocs(await Promise.all(finalData))
     let response = await db.put(table)
     table._rev = response._rev
   }
@@ -233,86 +233,4 @@ exports.getExternalTable = async (appId, datasourceId, tableName) => {
   return entities[tableName]
 }
 
-exports.getTable = async (appId, tableId) => {
-  const db = new CouchDB(appId)
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    return exports.getExternalTable(appId, datasourceId, tableName)
-  } else {
-    return db.get(tableId)
-  }
-}
-
-exports.checkForViewUpdates = async (db, table, rename, deletedColumns) => {
-  const views = await getViews(db)
-  const tableViews = views.filter(view => view.meta.tableId === table._id)
-
-  // Check each table view to see if impacted by this table action
-  for (let view of tableViews) {
-    let needsUpdated = false
-
-    // First check for renames, otherwise check for deletions
-    if (rename) {
-      // Update calculation field if required
-      if (view.meta.field === rename.old) {
-        view.meta.field = rename.updated
-        needsUpdated = true
-      }
-
-      // Update group by field if required
-      if (view.meta.groupBy === rename.old) {
-        view.meta.groupBy = rename.updated
-        needsUpdated = true
-      }
-
-      // Update filters if required
-      if (view.meta.filters) {
-        view.meta.filters.forEach(filter => {
-          if (filter.key === rename.old) {
-            filter.key = rename.updated
-            needsUpdated = true
-          }
-        })
-      }
-    } else if (deletedColumns) {
-      deletedColumns.forEach(column => {
-        // Remove calculation statement if required
-        if (view.meta.field === column) {
-          delete view.meta.field
-          delete view.meta.calculation
-          delete view.meta.groupBy
-          needsUpdated = true
-        }
-
-        // Remove group by field if required
-        if (view.meta.groupBy === column) {
-          delete view.meta.groupBy
-          needsUpdated = true
-        }
-
-        // Remove filters referencing deleted field if required
-        if (view.meta.filters && view.meta.filters.length) {
-          const initialLength = view.meta.filters.length
-          view.meta.filters = view.meta.filters.filter(filter => {
-            return filter.key !== column
-          })
-          if (initialLength !== view.meta.filters.length) {
-            needsUpdated = true
-          }
-        }
-      })
-    }
-
-    // Update view if required
-    if (needsUpdated) {
-      const newViewTemplate = viewTemplate(view.meta)
-      await saveView(db, null, view.name, newViewTemplate)
-      if (!newViewTemplate.meta.schema) {
-        newViewTemplate.meta.schema = table.schema
-      }
-      table.views[view.name] = newViewTemplate.meta
-    }
-  }
-}
-
 exports.TableSaveFunctions = TableSaveFunctions

package/src/api/controllers/templates.js

@@ -7,23 +7,11 @@ const DEFAULT_TEMPLATES_BUCKET =
 
 exports.fetch = async function (ctx) {
   const { type = "app" } = ctx.query
-  let response,
-    error = false
-  try {
-    response = await fetch(`https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`)
-    if (response.status !== 200) {
-      error = true
-    }
-  } catch (err) {
-    error = true
-  }
-  // if there is an error, simply return no templates
-  if (!error && response) {
-    const json = await response.json()
-    ctx.body = Object.values(json.templates[type])
-  } else {
-    ctx.body = []
-  }
+  const response = await fetch(
+    `https://${DEFAULT_TEMPLATES_BUCKET}/manifest.json`
+  )
+  const json = await response.json()
+  ctx.body = Object.values(json.templates[type])
 }
 
 // can't currently test this, have to ignore from coverage