@budibase/server 0.9.172 → 0.9.173-alpha.3

package/dist/utilities/fileSystem/index.js

@@ -98,53 +98,69 @@ exports.apiFileReturn = contents => {
     fs.writeFileSync(path, contents);
     return fs.createReadStream(path);
 };
+/**
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to backup
+ * @param {object} config Config to send to export DB
+ * @returns {*} either a string or a stream of the backup
+ */
+const backupAppData = (appId, config) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield exports.exportDB(appId, Object.assign(Object.assign({}, config), { filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
+            doc._id.includes(LINK_USER_METADATA_PREFIX)) }));
+});
 /**
  * Takes a copy of the database state for an app to the object store.
  * @param {string} appId The ID of the app which is to be backed up.
  * @param {string} backupName The name of the backup located in the object store.
- * @return The backup has been completed when this promise completes and returns a file stream
- * to the temporary backup file (to return via API if required).
+ * @return {*} a readable stream to the completed backup file
  */
 exports.performBackup = (appId, backupName) => __awaiter(void 0, void 0, void 0, function* () {
-    return exports.exportDB(appId, {
-        exportName: backupName,
-        filter: doc => !(doc._id.includes(USER_METDATA_PREFIX) ||
-            doc.includes(LINK_USER_METADATA_PREFIX)),
-    });
+    return yield backupAppData(appId, { exportName: backupName });
 });
 /**
- * exports a DB to either file or a variable (memory).
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+exports.streamBackup = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    return yield backupAppData(appId, { stream: true });
+});
+/**
+ * Exports a DB to either file or a variable (memory).
  * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - the file name to export to, if not in memory.
+ * @param {string} exportName optional - provide a filename to write the backup to a file
+ * @param {boolean} stream optional - whether to perform a full backup
  * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return Either the file stream or the variable (if no export name provided).
+ * @return {*} either a readable stream or a string
  */
-exports.exportDB = (dbName, { exportName, filter } = { exportName: undefined, filter: undefined }) => __awaiter(void 0, void 0, void 0, function* () {
-    let stream, appString = "", path = null;
-    if (exportName) {
-        path = join(budibaseTempDir(), exportName);
-        stream = fs.createWriteStream(path);
-    }
-    else {
-        stream = new MemoryStream();
-        stream.on("data", chunk => {
-            appString += chunk.toString();
-        });
-    }
-    // perform couch dump
+exports.exportDB = (dbName, { stream, filter, exportName } = {}) => __awaiter(void 0, void 0, void 0, function* () {
     const instanceDb = new CouchDB(dbName);
-    yield instanceDb.dump(stream, {
-        filter,
-    });
-    // just in memory, return the final string
-    if (!exportName) {
-        return appString;
+    // Stream the dump if required
+    if (stream) {
+        const memStream = new MemoryStream();
+        instanceDb.dump(memStream, { filter });
+        return memStream;
     }
-    // write the file to the object store
-    if (env.SELF_HOSTED) {
-        yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+    // Write the dump to file if required
+    if (exportName) {
+        const path = join(budibaseTempDir(), exportName);
+        const writeStream = fs.createWriteStream(path);
+        yield instanceDb.dump(writeStream, { filter });
+        // Upload the dump to the object store if self hosted
+        if (env.SELF_HOSTED) {
+            yield streamUpload(ObjectStoreBuckets.BACKUPS, join(dbName, exportName), fs.createReadStream(path));
+        }
+        return fs.createReadStream(path);
     }
-    return fs.createReadStream(path);
+    // Stringify the dump in memory if required
+    const memStream = new MemoryStream();
+    let appString = "";
+    memStream.on("data", chunk => {
+        appString += chunk.toString();
+    });
+    yield instanceDb.dump(memStream, { filter });
+    return appString;
 });
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.

package/dist/utilities/global.js

@@ -33,7 +33,7 @@ exports.updateAppRole = (appId, user) => {
         user.roleId = BUILTIN_ROLE_IDS.ADMIN;
     }
     else if (!user.roleId) {
-        user.roleId = BUILTIN_ROLE_IDS.BASIC;
+        user.roleId = BUILTIN_ROLE_IDS.PUBLIC;
     }
     delete user.roles;
     return user;

package/dist/utilities/queue/inMemoryQueue.js

@@ -87,5 +87,11 @@ class InMemoryQueue {
         // TODO: implement for testing
         console.log(cronJobId);
     }
+    /**
+     * Implemented for tests
+     */
+    getRepeatableJobs() {
+        return [];
+    }
 }
 module.exports = InMemoryQueue;

package/dist/utilities/rowProcessor/index.js

@@ -149,6 +149,10 @@ exports.processAutoColumn = processAutoColumn;
  * @returns {object} The coerced value
  */
 exports.coerce = (row, type) => {
+    // no coercion specified for type, skip it
+    if (!TYPE_TRANSFORM_MAP[type]) {
+        return row;
+    }
     // eslint-disable-next-line no-prototype-builtins
     if (TYPE_TRANSFORM_MAP[type].hasOwnProperty(row)) {
         return TYPE_TRANSFORM_MAP[type][row];
@@ -190,6 +194,10 @@ exports.inputProcessing = (user = {}, table, row, opts = { noAutoRelationships:
             clonedRow[key] = exports.coerce(value, field.type);
         }
     }
+    if (!clonedRow._id || !clonedRow._rev) {
+        clonedRow._id = row._id;
+        clonedRow._rev = row._rev;
+    }
     // handle auto columns - this returns an object like {table, row}
     return processAutoColumn(user, copiedTable, clonedRow, opts);
 };

package/dist/utilities/scriptRunner.js

@@ -0,0 +1,19 @@
+"use strict";
+const fetch = require("node-fetch");
+const { VM, VMScript } = require("vm2");
+class ScriptRunner {
+    constructor(script, context) {
+        const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
+        this.vm = new VM();
+        this.results = { out: "" };
+        this.vm.setGlobals(context);
+        this.vm.setGlobal("fetch", fetch);
+        this.vm.setGlobal("results", this.results);
+        this.script = new VMScript(code);
+    }
+    execute() {
+        this.vm.run(this.script);
+        return this.results.out;
+    }
+}
+module.exports = ScriptRunner;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.172",
+  "version": "0.9.173-alpha.3",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,13 +68,13 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.172",
-    "@budibase/client": "^0.9.172",
-    "@budibase/string-templates": "^0.9.172",
+    "@budibase/auth": "^0.9.173-alpha.3",
+    "@budibase/client": "^0.9.173-alpha.3",
+    "@budibase/string-templates": "^0.9.173-alpha.3",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
-    "@sentry/node": "5.19.2",
+    "@sentry/node": "^6.0.0",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "^2.767.0",
@@ -103,7 +103,7 @@
     "memorystream": "^0.3.1",
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
-    "mysql": "2.18.1",
+    "mysql2": "^2.3.1",
     "node-fetch": "2.6.0",
     "open": "7.3.0",
     "pg": "8.5.1",
@@ -119,6 +119,7 @@
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
+    "vm2": "^3.9.3",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -148,5 +149,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "f33cc5a9cd747ffb8491af36ac7baf0bf66de290"
+  "gitHead": "7bbd460f86bae19117a7a7ed48dc984aa6eb7826"
 }

package/scripts/jestSetup.js

@@ -8,3 +8,5 @@ env._set("CLIENT_ID", "test-client-id")
 env._set("BUDIBASE_DIR", tmpdir("budibase-unittests"))
 env._set("LOG_LEVEL", "silent")
 env._set("PORT", 0)
+
+global.console.log = jest.fn() // console.log are ignored in tests

package/src/api/controllers/analytics.js

@@ -21,6 +21,10 @@ exports.endUserPing = async ctx => {
     return
   }
 
+  posthogClient.identify({
+    distinctId: ctx.user && ctx.user._id,
+    properties: {},
+  })
   posthogClient.capture({
     event: "budibase:end_user_ping",
     distinctId: ctx.user && ctx.user._id,

package/src/api/controllers/application.js

@@ -25,7 +25,12 @@ const { BASE_LAYOUTS } = require("../../constants/layouts")
 const { createHomeScreen } = require("../../constants/screens")
 const { cloneDeep } = require("lodash/fp")
 const { processObject } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/auth/db")
+const {
+  getAllApps,
+  isDevAppID,
+  getDeployedAppID,
+  Replication,
+} = require("@budibase/auth/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
 const {
   getDeployedApps,
@@ -134,7 +139,7 @@ async function createInstance(template) {
   return { _id: appId }
 }
 
-exports.fetch = async function (ctx) {
+exports.fetch = async ctx => {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = await getAllApps(CouchDB, { dev, all })
@@ -159,7 +164,7 @@ exports.fetch = async function (ctx) {
   ctx.body = apps
 }
 
-exports.fetchAppDefinition = async function (ctx) {
+exports.fetchAppDefinition = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const layouts = await getLayouts(db)
   const userRoleId = getUserRoleId(ctx)
@@ -175,7 +180,7 @@ exports.fetchAppDefinition = async function (ctx) {
   }
 }
 
-exports.fetchAppPackage = async function (ctx) {
+exports.fetchAppPackage = async ctx => {
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
   const layouts = await getLayouts(db)
@@ -196,7 +201,7 @@ exports.fetchAppPackage = async function (ctx) {
   }
 }
 
-exports.create = async function (ctx) {
+exports.create = async ctx => {
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -252,13 +257,13 @@ exports.create = async function (ctx) {
   ctx.body = newApplication
 }
 
-exports.update = async function (ctx) {
+exports.update = async ctx => {
   const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
 
-exports.updateClient = async function (ctx) {
+exports.updateClient = async ctx => {
   // Get current app version
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -280,7 +285,7 @@ exports.updateClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.revertClient = async function (ctx) {
+exports.revertClient = async ctx => {
   // Check app can be reverted
   const db = new CouchDB(ctx.params.appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
@@ -303,7 +308,7 @@ exports.revertClient = async function (ctx) {
   ctx.body = data
 }
 
-exports.delete = async function (ctx) {
+exports.delete = async ctx => {
   const db = new CouchDB(ctx.params.appId)
 
   const result = await db.destroy()
@@ -318,6 +323,35 @@ exports.delete = async function (ctx) {
   ctx.body = result
 }
 
+exports.sync = async ctx => {
+  const appId = ctx.params.appId
+  if (!isDevAppID(appId)) {
+    ctx.throw(400, "This action cannot be performed for production apps")
+  }
+  const prodAppId = getDeployedAppID(appId)
+  const replication = new Replication({
+    source: prodAppId,
+    target: appId,
+  })
+  let error
+  try {
+    await replication.replicate({
+      filter: function (doc) {
+        return doc._id !== DocumentTypes.APP_METADATA
+      },
+    })
+  } catch (err) {
+    error = err
+  }
+  if (error) {
+    ctx.throw(400, error)
+  } else {
+    ctx.body = {
+      message: "App sync completed successfully.",
+    }
+  }
+}
+
 const updateAppPackage = async (ctx, appPackage, appId) => {
   const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)

package/src/api/controllers/auth.js

@@ -28,14 +28,23 @@ exports.fetchSelf = async ctx => {
         ...metadata,
       })
     } catch (err) {
+      let response
       // user didn't exist in app, don't pretend they do
       if (user.roleId === BUILTIN_ROLE_IDS.PUBLIC) {
-        ctx.body = {}
+        response = {}
       }
       // user has a role of some sort, return them
-      else {
-        ctx.body = user
+      else if (err.status === 404) {
+        const metadata = {
+          _id: userId,
+        }
+        const dbResp = await db.put(metadata)
+        user._rev = dbResp.rev
+        response = user
+      } else {
+        response = user
       }
+      ctx.body = response
     }
   } else {
     ctx.body = user

package/src/api/controllers/backup.js

@@ -1,10 +1,9 @@
-const { performBackup } = require("../../utilities/fileSystem")
+const { streamBackup } = require("../../utilities/fileSystem")
 
 exports.exportAppDump = async function (ctx) {
   const { appId } = ctx.query
-  const appname = decodeURI(ctx.query.appname)
-  const backupIdentifier = `${appname}Backup${new Date().getTime()}.txt`
-
+  const appName = decodeURI(ctx.query.appname)
+  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
   ctx.attachment(backupIdentifier)
-  ctx.body = await performBackup(appId, backupIdentifier)
+  ctx.body = await streamBackup(appId)
 }

package/src/api/controllers/deploy/index.js

@@ -1,6 +1,6 @@
 const CouchDB = require("../../../db")
 const Deployment = require("./Deployment")
-const { Replication } = require("@budibase/auth/db")
+const { Replication, getDeployedAppID } = require("@budibase/auth/db")
 const { DocumentTypes, getAutomationParams } = require("../../../db/utils")
 const {
   disableAllCrons,
@@ -87,7 +87,7 @@ async function initDeployedApp(prodAppId) {
 
 async function deployApp(deployment) {
   try {
-    const productionAppId = deployment.appId.replace("_dev", "")
+    const productionAppId = getDeployedAppID(deployment.appId)
 
     const replication = new Replication({
       source: deployment.appId,
@@ -104,23 +104,8 @@ async function deployApp(deployment) {
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
     console.log("New app doc written successfully.")
-
-    console.log("Setting up live repl between dev and prod")
-    // Set up live sync between the live and dev instances
-    const liveReplication = new Replication({
-      source: productionAppId,
-      target: deployment.appId,
-    })
-    liveReplication.subscribe({
-      filter: function (doc) {
-        return doc._id !== DocumentTypes.APP_METADATA
-      },
-    })
-    console.log("Set up live repl between dev and prod")
-
-    console.log("Initialising deployed app")
     await initDeployedApp(productionAppId)
-    console.log("Init complete, setting deployment to successful")
+    console.log("Deployed app initialised, setting deployment to successful")
     deployment.setStatus(DeploymentStatus.SUCCESS)
     await storeDeploymentHistory(deployment)
   } catch (err) {

package/src/api/controllers/dev.js

@@ -7,11 +7,13 @@ const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
 
-async function redirect(ctx, method) {
+async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
   const queryString = ctx.originalUrl.split("?")[1] || ""
   const response = await fetch(
-    checkSlashesInUrl(`${env.WORKER_URL}/api/global/${devPath}?${queryString}`),
+    checkSlashesInUrl(
+      `${env.WORKER_URL}/api/${path}/${devPath}?${queryString}`
+    ),
     request(
       ctx,
       {
@@ -41,16 +43,22 @@ async function redirect(ctx, method) {
   ctx.cookies
 }
 
-exports.redirectGet = async ctx => {
-  await redirect(ctx, "GET")
+exports.buildRedirectGet = path => {
+  return async ctx => {
+    await redirect(ctx, "GET", path)
+  }
 }
 
-exports.redirectPost = async ctx => {
-  await redirect(ctx, "POST")
+exports.buildRedirectPost = path => {
+  return async ctx => {
+    await redirect(ctx, "POST", path)
+  }
 }
 
-exports.redirectDelete = async ctx => {
-  await redirect(ctx, "DELETE")
+exports.buildRedirectDelete = path => {
+  return async ctx => {
+    await redirect(ctx, "DELETE", path)
+  }
 }
 
 exports.clearLock = async ctx => {

package/src/api/controllers/permission.js

@@ -147,6 +147,7 @@ exports.getResourcePerms = async function (ctx) {
       const rolePerms = role.permissions
       if (
         rolePerms &&
+        rolePerms[resourceId] &&
         (rolePerms[resourceId] === level ||
           rolePerms[resourceId].indexOf(level) !== -1)
       ) {

package/src/api/controllers/query.js

@@ -4,6 +4,7 @@ const { generateQueryID, getQueryParams } = require("../../db/utils")
 const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
+const ScriptRunner = require("../../utilities/scriptRunner")
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -28,12 +29,39 @@ function formatResponse(resp) {
       resp = { response: resp }
     }
   }
-  if (!Array.isArray(resp)) {
-    resp = [resp]
-  }
   return resp
 }
 
+async function runAndTransform(
+  integration,
+  queryVerb,
+  enrichedQuery,
+  transformer
+) {
+  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
+
+  // transform as required
+  if (transformer) {
+    const runner = new ScriptRunner(transformer, { data: rows })
+    rows = runner.execute()
+  }
+
+  // needs to an array for next step
+  if (!Array.isArray(rows)) {
+    rows = [rows]
+  }
+
+  // map into JSON if just raw primitive here
+  if (rows.find(row => typeof row !== "object")) {
+    rows = rows.map(value => ({ value }))
+  }
+
+  // get all the potential fields in the schema
+  let keys = rows.flatMap(Object.keys)
+
+  return { rows, keys }
+}
+
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -122,15 +150,16 @@ exports.preview = async function (ctx) {
     ctx.throw(400, "Integration type does not exist.")
   }
 
-  const { fields, parameters, queryVerb } = ctx.request.body
-
+  const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-
   const integration = new Integration(datasource.config)
-  const rows = formatResponse(await integration[queryVerb](enrichedQuery))
 
-  // get all the potential fields in the schema
-  const keys = rows.flatMap(Object.keys)
+  const { rows, keys } = await runAndTransform(
+    integration,
+    queryVerb,
+    enrichedQuery,
+    transformer
+  )
 
   ctx.body = {
     rows,
@@ -158,10 +187,16 @@ exports.execute = async function (ctx) {
     query.fields,
     ctx.request.body.parameters
   )
-
   const integration = new Integration(datasource.config)
+
   // call the relevant CRUD method on the integration class
-  ctx.body = formatResponse(await integration[query.queryVerb](enrichedQuery))
+  const { rows } = await runAndTransform(
+    integration,
+    query.queryVerb,
+    enrichedQuery,
+    query.transformer
+  )
+  ctx.body = rows
   // cleanup
   if (integration.end) {
     integration.end()