@budibase/server 0.9.185-alpha.0 → 0.9.185-alpha.12

package/dist/utilities/csvParser.js

@@ -56,7 +56,7 @@ function parse(csvString, parsers) {
         });
         result.subscribe(row => {
             // For each CSV row parse all the columns that need parsed
-            for (let key in parsers) {
+            for (let key of Object.keys(parsers)) {
                 if (!schema[key] || schema[key].success) {
                     // get the validator for the column type
                     const validator = VALIDATORS[parsers[key].type];
@@ -79,15 +79,55 @@ function parse(csvString, parsers) {
         });
     });
 }
-function transform({ schema, csvString }) {
+function updateSchema({ schema, existingTable }) {
+    if (!schema) {
+        return schema;
+    }
+    const finalSchema = {};
+    const schemaKeyMap = {};
+    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+    for (let [key, field] of Object.entries(existingTable.schema)) {
+        const lcKey = key.toLowerCase();
+        const foundKey = schemaKeyMap[lcKey];
+        if (foundKey) {
+            finalSchema[key] = schema[foundKey];
+            finalSchema[key].type = field.type;
+        }
+    }
+    return finalSchema;
+}
+function transform({ schema, csvString, existingTable }) {
     return __awaiter(this, void 0, void 0, function* () {
         const colParser = {};
-        for (let key in schema) {
+        // make sure the table has all the columns required for import
+        if (existingTable) {
+            schema = updateSchema({ schema, existingTable });
+        }
+        for (let key of Object.keys(schema)) {
             colParser[key] = PARSERS[schema[key].type] || schema[key].type;
         }
         try {
-            const json = yield csv({ colParser }).fromString(csvString);
-            return json;
+            const data = yield csv({ colParser }).fromString(csvString);
+            const schemaKeyMap = {};
+            Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key));
+            for (let element of data) {
+                if (!data) {
+                    continue;
+                }
+                for (let key of Object.keys(element)) {
+                    const mappedKey = schemaKeyMap[key.toLowerCase()];
+                    // isn't a column in the table, remove it
+                    if (mappedKey == null) {
+                        delete element[key];
+                    }
+                    // casing is different, fix it in row
+                    else if (key !== mappedKey) {
+                        element[mappedKey] = element[key];
+                        delete element[key];
+                    }
+                }
+            }
+            return data;
         }
         catch (err) {
             console.error(`Error transforming CSV to JSON for data import`, err);
@@ -98,4 +138,5 @@ function transform({ schema, csvString }) {
 module.exports = {
     parse,
     transform,
+    updateSchema,
 };

package/dist/utilities/index.js

@@ -52,6 +52,7 @@ exports.objectStoreUrl = () => {
  * via a specific endpoint (under /api/assets/client).
  * @param {string} appId In production we need the appId to look up the correct bucket, as the
  * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
  * @return {string} The URL to be inserted into appPackage response or server rendered
  * app index file.
  */

package/dist/utilities/queue/inMemoryQueue.js

@@ -93,5 +93,13 @@ class InMemoryQueue {
     getRepeatableJobs() {
         return [];
     }
+    /**
+     * Implemented for tests
+     */
+    clean() {
+        return __awaiter(this, void 0, void 0, function* () {
+            return [];
+        });
+    }
 }
 module.exports = InMemoryQueue;

package/dist/utilities/scriptRunner.js

@@ -1,10 +1,13 @@
 "use strict";
 const fetch = require("node-fetch");
 const { VM, VMScript } = require("vm2");
+const JS_TIMEOUT_MS = 1000;
 class ScriptRunner {
     constructor(script, context) {
         const code = `let fn = () => {\n${script}\n}; results.out = fn();`;
-        this.vm = new VM();
+        this.vm = new VM({
+            timeout: JS_TIMEOUT_MS,
+        });
         this.results = { out: "" };
         this.vm.setGlobals(context);
         this.vm.setGlobal("fetch", fetch);

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "0.9.185-alpha.0",
+  "version": "0.9.185-alpha.12",
   "description": "Budibase Web Server",
   "main": "src/index.js",
   "repository": {
@@ -68,9 +68,11 @@
   "author": "Budibase",
   "license": "AGPL-3.0-or-later",
   "dependencies": {
-    "@budibase/auth": "^0.9.185-alpha.0",
-    "@budibase/client": "^0.9.185-alpha.0",
-    "@budibase/string-templates": "^0.9.185-alpha.0",
+    "@budibase/auth": "^0.9.185-alpha.12",
+    "@budibase/client": "^0.9.185-alpha.12",
+    "@budibase/string-templates": "^0.9.185-alpha.12",
+    "@bull-board/api": "^3.7.0",
+    "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
     "@koa/router": "8.0.0",
     "@sendgrid/mail": "7.1.1",
@@ -80,7 +82,6 @@
     "aws-sdk": "^2.767.0",
     "bcryptjs": "2.4.3",
     "bull": "^3.22.4",
-    "bull-board": "^2.0.1",
     "chmodr": "1.2.0",
     "csvtojson": "2.0.10",
     "dotenv": "8.2.0",
@@ -120,6 +121,7 @@
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
     "vm2": "^3.9.3",
+    "worker-farm": "^1.7.0",
     "yargs": "13.2.4",
     "zlib": "1.0.5"
   },
@@ -138,7 +140,6 @@
     "copyfiles": "^2.4.1",
     "docker-compose": "^0.23.6",
     "eslint": "^6.8.0",
-    "express": "^4.17.1",
     "jest": "^27.0.5",
     "nodemon": "^2.0.4",
     "prettier": "^2.3.1",
@@ -149,5 +150,5 @@
     "typescript": "^4.3.5",
     "update-dotenv": "^1.1.1"
   },
-  "gitHead": "04cfca8b8ede8802307809b10e1215a5d87f22ad"
+  "gitHead": "c48ead6f4930f1dcaa2e6efc974bb38406e31e6d"
 }

package/scripts/likeCypress.ts

@@ -0,0 +1,33 @@
+/******************************************************
+ * This script just makes it easy to re-create        *
+ * a cypress like environment for testing the backend *
+ ******************************************************/
+const path = require("path")
+const tmpdir = path.join(require("os").tmpdir(), ".budibase")
+
+const MAIN_PORT = "10001"
+const WORKER_PORT = "10002"
+
+// @ts-ignore
+process.env.PORT = MAIN_PORT
+process.env.BUDIBASE_API_KEY = "6BE826CB-6B30-4AEC-8777-2E90464633DE"
+process.env.NODE_ENV = "cypress"
+process.env.ENABLE_ANALYTICS = "false"
+process.env.JWT_SECRET = "budibase"
+process.env.COUCH_URL = `leveldb://${tmpdir}/.data/`
+process.env.SELF_HOSTED = "1"
+process.env.WORKER_URL = `http://localhost:${WORKER_PORT}/`
+process.env.MINIO_URL = `http://localhost:${MAIN_PORT}/`
+process.env.MINIO_ACCESS_KEY = "budibase"
+process.env.MINIO_SECRET_KEY = "budibase"
+process.env.COUCH_DB_USER = "budibase"
+process.env.COUCH_DB_PASSWORD = "budibase"
+process.env.INTERNAL_API_KEY = "budibase"
+process.env.ALLOW_DEV_AUTOMATIONS = "1"
+
+// don't make this a variable or top level require
+// it will cause environment module to be loaded prematurely
+const server = require("../src/app")
+process.env.PORT = WORKER_PORT
+const worker = require("../../worker/src/index")
+process.env.PORT = MAIN_PORT

package/src/api/controllers/application.js

@@ -45,6 +45,8 @@ const {
 } = require("../../utilities/fileSystem/clientLibrary")
 const { getTenantId, isMultiTenant } = require("@budibase/auth/tenancy")
 const { syncGlobalUsers } = require("./user")
+const { app: appCache } = require("@budibase/auth/cache")
+const { cleanupAutomations } = require("../../automations/utils")
 
 const URL_REGEX_SLASH = /\/|\\/g
 
@@ -254,6 +256,7 @@ exports.create = async ctx => {
     await createApp(appId)
   }
 
+  await appCache.invalidateAppMetadata(appId, newApplication)
   ctx.status = 200
   ctx.body = newApplication
 }
@@ -317,8 +320,12 @@ exports.delete = async ctx => {
   if (!env.isTest() && !ctx.query.unpublish) {
     await deleteApp(ctx.params.appId)
   }
+  if (ctx.query && ctx.query.unpublish) {
+    await cleanupAutomations(ctx.params.appId)
+  }
   // make sure the app/role doesn't stick around after the app has been deleted
   await removeAppFromUserRoles(ctx, ctx.params.appId)
+  await appCache.invalidateAppMetadata(ctx.params.appId)
 
   ctx.status = 200
   ctx.body = result
@@ -387,7 +394,10 @@ const updateAppPackage = async (ctx, appPackage, appId) => {
   // Redis, shouldn't ever store it
   delete newAppPackage.lockedBy
 
-  return await db.put(newAppPackage)
+  const response = await db.put(newAppPackage)
+  // remove any cached metadata, so that it will be updated
+  await appCache.invalidateAppMetadata(appId)
+  return response
 }
 
 const createEmptyAppPackage = async (ctx, app) => {

package/src/api/controllers/datasource.js

@@ -119,8 +119,16 @@ exports.destroy = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
   // Delete all queries for the datasource
-  const rows = await db.allDocs(getQueryParams(ctx.params.datasourceId, null))
-  await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  const queries = await db.allDocs(
+    getQueryParams(ctx.params.datasourceId, null)
+  )
+  await db.bulkDocs(
+    queries.rows.map(row => ({
+      _id: row.id,
+      _rev: row.value.rev,
+      _deleted: true,
+    }))
+  )
 
   // delete the datasource
   await db.remove(ctx.params.datasourceId, ctx.params.revId)

package/src/api/controllers/deploy/index.js

@@ -6,6 +6,7 @@ const {
   disableAllCrons,
   enableCronTrigger,
 } = require("../../../automations/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 // the max time we can wait for an invalidation to complete before considering it failed
 const MAX_PENDING_TIME_MS = 30 * 60000
@@ -103,6 +104,7 @@ async function deployApp(deployment) {
     appDoc.appId = productionAppId
     appDoc.instance._id = productionAppId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(productionAppId)
     console.log("New app doc written successfully.")
     await initDeployedApp(productionAppId)
     console.log("Deployed app initialised, setting deployment to successful")

package/src/api/controllers/dev.js

@@ -6,6 +6,7 @@ const { request } = require("../../utilities/workerRequests")
 const { clearLock } = require("../../utilities/redis")
 const { Replication } = require("@budibase/auth").db
 const { DocumentTypes } = require("../../db/utils")
+const { app: appCache } = require("@budibase/auth/cache")
 
 async function redirect(ctx, method, path = "global") {
   const { devPath } = ctx.params
@@ -24,7 +25,8 @@ async function redirect(ctx, method, path = "global") {
     )
   )
   if (response.status !== 200) {
-    ctx.throw(response.status, response.statusText)
+    const err = await response.text()
+    ctx.throw(400, err)
   }
   const cookie = response.headers.get("set-cookie")
   if (cookie) {
@@ -106,6 +108,7 @@ exports.revert = async ctx => {
     appDoc.appId = appId
     appDoc.instance._id = appId
     await db.put(appDoc)
+    await appCache.invalidateAppMetadata(appId)
     ctx.body = {
       message: "Reverted changes successfully.",
     }

package/src/api/controllers/query.js

@@ -1,10 +1,11 @@
 const { processString } = require("@budibase/string-templates")
 const CouchDB = require("../../db")
 const { generateQueryID, getQueryParams } = require("../../db/utils")
-const { integrations } = require("../../integrations")
 const { BaseQueryVerbs } = require("../../constants")
 const env = require("../../environment")
-const ScriptRunner = require("../../utilities/scriptRunner")
+const { Thread, ThreadType } = require("../../threads")
+
+const Runner = new Thread(ThreadType.QUERY, { timeoutMs: 10000 })
 
 // simple function to append "readable" to all read queries
 function enrichQueries(input) {
@@ -18,47 +19,6 @@ function enrichQueries(input) {
   return wasArray ? queries : queries[0]
 }
 
-function formatResponse(resp) {
-  if (typeof resp === "string") {
-    try {
-      resp = JSON.parse(resp)
-    } catch (err) {
-      resp = { response: resp }
-    }
-  }
-  return resp
-}
-
-async function runAndTransform(
-  integration,
-  queryVerb,
-  enrichedQuery,
-  transformer
-) {
-  let rows = formatResponse(await integration[queryVerb](enrichedQuery))
-
-  // transform as required
-  if (transformer) {
-    const runner = new ScriptRunner(transformer, { data: rows })
-    rows = runner.execute()
-  }
-
-  // needs to an array for next step
-  if (!Array.isArray(rows)) {
-    rows = [rows]
-  }
-
-  // map into JSON if just raw primitive here
-  if (rows.find(row => typeof row !== "object")) {
-    rows = rows.map(value => ({ value }))
-  }
-
-  // get all the potential fields in the schema
-  let keys = rows.flatMap(Object.keys)
-
-  return { rows, keys }
-}
-
 exports.fetch = async function (ctx) {
   const db = new CouchDB(ctx.appId)
 
@@ -143,30 +103,23 @@ exports.preview = async function (ctx) {
 
   const datasource = await db.get(ctx.request.body.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const { fields, parameters, queryVerb, transformer } = ctx.request.body
   const enrichedQuery = await enrichQueryFields(fields, parameters)
-  const integration = new Integration(datasource.config)
 
-  const { rows, keys } = await runAndTransform(
-    integration,
-    queryVerb,
-    enrichedQuery,
-    transformer
-  )
+  try {
+    const { rows, keys } = await Runner.run({
+      datasource,
+      queryVerb,
+      query: enrichedQuery,
+      transformer,
+    })
 
-  ctx.body = {
-    rows,
-    schemaFields: [...new Set(keys)],
-  }
-  // cleanup
-  if (integration.end) {
-    integration.end()
+    ctx.body = {
+      rows,
+      schemaFields: [...new Set(keys)],
+    }
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 
@@ -176,29 +129,22 @@ exports.execute = async function (ctx) {
   const query = await db.get(ctx.params.queryId)
   const datasource = await db.get(query.datasourceId)
 
-  const Integration = integrations[datasource.source]
-
-  if (!Integration) {
-    ctx.throw(400, "Integration type does not exist.")
-  }
-
   const enrichedQuery = await enrichQueryFields(
     query.fields,
     ctx.request.body.parameters
   )
-  const integration = new Integration(datasource.config)
 
   // call the relevant CRUD method on the integration class
-  const { rows } = await runAndTransform(
-    integration,
-    query.queryVerb,
-    enrichedQuery,
-    query.transformer
-  )
-  ctx.body = rows
-  // cleanup
-  if (integration.end) {
-    integration.end()
+  try {
+    const { rows } = await Runner.run({
+      datasource,
+      queryVerb: query.queryVerb,
+      query: enrichedQuery,
+      transformer: query.transformer,
+    })
+    ctx.body = rows
+  } catch (err) {
+    ctx.throw(400, err)
   }
 }
 

package/src/api/controllers/row/ExternalRequest.ts

@@ -33,6 +33,7 @@ interface RunConfig {
   sort?: SortJson
   paginate?: PaginationJson
   row?: Row
+  rows?: Row[]
 }
 
 module External {
@@ -600,7 +601,10 @@ module External {
         throw `Unable to process query, table "${tableName}" not defined.`
       }
       // look for specific components of config which may not be considered acceptable
-      let { id, row, filters, sort, paginate } = cleanupConfig(config, table)
+      let { id, row, filters, sort, paginate, rows } = cleanupConfig(
+        config,
+        table
+      )
       filters = buildFilters(id, filters || {}, table)
       const relationships = this.buildRelationships(table)
       // clean up row on ingress using schema
@@ -626,7 +630,7 @@ module External {
         sort,
         paginate,
         relationships,
-        body: row,
+        body: row || rows,
         // pass an id filter into extra, purely for mysql/returning
         extra: {
           idFilter: buildFilters(id || generateIdForRow(row, table), {}, table),

package/src/api/controllers/row/external.js

@@ -30,6 +30,8 @@ async function handleRequest(appId, operation, tableId, opts = {}) {
   )
 }
 
+exports.handleRequest = handleRequest
+
 exports.patch = async ctx => {
   const appId = ctx.appId
   const inputs = ctx.request.body

package/src/api/controllers/table/external.js

@@ -17,6 +17,8 @@ const {
 } = require("../../../constants")
 const { makeExternalQuery } = require("../../../integrations/base/utils")
 const { cloneDeep } = require("lodash/fp")
+const csvParser = require("../../../utilities/csvParser")
+const { handleRequest } = require("../row/external")
 
 async function makeTableRequest(
   datasource,
@@ -279,3 +281,20 @@ exports.destroy = async function (ctx) {
 
   return tableToDelete
 }
+
+exports.bulkImport = async function (ctx) {
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
+  const { dataImport } = ctx.request.body
+  if (!dataImport || !dataImport.schema || !dataImport.csvString) {
+    ctx.throw(400, "Provided data import information is invalid.")
+  }
+  const rows = await csvParser.transform({
+    ...dataImport,
+    existingTable: table,
+  })
+  await handleRequest(appId, DataSourceOperation.BULK_CREATE, table._id, {
+    rows,
+  })
+  return table
+}

package/src/api/controllers/table/index.js

@@ -81,8 +81,26 @@ exports.destroy = async function (ctx) {
   ctx.body = { message: `Table ${tableId} deleted.` }
 }
 
+exports.bulkImport = async function (ctx) {
+  const tableId = ctx.params.tableId
+  await pickApi({ tableId }).bulkImport(ctx)
+  // right now we don't trigger anything for bulk import because it
+  // can only be done in the builder, but in the future we may need to
+  // think about events for bulk items
+  ctx.status = 200
+  ctx.body = { message: `Bulk rows created.` }
+}
+
 exports.validateCSVSchema = async function (ctx) {
-  const { csvString, schema = {} } = ctx.request.body
-  const result = await csvParser.parse(csvString, schema)
+  // tableId being specified means its an import to an existing table
+  const { csvString, schema = {}, tableId } = ctx.request.body
+  let existingTable
+  if (tableId) {
+    existingTable = await getTable(ctx.appId, tableId)
+  }
+  let result = await csvParser.parse(csvString, schema)
+  if (existingTable) {
+    result = csvParser.updateSchema({ schema: result, existingTable })
+  }
   ctx.body = { schema: result }
 }

package/src/api/controllers/table/internal.js

@@ -2,7 +2,12 @@ const CouchDB = require("../../../db")
 const linkRows = require("../../../db/linkedRows")
 const { getRowParams, generateTableID } = require("../../../db/utils")
 const { FieldTypes } = require("../../../constants")
-const { TableSaveFunctions, hasTypeChanged } = require("./utils")
+const {
+  TableSaveFunctions,
+  hasTypeChanged,
+  getTable,
+  handleDataImport,
+} = require("./utils")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -140,3 +145,11 @@ exports.destroy = async function (ctx) {
 
   return tableToDelete
 }
+
+exports.bulkImport = async function (ctx) {
+  const appId = ctx.appId
+  const table = await getTable(appId, ctx.params.tableId)
+  const { dataImport } = ctx.request.body
+  await handleDataImport(appId, ctx.user, table, dataImport)
+  return table
+}

package/src/api/controllers/table/utils.js

@@ -72,43 +72,47 @@ exports.makeSureTableUpToDate = (table, tableToSave) => {
 }
 
 exports.handleDataImport = async (appId, user, table, dataImport) => {
+  if (!dataImport || !dataImport.csvString) {
+    return table
+  }
   const db = new CouchDB(appId)
-  if (dataImport && dataImport.csvString) {
-    // Populate the table with rows imported from CSV in a bulk update
-    const data = await csvParser.transform(dataImport)
-
-    let finalData = []
-    for (let i = 0; i < data.length; i++) {
-      let row = data[i]
-      row._id = generateRowID(table._id)
-      row.tableId = table._id
-      const processed = inputProcessing(user, table, row, {
-        noAutoRelationships: true,
-      })
-      table = processed.table
-      row = processed.row
-
-      for (let [fieldName, schema] of Object.entries(table.schema)) {
-        // check whether the options need to be updated for inclusion as part of the data import
-        if (
-          schema.type === FieldTypes.OPTIONS &&
-          (!schema.constraints.inclusion ||
-            schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
-        ) {
-          schema.constraints.inclusion = [
-            ...schema.constraints.inclusion,
-            row[fieldName],
-          ]
-        }
+  // Populate the table with rows imported from CSV in a bulk update
+  const data = await csvParser.transform({
+    ...dataImport,
+    existingTable: table,
+  })
+
+  let finalData = []
+  for (let i = 0; i < data.length; i++) {
+    let row = data[i]
+    row._id = generateRowID(table._id)
+    row.tableId = table._id
+    const processed = inputProcessing(user, table, row, {
+      noAutoRelationships: true,
+    })
+    table = processed.table
+    row = processed.row
+
+    for (let [fieldName, schema] of Object.entries(table.schema)) {
+      // check whether the options need to be updated for inclusion as part of the data import
+      if (
+        schema.type === FieldTypes.OPTIONS &&
+        (!schema.constraints.inclusion ||
+          schema.constraints.inclusion.indexOf(row[fieldName]) === -1)
+      ) {
+        schema.constraints.inclusion = [
+          ...schema.constraints.inclusion,
+          row[fieldName],
+        ]
       }
-
-      finalData.push(row)
     }
 
-    await db.bulkDocs(finalData)
-    let response = await db.put(table)
-    table._rev = response._rev
+    finalData.push(row)
   }
+
+  await db.bulkDocs(finalData)
+  let response = await db.put(table)
+  table._rev = response._rev
   return table
 }