@budibase/server 1.0.46-alpha.7 → 1.0.48

package/dist/utilities/{usageQuota/index.js → usageQuota.js}

@@ -8,24 +8,12 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const env = require("../../environment");
-const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
+const env = require("../environment");
+const { getGlobalDB } = require("@budibase/backend-core/tenancy");
 const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db");
-exports.useQuotas = () => {
-    // check if quotas are enabled
-    if (env.USE_QUOTAS) {
-        // check if there are any tenants without limits
-        if (env.EXCLUDE_QUOTAS_TENANTS) {
-            const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(/\s/g, "").split(",");
-            const tenantId = getTenantId();
-            if (excludedTenants.includes(tenantId)) {
-                return false;
-            }
-        }
-        return true;
-    }
-    return false;
-};
+function getNewQuotaReset() {
+    return Date.now() + 2592000000;
+}
 exports.Properties = {
     ROW: "rows",
     UPLOAD: "storage",
@@ -33,21 +21,21 @@ exports.Properties = {
     USER: "users",
     AUTOMATION: "automationRuns",
     APPS: "apps",
-    EMAILS: "emails", // doesn't work yet
+    EMAILS: "emails",
 };
-exports.getUsageQuotaDoc = (db) => __awaiter(void 0, void 0, void 0, function* () {
-    let quota;
-    try {
-        quota = yield db.get(StaticDatabases.GLOBAL.docs.usageQuota);
-    }
-    catch (err) {
-        // doc doesn't exist. Create it
-        quota = generateNewUsageQuotaDoc();
-        const response = yield db.put(quota);
-        quota._rev = response.rev;
-    }
-    return quota;
-});
+function getUsageQuotaDoc(db) {
+    return __awaiter(this, void 0, void 0, function* () {
+        let quota;
+        try {
+            quota = yield db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota);
+        }
+        catch (err) {
+            // doc doesn't exist. Create it
+            quota = yield db.post(generateNewUsageQuotaDoc());
+        }
+        return quota;
+    });
+}
 /**
  * Given a specified tenantId this will add to the usage object for the specified property.
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
@@ -55,28 +43,27 @@ exports.getUsageQuotaDoc = (db) => __awaiter(void 0, void 0, void 0, function* (
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = (property, usage, opts = { dryRun: false }) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!exports.useQuotas()) {
+exports.update = (property, usage) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!env.USE_QUOTAS) {
         return;
     }
     try {
         const db = getGlobalDB();
-        const quota = yield exports.getUsageQuotaDoc(db);
+        const quota = yield getUsageQuotaDoc(db);
+        // Check if the quota needs reset
+        if (Date.now() >= quota.quotaReset) {
+            quota.quotaReset = getNewQuotaReset();
+            for (let prop of Object.keys(quota.usageQuota)) {
+                quota.usageQuota[prop] = 0;
+            }
+        }
         // increment the quota
         quota.usageQuota[property] += usage;
-        if (quota.usageQuota[property] > quota.usageLimits[property] &&
-            usage > 0 // allow for decrementing usage when the quota is already exceeded
-        ) {
+        if (quota.usageQuota[property] > quota.usageLimits[property]) {
             throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`);
         }
-        if (quota.usageQuota[property] < 0) {
-            // never go negative if the quota has previously been exceeded
-            quota.usageQuota[property] = 0;
-        }
         // update the usage quotas
-        if (!opts.dryRun) {
-            yield db.put(quota);
-        }
+        yield db.put(quota);
     }
     catch (err) {
         console.error(`Error updating usage quotas for ${property}`, err);

package/dist/utilities/workerRequests.js

@@ -61,6 +61,26 @@ exports.sendSmtpEmail = (to, from, subject, contents, automation) => __awaiter(v
     }
     return response.json();
 });
+exports.getDeployedApps = () => __awaiter(void 0, void 0, void 0, function* () {
+    try {
+        const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + `/api/apps`), request(null, {
+            method: "GET",
+        }));
+        const json = yield response.json();
+        const apps = {};
+        for (let [key, value] of Object.entries(json)) {
+            if (value.url) {
+                value.url = value.url.toLowerCase();
+                apps[key.toLowerCase()] = value;
+            }
+        }
+        return apps;
+    }
+    catch (err) {
+        // error, cannot determine deployed apps, don't stop app creation - sort this later
+        return {};
+    }
+});
 exports.getGlobalSelf = (ctx, appId = null) => __awaiter(void 0, void 0, void 0, function* () {
     const endpoint = `/api/global/users/self`;
     const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + endpoint), 

package/nodemon.json

@@ -1,5 +1,5 @@
 {
-  "watch": ["src", "../backend-core"],
+  "watch": ["src", "../auth"],
   "ext": "js,ts,json",
   "ignore": ["src/**/*.spec.ts", "src/**/*.spec.js"],
   "exec": "ts-node src/index.ts"

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.7",
+  "version": "1.0.48",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.7",
-    "@budibase/client": "^1.0.46-alpha.7",
-    "@budibase/string-templates": "^1.0.46-alpha.7",
+    "@budibase/backend-core": "^1.0.48",
+    "@budibase/client": "^1.0.48",
+    "@budibase/string-templates": "^1.0.48",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -92,8 +92,6 @@
     "fix-path": "3.0.0",
     "form-data": "^4.0.0",
     "fs-extra": "8.1.0",
-    "google-auth-library": "^7.11.0",
-    "google-spreadsheet": "^3.2.0",
     "jimp": "0.16.1",
     "joi": "17.2.1",
     "js-yaml": "^4.1.0",
@@ -112,7 +110,7 @@
     "mongodb": "3.6.3",
     "mssql": "6.2.3",
     "mysql2": "^2.3.1",
-    "node-fetch": "2.6.7",
+    "node-fetch": "2.6.0",
     "open": "^8.4.0",
     "pg": "8.5.1",
     "pino-pretty": "4.0.0",
@@ -141,7 +139,6 @@
     "@jest/test-sequencer": "^24.8.0",
     "@types/apidoc": "^0.50.0",
     "@types/bull": "^3.15.1",
-    "@types/google-spreadsheet": "^3.1.5",
     "@types/jest": "^26.0.23",
     "@types/koa": "^2.13.3",
     "@types/koa-router": "^7.4.2",
@@ -168,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "5718d4925a31131daadf694e7f0ba837084a61f2"
+  "gitHead": "a5f51d83ba37e2361daf4ff579bb039a2247dd5e"
 }

package/src/api/controllers/application.js

@@ -33,7 +33,10 @@ const {
   Replication,
 } = require("@budibase/backend-core/db")
 const { USERS_TABLE_SCHEMA } = require("../../constants")
-const { removeAppFromUserRoles } = require("../../utilities/workerRequests")
+const {
+  getDeployedApps,
+  removeAppFromUserRoles,
+} = require("../../utilities/workerRequests")
 const { clientLibraryPath, stringToReadStream } = require("../../utilities")
 const { getAllLocks } = require("../../utilities/redis")
 const {
@@ -75,42 +78,29 @@ function getUserRoleId(ctx) {
     : ctx.user.role._id
 }
 
-async function getAppUrl(ctx) {
-  // construct the url
+async function getAppUrlIfNotInUse(ctx) {
   let url
   if (ctx.request.body.url) {
-    // if the url is provided, use that
     url = encodeURI(ctx.request.body.url)
   } else if (ctx.request.body.name) {
-    // otherwise use the name
     url = encodeURI(`${ctx.request.body.name}`)
   }
   if (url) {
     url = `/${url.replace(URL_REGEX_SLASH, "")}`.toLowerCase()
   }
-  return url
-}
-
-const checkAppUrl = (ctx, apps, url, currentAppId) => {
-  if (currentAppId) {
-    apps = apps.filter(app => app.appId !== currentAppId)
-  }
-  if (apps.some(app => app.url === url)) {
-    ctx.throw(400, "App URL is already in use.")
-  }
-}
-
-const checkAppName = (ctx, apps, name, currentAppId) => {
-  // TODO: Replace with Joi
-  if (!name) {
-    ctx.throw(400, "Name is required")
+  if (!env.SELF_HOSTED) {
+    return url
   }
-  if (currentAppId) {
-    apps = apps.filter(app => app.appId !== currentAppId)
-  }
-  if (apps.some(app => app.name === name)) {
-    ctx.throw(400, "App name is already in use.")
+  const deployedApps = await getDeployedApps()
+  if (
+    url &&
+    deployedApps[url] != null &&
+    ctx.params != null &&
+    deployedApps[url].appId !== ctx.params.appId
+  ) {
+    ctx.throw(400, "App name/URL is already in use.")
   }
+  return url
 }
 
 async function createInstance(template) {
@@ -216,12 +206,6 @@ exports.fetchAppPackage = async ctx => {
 }
 
 exports.create = async ctx => {
-  const apps = await getAllApps(CouchDB, { dev: true })
-  const name = ctx.request.body.name
-  checkAppName(ctx, apps, name)
-  const url = await getAppUrl(ctx)
-  checkAppUrl(ctx, apps, url)
-
   const { useTemplate, templateKey, templateString } = ctx.request.body
   const instanceConfig = {
     useTemplate,
@@ -234,6 +218,7 @@ exports.create = async ctx => {
   const instance = await createInstance(instanceConfig)
   const appId = instance._id
 
+  const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)
   let _rev
   try {
@@ -250,7 +235,7 @@ exports.create = async ctx => {
     type: "app",
     version: packageJson.version,
     componentLibraries: ["@budibase/standard-components"],
-    name: name,
+    name: ctx.request.body.name,
     url: url,
     template: ctx.request.body.template,
     instance: instance,
@@ -277,22 +262,8 @@ exports.create = async ctx => {
   ctx.body = newApplication
 }
 
-// This endpoint currently operates as a PATCH rather than a PUT
-// Thus name and url fields are handled only if present
 exports.update = async ctx => {
-  const apps = await getAllApps(CouchDB, { dev: true })
-  // validation
-  const name = ctx.request.body.name
-  if (name) {
-    checkAppName(ctx, apps, name, ctx.params.appId)
-  }
-  const url = await getAppUrl(ctx)
-  if (url) {
-    checkAppUrl(ctx, apps, url, ctx.params.appId)
-    ctx.request.body.url = url
-  }
-
-  const data = await updateAppPackage(ctx.request.body, ctx.params.appId)
+  const data = await updateAppPackage(ctx, ctx.request.body, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -314,7 +285,7 @@ exports.updateClient = async ctx => {
     version: packageJson.version,
     revertableVersion: currentVersion,
   }
-  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -337,7 +308,7 @@ exports.revertClient = async ctx => {
     version: application.revertableVersion,
     revertableVersion: null,
   }
-  const data = await updateAppPackage(appPackageUpdates, ctx.params.appId)
+  const data = await updateAppPackage(ctx, appPackageUpdates, ctx.params.appId)
   ctx.status = 200
   ctx.body = data
 }
@@ -410,11 +381,12 @@ exports.sync = async (ctx, next) => {
   }
 }
 
-const updateAppPackage = async (appPackage, appId) => {
+const updateAppPackage = async (ctx, appPackage, appId) => {
+  const url = await getAppUrlIfNotInUse(ctx)
   const db = new CouchDB(appId)
   const application = await db.get(DocumentTypes.APP_METADATA)
 
-  const newAppPackage = { ...application, ...appPackage }
+  const newAppPackage = { ...application, ...appPackage, url }
   if (appPackage._rev !== application._rev) {
     newAppPackage._rev = application._rev
   }

package/src/api/controllers/datasource.js

@@ -38,13 +38,6 @@ exports.fetch = async function (ctx) {
     )
   ).rows.map(row => row.doc)
 
-  for (let datasource of datasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-  }
-
   ctx.body = [bbInternalDb, ...datasources]
 }
 
@@ -101,13 +94,8 @@ exports.update = async function (ctx) {
   const db = new CouchDB(ctx.appId)
   const datasourceId = ctx.params.datasourceId
   let datasource = await db.get(datasourceId)
-  const auth = datasource.config.auth
   await invalidateVariables(datasource, ctx.request.body)
   datasource = { ...datasource, ...ctx.request.body }
-  if (auth && !ctx.request.body.auth) {
-    // don't strip auth config from DB
-    datasource.config.auth = auth
-  }
 
   const response = await db.put(datasource)
   datasource._rev = response.rev

package/src/api/controllers/hosting.js

@@ -0,0 +1,22 @@
+const CouchDB = require("../../db")
+const { getDeployedApps } = require("../../utilities/workerRequests")
+const { getScopedConfig } = require("@budibase/backend-core/db")
+const { Configs } = require("@budibase/backend-core/constants")
+const { checkSlashesInUrl } = require("../../utilities")
+
+exports.fetchUrls = async ctx => {
+  const appId = ctx.appId
+  const db = new CouchDB(appId)
+  const settings = await getScopedConfig(db, { type: Configs.SETTINGS })
+  let appUrl = "http://localhost:10000/app"
+  if (settings && settings["platformUrl"]) {
+    appUrl = checkSlashesInUrl(`${settings["platformUrl"]}/app`)
+  }
+  ctx.body = {
+    app: appUrl,
+  }
+}
+
+exports.getDeployedApps = async ctx => {
+  ctx.body = await getDeployedApps()
+}

package/src/api/controllers/row/ExternalRequest.ts

@@ -525,7 +525,7 @@ module External {
         const linkTable = this.getTable(tableId)
         // @ts-ignore
         const linkPrimary = linkTable.primary[0]
-        const rows = related[key].rows || []
+        const rows = related[key]?.rows || []
         const found = rows.find(
           (row: { [key: string]: any }) =>
             row[linkPrimary] === relationship.id ||

package/src/api/controllers/row/utils.js

@@ -58,16 +58,22 @@ exports.validate = async ({ appId, tableId, row, table }) => {
     let res
 
     // Validate.js doesn't seem to handle array
-    if (type === FieldTypes.ARRAY && row[fieldName]) {
-      if (row[fieldName].length) {
+    if (type === FieldTypes.ARRAY) {
+      const hasValues =
+        Array.isArray(row[fieldName]) && row[fieldName].length > 0
+
+      // Check values are valid if values are specified
+      if (hasValues) {
         row[fieldName].map(val => {
           if (!constraints.inclusion.includes(val)) {
-            errors[fieldName] = "Field not in list"
+            errors[fieldName] = "Value not in list"
           }
         })
-      } else if (constraints.presence && row[fieldName].length === 0) {
-        // non required MultiSelect creates an empty array, which should not throw errors
-        errors[fieldName] = [`${fieldName} is required`]
+      }
+
+      // Check for required constraint
+      if (constraints.presence === true && !hasValues) {
+        errors[fieldName] = "Required field"
       }
     } else if (type === FieldTypes.JSON && typeof row[fieldName] === "string") {
       // this should only happen if there is an error

package/src/api/controllers/static/index.js

@@ -5,7 +5,7 @@ const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
 const { ObjectStoreBuckets } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
-const { getAllApps } = require("@budibase/backend-core/db")
+const { getDeployedApps } = require("../../../utilities/workerRequests")
 const CouchDB = require("../../../db")
 const {
   loadHandlebarsFile,
@@ -17,8 +17,6 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
-const AWS = require("aws-sdk")
-const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -39,18 +37,12 @@ async function prepareUpload({ s3Key, bucket, metadata, file }) {
   }
 }
 
-async function getAppIdFromUrl(ctx) {
-  // the "appId" component of the URL can be the id or the custom url
+async function checkForSelfHostedURL(ctx) {
+  // the "appId" component of the URL may actually be a specific self hosted URL
   let possibleAppUrl = `/${encodeURI(ctx.params.appId).toLowerCase()}`
-
-  // search prod apps for a url that matches, exclude dev where id is always used
-  const apps = await getAllApps(CouchDB, { dev: false })
-  const app = apps.filter(
-    a => a.url && a.url.toLowerCase() === possibleAppUrl
-  )[0]
-
-  if (app && app.appId) {
-    return app.appId
+  const apps = await getDeployedApps()
+  if (apps[possibleAppUrl] && apps[possibleAppUrl].appId) {
+    return apps[possibleAppUrl].appId
   } else {
     return ctx.params.appId
   }
@@ -83,7 +75,10 @@ exports.uploadFile = async function (ctx) {
 }
 
 exports.serveApp = async function (ctx) {
-  let appId = await getAppIdFromUrl(ctx)
+  let appId = ctx.params.appId
+  if (env.SELF_HOSTED) {
+    appId = await checkForSelfHostedURL(ctx)
+  }
   const App = require("./templates/BudibaseApp.svelte").default
   const db = new CouchDB(appId, { skip_setup: true })
   const appInfo = await db.get(DocumentTypes.APP_METADATA)
@@ -109,51 +104,3 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
-
-exports.getSignedUploadURL = async function (ctx) {
-  const database = new CouchDB(ctx.appId)
-
-  // Ensure datasource is valid
-  let datasource
-  try {
-    const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
-    if (!datasource) {
-      ctx.throw(400, "The specified datasource could not be found")
-    }
-  } catch (error) {
-    ctx.throw(400, "The specified datasource could not be found")
-  }
-
-  // Ensure we aren't using a custom endpoint
-  if (datasource?.config?.endpoint) {
-    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
-  }
-
-  // Determine type of datasource and generate signed URL
-  let signedUrl
-  let publicUrl
-  if (datasource.source === "S3") {
-    const { bucket, key } = ctx.request.body || {}
-    if (!bucket || !key) {
-      ctx.throw(400, "bucket and key values are required")
-      return
-    }
-    try {
-      const s3 = new AWS.S3({
-        region: AWS_REGION,
-        accessKeyId: datasource?.config?.accessKeyId,
-        secretAccessKey: datasource?.config?.secretAccessKey,
-        apiVersion: "2006-03-01",
-        signatureVersion: "v4",
-      })
-      const params = { Bucket: bucket, Key: key }
-      signedUrl = s3.getSignedUrl("putObject", params)
-      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
-    } catch (error) {
-      ctx.throw(400, error)
-    }
-  }
-
-  ctx.body = { signedUrl, publicUrl }
-}

package/src/api/controllers/table/index.js

@@ -2,7 +2,7 @@ const CouchDB = require("../../../db")
 const internal = require("./internal")
 const external = require("./external")
 const csvParser = require("../../../utilities/csvParser")
-const { isExternalTable, isSQL } = require("../../../integrations/utils")
+const { isExternalTable } = require("../../../integrations/utils")
 const {
   getTableParams,
   getDatasourceParams,
@@ -32,8 +32,8 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const internal = internalTables.rows.map(tableDoc => ({
-    ...tableDoc.doc,
+  const internal = internalTables.rows.map(row => ({
+    ...row.doc,
     type: "internal",
     sourceId: BudibaseInternalDB._id,
   }))
@@ -44,18 +44,12 @@ exports.fetch = async function (ctx) {
     })
   )
 
-  const external = externalTables.rows.flatMap(tableDoc => {
-    let entities = tableDoc.doc.entities
-    if (entities) {
-      return Object.values(entities).map(entity => ({
-        ...entity,
-        type: "external",
-        sourceId: tableDoc.doc._id,
-        sql: isSQL(tableDoc.doc),
-      }))
-    } else {
-      return []
-    }
+  const external = externalTables.rows.flatMap(row => {
+    return Object.values(row.doc.entities || {}).map(entity => ({
+      ...entity,
+      type: "external",
+      sourceId: row.doc._id,
+    }))
   })
 
   ctx.body = [...internal, ...external]

package/src/api/controllers/table/internal.js

@@ -8,7 +8,6 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
-const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -120,7 +119,6 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
-  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({