@budibase/server 1.0.123-alpha.0 → 1.0.123

package/src/automations/{utils.ts → utils.js}

@@ -1,31 +1,26 @@
-import { Thread, ThreadType } from "../threads"
-import { definitions } from "./triggerInfo"
-import * as webhooks from "../api/controllers/webhook"
-import { queue } from "./bullboard"
-import newid from "../db/newid"
-import { updateEntityMetadata } from "../utilities"
-import { MetadataTypes, WebhookType } from "../constants"
-import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
-import { cloneDeep } from "lodash/fp"
-import { getAppDB, getAppId } from "@budibase/backend-core/context"
-import { tenancy } from "@budibase/backend-core"
-import { quotas } from "@budibase/pro"
+const { Thread, ThreadType } = require("../threads")
+const { definitions } = require("./triggerInfo")
+const webhooks = require("../api/controllers/webhook")
+const { queue } = require("./bullboard")
+const newid = require("../db/newid")
+const { updateEntityMetadata } = require("../utilities")
+const { MetadataTypes, WebhookType } = require("../constants")
+const { getProdAppID } = require("@budibase/backend-core/db")
+const { cloneDeep } = require("lodash/fp")
+const { doWithDB } = require("@budibase/backend-core/db")
+const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
-export async function processEvent(job: any) {
+exports.processEvent = async job => {
   try {
+    // need to actually await these so that an error can be captured properly
     console.log(
       `${job.data.automation.appId} automation ${job.data.automation._id} running`
     )
-    // need to actually await these so that an error can be captured properly
-    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
-    return await tenancy.doInTenant(tenantId, async () => {
-      const runFn = () => Runner.run(job)
-      return quotas.addAutomation(runFn)
-    })
+    return await Runner.run(job)
   } catch (err) {
     const errJson = JSON.stringify(err)
     console.error(
@@ -36,15 +31,11 @@ export async function processEvent(job: any) {
   }
 }
 
-export async function updateTestHistory(
-  appId: any,
-  automation: any,
-  history: any
-) {
+exports.updateTestHistory = async (appId, automation, history) => {
   return updateEntityMetadata(
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automation._id,
-    (metadata: any) => {
+    metadata => {
       if (metadata && Array.isArray(metadata.history)) {
         metadata.history.push(history)
       } else {
@@ -57,7 +48,7 @@ export async function updateTestHistory(
   )
 }
 
-export function removeDeprecated(definitions: any) {
+exports.removeDeprecated = definitions => {
   const base = cloneDeep(definitions)
   for (let key of Object.keys(base)) {
     if (base[key].deprecated) {
@@ -68,15 +59,13 @@ export function removeDeprecated(definitions: any) {
 }
 
 // end the repetition and the job itself
-export async function disableAllCrons(appId: any) {
+exports.disableAllCrons = async appId => {
   const promises = []
   const jobs = await queue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
       promises.push(queue.removeRepeatableByKey(job.key))
-      if (job.id) {
-        promises.push(queue.removeJobs(job.id))
-      }
+      promises.push(queue.removeJobs(job.id))
     }
   }
   return Promise.all(promises)
@@ -87,9 +76,9 @@ export async function disableAllCrons(appId: any) {
  * @param {string} appId The ID of the app in which we are checking for webhooks
  * @param {object|undefined} automation The automation object to be updated.
  */
-export async function enableCronTrigger(appId: any, automation: any) {
+exports.enableCronTrigger = async (appId, automation) => {
   const trigger = automation ? automation.definition.trigger : null
-  function isCronTrigger(auto: any) {
+  function isCronTrigger(auto) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -100,7 +89,7 @@ export async function enableCronTrigger(appId: any, automation: any) {
   if (isCronTrigger(automation)) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job: any = await queue.add(
+    const job = await queue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },
@@ -112,7 +101,7 @@ export async function enableCronTrigger(appId: any, automation: any) {
     // can't use getAppDB here as this is likely to be called from dev app,
     // but this call could be for dev app or prod app, need to just use what
     // was passed in
-    await doWithDB(appId, async (db: any) => {
+    await doWithDB(appId, async db => {
       const response = await db.put(automation)
       automation._id = response.id
       automation._rev = response.rev
@@ -129,13 +118,13 @@ export async function enableCronTrigger(appId: any, automation: any) {
  * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
  * written to DB (this does not write to DB as it would be wasteful to repeat).
  */
-export async function checkForWebhooks({ oldAuto, newAuto }: any) {
+exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
   const appId = getAppId()
   const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
   const newTrigger = newAuto ? newAuto.definition.trigger : null
   const triggerChanged =
     oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
-  function isWebhookTrigger(auto: any) {
+  function isWebhookTrigger(auto) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -171,7 +160,7 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx: any = {
+    const ctx = {
       appId,
       request: {
         body: new webhooks.Webhook(
@@ -201,6 +190,6 @@ export async function checkForWebhooks({ oldAuto, newAuto }: any) {
  * @param appId {string} the app that is being removed.
  * @return {Promise<void>} clean is complete if this succeeds.
  */
-export async function cleanupAutomations(appId: any) {
-  await disableAllCrons(appId)
+exports.cleanupAutomations = async appId => {
+  await exports.disableAllCrons(appId)
 }

package/src/constants/index.js

@@ -162,6 +162,14 @@ exports.AutoFieldSubTypes = {
   AUTO_ID: "autoID",
 }
 
+exports.AutoFieldDefaultNames = {
+  CREATED_BY: "Created By",
+  CREATED_AT: "Created At",
+  UPDATED_BY: "Updated By",
+  UPDATED_AT: "Updated At",
+  AUTO_ID: "Auto ID",
+}
+
 exports.OBJ_STORE_DIRECTORY = "/prod-budi-app-assets"
 exports.BaseQueryVerbs = {
   CREATE: "create",
@@ -190,11 +198,5 @@ exports.WebhookType = {
   AUTOMATION: "automation",
 }
 
-exports.AutomationErrors = {
-  INCORRECT_TYPE: "INCORRECT_TYPE",
-  MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
-  FAILURE_CONDITION: "FAILURE_CONDITION_MET",
-}
-
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/db/linkedRows/LinkController.js

@@ -376,6 +376,7 @@ class LinkController {
 
         if (field.autocolumn) {
           linkedField.autocolumn = field.autocolumn
+          linkedField.subtype = field.subtype
         }
 
         // check the linked table to make sure we aren't overwriting an existing column

package/src/definitions/datasource.ts

@@ -49,7 +49,6 @@ export enum SourceNames {
   ORACLE = "ORACLE",
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIREBASE = "FIREBASE",
-  REDIS = "REDIS",
 }
 
 export enum IncludeRelationships {

package/src/environment.js

@@ -34,7 +34,7 @@ let inThread = false
 
 module.exports = {
   // important
-  PORT: process.env.PORT || process.env.APP_PORT,
+  PORT: process.env.PORT,
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
@@ -44,6 +44,8 @@ module.exports = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
+  USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
@@ -63,7 +65,6 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
-  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,

package/src/index.ts

@@ -1,4 +1,3 @@
-import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
 const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
 
@@ -7,8 +6,6 @@ function runServer() {
   // will print an error explaining what to do
   checkDevelopmentEnvironment()
   fixPath()
-  // this will setup http and https proxies form env variables
-  bootstrap()
   require("./app")
 }
 

package/src/integrations/index.ts

@@ -11,7 +11,6 @@ const arangodb = require("./arangodb")
 const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
-const redis = require("./redis")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -27,8 +26,6 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
-  [SourceNames.FIREBASE]: firebase.schema,
-  [SourceNames.REDIS]: redis.schema,
 }
 
 const INTEGRATIONS = {
@@ -45,7 +42,6 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
   [SourceNames.FIREBASE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
-  [SourceNames.REDIS]: redis.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/middleware/{authorized.ts → authorized.js}

@@ -1,8 +1,8 @@
-import {
+const {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} from "@budibase/backend-core/roles"
+} = require("@budibase/backend-core/roles")
 const {
   PermissionTypes,
   PermissionLevels,
@@ -13,7 +13,7 @@ const { isWebhookEndpoint } = require("./utils")
 const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
 const { getAppId } = require("@budibase/backend-core/context")
 
-function hasResource(ctx: any) {
+function hasResource(ctx) {
   return ctx.resourceId != null
 }
 
@@ -25,12 +25,7 @@ const csrf = buildCsrfMiddleware()
  * - Builders can access all resources.
  * - Otherwise the user must have the required role.
  */
-const checkAuthorized = async (
-  ctx: any,
-  resourceRoles: any,
-  permType: any,
-  permLevel: any
-) => {
+const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   const isBuilderApi = permType === PermissionTypes.BUILDER
@@ -45,10 +40,10 @@ const checkAuthorized = async (
 }
 
 const checkAuthorizedResource = async (
-  ctx: any,
-  resourceRoles: any,
-  permType: any,
-  permLevel: any
+  ctx,
+  resourceRoles,
+  permType,
+  permLevel
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
@@ -59,9 +54,7 @@ const checkAuthorizedResource = async (
   // check if the user has the required role
   if (resourceRoles.length > 0) {
     // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(
-      (role: any) => resourceRoles.indexOf(role._id) !== -1
-    )
+    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
     if (!found) {
       ctx.throw(403, permError)
     }
@@ -71,8 +64,9 @@ const checkAuthorizedResource = async (
   }
 }
 
-export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
-  async (ctx: any, next: any) => {
+module.exports =
+  (permType, permLevel = null, opts = { schema: false }) =>
+  async (ctx, next) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized
     if (isWebhookEndpoint(ctx) || ctx.internal) {
@@ -88,8 +82,8 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     await builderMiddleware(ctx, permType)
 
     // get the resource roles
-    let resourceRoles: any = []
-    let otherLevelRoles: any = []
+    let resourceRoles = [],
+      otherLevelRoles
     const otherLevel =
       permLevel === PermissionLevels.READ
         ? PermissionLevels.WRITE

package/src/middleware/currentapp.js

@@ -4,7 +4,7 @@ const {
   getCookie,
   clearCookie,
 } = require("@budibase/backend-core/utils")
-const { Cookies, Headers } = require("@budibase/backend-core/constants")
+const { Cookies } = require("@budibase/backend-core/constants")
 const { getRole } = require("@budibase/backend-core/roles")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
@@ -63,21 +63,6 @@ module.exports = async (ctx, next) => {
     appId = requestAppId
     // retrieving global user gets the right role
     roleId = globalUser.roleId || roleId
-
-    // Allow builders to specify their role via a header
-    const isBuilder =
-      globalUser && globalUser.builder && globalUser.builder.global
-    const isDevApp = appId && isDevAppID(appId)
-    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
-    if (isBuilder && isDevApp && roleHeader) {
-      // Ensure the role is valid by ensuring a definition exists
-      try {
-        await getRole(roleHeader)
-        roleId = roleHeader
-      } catch (error) {
-        // Swallow error and do nothing
-      }
-    }
   }
 
   // nothing more to do

package/src/middleware/tests/usageQuota.spec.js

@@ -0,0 +1,120 @@
+jest.mock("../../db")
+jest.mock("../../utilities/usageQuota")
+jest.mock("@budibase/backend-core/tenancy", () => ({
+  getTenantId: () => "testing123"
+}))
+
+const usageQuotaMiddleware = require("../usageQuota")
+const usageQuota = require("../../utilities/usageQuota")
+
+jest.mock("@budibase/backend-core/db")
+const { dangerousGetDB } = require("@budibase/backend-core/db")
+
+const env = require("../../environment")
+
+class TestConfiguration {
+  constructor() {
+    this.throw = jest.fn()
+    this.next = jest.fn()
+    this.middleware = usageQuotaMiddleware
+    this.ctx = {
+      throw: this.throw,
+      next: this.next,
+      appId: "test",
+      request: {
+        body: {}
+      },
+      req: {
+        method: "POST",
+        url: "/applications"
+      }
+    }
+    usageQuota.useQuotas = () => true
+  }
+
+  executeMiddleware() {
+    return this.middleware(this.ctx, this.next)
+  }
+
+  setProd(bool) {
+    if (bool) {
+      env.isDev = () => false
+      env.isProd = () => true
+      this.ctx.user = { tenantId: "test" }
+    } else {
+      env.isDev = () => true
+      env.isProd = () => false
+    }
+  }
+
+  setMethod(method) {
+    this.ctx.req.method = method
+  }
+
+  setUrl(url) {
+    this.ctx.req.url = url
+  }
+
+  setBody(body) {
+    this.ctx.request.body = body
+  }
+
+  setFiles(files) {
+    this.ctx.request.files = { file: files }
+  }
+}
+
+describe("usageQuota middleware", () => {
+  let config
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  it("skips the middleware if there is no usage property or method", async () => {
+    await config.executeMiddleware()
+    expect(config.next).toHaveBeenCalled()
+  })
+
+  it("passes through to next middleware if document already exists", async () => {
+    config.setProd(true)
+    config.setBody({
+      _id: "test",
+      _rev: "test",
+    })
+
+    dangerousGetDB.mockImplementationOnce(() => ({
+      get: async () => true
+    }))
+
+    await config.executeMiddleware()
+
+    expect(config.next).toHaveBeenCalled()
+  })
+
+  it("throws if request has _id, but the document no longer exists", async () => {
+    config.setBody({
+      _id: "123",
+      _rev: "test",
+    })
+    config.setProd(true)
+
+    dangerousGetDB.mockImplementationOnce(() => ({
+      get: async () => {
+        throw new Error()
+      } 
+    }))
+
+    await config.executeMiddleware()
+    expect(config.throw).toHaveBeenCalledWith(404, `${config.ctx.request.body._id} does not exist`)
+  })
+
+  it("calculates and persists the correct usage quota for the relevant action", async () => {
+    config.setUrl("/rows")
+
+    await config.executeMiddleware()
+
+    expect(usageQuota.update).toHaveBeenCalledWith("rows", 1)
+    expect(config.next).toHaveBeenCalled()
+  })
+})

package/src/middleware/usageQuota.js

@@ -0,0 +1,164 @@
+const usageQuota = require("../utilities/usageQuota")
+const { getUniqueRows } = require("../utilities/usageQuota/rows")
+const {
+  isExternalTable,
+  isRowId: isExternalRowId,
+} = require("../integrations/utils")
+const { getAppDB } = require("@budibase/backend-core/context")
+
+// currently only counting new writes and deletes
+const METHOD_MAP = {
+  POST: 1,
+  DELETE: -1,
+}
+
+const DOMAIN_MAP = {
+  rows: usageQuota.Properties.ROW,
+  // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+  // views: usageQuota.Properties.VIEW,               // doesn't work yet
+  // users: usageQuota.Properties.USER,               // doesn't work yet
+  applications: usageQuota.Properties.APPS,
+  // this will not be updated by endpoint calls
+  // instead it will be updated by triggerInfo
+  // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
+}
+
+function getProperty(url) {
+  for (let domain of Object.keys(DOMAIN_MAP)) {
+    if (url.indexOf(domain) !== -1) {
+      return DOMAIN_MAP[domain]
+    }
+  }
+}
+
+module.exports = async (ctx, next) => {
+  if (!usageQuota.useQuotas()) {
+    return next()
+  }
+
+  let usage = METHOD_MAP[ctx.req.method]
+  const property = getProperty(ctx.req.url)
+  if (usage == null || property == null) {
+    return next()
+  }
+  // post request could be a save of a pre-existing entry
+  if (ctx.request.body && ctx.request.body._id && ctx.request.body._rev) {
+    const usageId = ctx.request.body._id
+    try {
+      if (ctx.appId) {
+        const db = getAppDB()
+        await db.get(usageId)
+      }
+      return next()
+    } catch (err) {
+      if (
+        isExternalTable(usageId) ||
+        (ctx.request.body.tableId &&
+          isExternalTable(ctx.request.body.tableId)) ||
+        isExternalRowId(usageId)
+      ) {
+        return next()
+      } else {
+        ctx.throw(404, `${usageId} does not exist`)
+      }
+    }
+  }
+
+  // update usage for uploads to be the total size
+  if (property === usageQuota.Properties.UPLOAD) {
+    const files =
+      ctx.request.files.file.length > 1
+        ? Array.from(ctx.request.files.file)
+        : [ctx.request.files.file]
+    usage = files.map(file => file.size).reduce((total, size) => total + size)
+  }
+  try {
+    await performRequest(ctx, next, property, usage)
+  } catch (err) {
+    ctx.throw(400, err)
+  }
+}
+
+const performRequest = async (ctx, next, property, usage) => {
+  const usageContext = {
+    skipNext: false,
+    skipUsage: false,
+    [usageQuota.Properties.APPS]: {},
+  }
+
+  if (usage === -1) {
+    if (PRE_DELETE[property]) {
+      await PRE_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (PRE_CREATE[property]) {
+      await PRE_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // run the request
+  if (!usageContext.skipNext) {
+    await usageQuota.update(property, usage, { dryRun: true })
+    await next()
+  }
+
+  if (usage === -1) {
+    if (POST_DELETE[property]) {
+      await POST_DELETE[property](ctx, usageContext)
+    }
+  } else {
+    if (POST_CREATE[property]) {
+      await POST_CREATE[property](ctx, usageContext)
+    }
+  }
+
+  // update the usage
+  if (!usageContext.skipUsage) {
+    await usageQuota.update(property, usage)
+  }
+}
+
+const appPreDelete = async (ctx, usageContext) => {
+  if (ctx.query.unpublish) {
+    // don't run usage decrement for unpublish
+    usageContext.skipUsage = true
+    return
+  }
+
+  // store the row count to delete
+  const rows = await getUniqueRows([ctx.appId])
+  if (rows.length) {
+    usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length }
+  }
+}
+
+const appPostDelete = async (ctx, usageContext) => {
+  // delete the app rows from usage
+  const rowCount = usageContext[usageQuota.Properties.APPS].rowCount
+  if (rowCount) {
+    await usageQuota.update(usageQuota.Properties.ROW, -rowCount)
+  }
+}
+
+const appPostCreate = async ctx => {
+  // app import & template creation
+  if (ctx.request.body.useTemplate === "true") {
+    const rows = await getUniqueRows([ctx.response.body.appId])
+    const rowCount = rows ? rows.length : 0
+    await usageQuota.update(usageQuota.Properties.ROW, rowCount)
+  }
+}
+
+const PRE_DELETE = {
+  [usageQuota.Properties.APPS]: appPreDelete,
+}
+
+const POST_DELETE = {
+  [usageQuota.Properties.APPS]: appPostDelete,
+}
+
+const PRE_CREATE = {}
+
+const POST_CREATE = {
+  [usageQuota.Properties.APPS]: appPostCreate,
+}

package/src/migrations/functions/tests/quotas1.spec.js

@@ -1,3 +1,4 @@
+const env = require("../../../environment")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
@@ -13,6 +14,7 @@ describe("run", () => {
 
   beforeEach(async () => {
     await config.init()
+    env._set("USE_QUOTAS", 1)
   })
 
   afterAll(config.end)  

package/src/migrations/functions/usageQuotas/index.ts

@@ -1,3 +1,8 @@
+const { useQuotas } = require("../../../utilities/usageQuota")
+
 export const runQuotaMigration = async (migration: Function) => {
+  if (!useQuotas()) {
+    return
+  }
   await migration()
 }