@budibase/server 1.0.123 → 1.0.125

package/src/automations/{utils.js → utils.ts}

@@ -1,26 +1,31 @@
-const { Thread, ThreadType } = require("../threads")
-const { definitions } = require("./triggerInfo")
-const webhooks = require("../api/controllers/webhook")
-const { queue } = require("./bullboard")
-const newid = require("../db/newid")
-const { updateEntityMetadata } = require("../utilities")
-const { MetadataTypes, WebhookType } = require("../constants")
-const { getProdAppID } = require("@budibase/backend-core/db")
-const { cloneDeep } = require("lodash/fp")
-const { doWithDB } = require("@budibase/backend-core/db")
-const { getAppDB, getAppId } = require("@budibase/backend-core/context")
+import { Thread, ThreadType } from "../threads"
+import { definitions } from "./triggerInfo"
+import * as webhooks from "../api/controllers/webhook"
+import { queue } from "./bullboard"
+import newid from "../db/newid"
+import { updateEntityMetadata } from "../utilities"
+import { MetadataTypes, WebhookType } from "../constants"
+import { getProdAppID, doWithDB } from "@budibase/backend-core/db"
+import { cloneDeep } from "lodash/fp"
+import { getAppDB, getAppId } from "@budibase/backend-core/context"
+import { tenancy } from "@budibase/backend-core"
+import { quotas } from "@budibase/pro"
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
 const Runner = new Thread(ThreadType.AUTOMATION)
 
-exports.processEvent = async job => {
+export async function processEvent(job: any) {
   try {
-    // need to actually await these so that an error can be captured properly
     console.log(
       `${job.data.automation.appId} automation ${job.data.automation._id} running`
     )
-    return await Runner.run(job)
+    // need to actually await these so that an error can be captured properly
+    const tenantId = tenancy.getTenantIDFromAppID(job.data.event.appId)
+    return await tenancy.doInTenant(tenantId, async () => {
+      const runFn = () => Runner.run(job)
+      return quotas.addAutomation(runFn)
+    })
   } catch (err) {
     const errJson = JSON.stringify(err)
     console.error(
@@ -31,11 +36,15 @@ exports.processEvent = async job => {
   }
 }
 
-exports.updateTestHistory = async (appId, automation, history) => {
+export async function updateTestHistory(
+  appId: any,
+  automation: any,
+  history: any
+) {
   return updateEntityMetadata(
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automation._id,
-    metadata => {
+    (metadata: any) => {
       if (metadata && Array.isArray(metadata.history)) {
         metadata.history.push(history)
       } else {
@@ -48,7 +57,7 @@ exports.updateTestHistory = async (appId, automation, history) => {
   )
 }
 
-exports.removeDeprecated = definitions => {
+export function removeDeprecated(definitions: any) {
   const base = cloneDeep(definitions)
   for (let key of Object.keys(base)) {
     if (base[key].deprecated) {
@@ -59,13 +68,15 @@ exports.removeDeprecated = definitions => {
 }
 
 // end the repetition and the job itself
-exports.disableAllCrons = async appId => {
+export async function disableAllCrons(appId: any) {
   const promises = []
   const jobs = await queue.getRepeatableJobs()
   for (let job of jobs) {
     if (job.key.includes(`${appId}_cron`)) {
       promises.push(queue.removeRepeatableByKey(job.key))
-      promises.push(queue.removeJobs(job.id))
+      if (job.id) {
+        promises.push(queue.removeJobs(job.id))
+      }
     }
   }
   return Promise.all(promises)
@@ -76,9 +87,9 @@ exports.disableAllCrons = async appId => {
  * @param {string} appId The ID of the app in which we are checking for webhooks
  * @param {object|undefined} automation The automation object to be updated.
  */
-exports.enableCronTrigger = async (appId, automation) => {
+export async function enableCronTrigger(appId: any, automation: any) {
   const trigger = automation ? automation.definition.trigger : null
-  function isCronTrigger(auto) {
+  function isCronTrigger(auto: any) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -89,7 +100,7 @@ exports.enableCronTrigger = async (appId, automation) => {
   if (isCronTrigger(automation)) {
     // make a job id rather than letting Bull decide, makes it easier to handle on way out
     const jobId = `${appId}_cron_${newid()}`
-    const job = await queue.add(
+    const job: any = await queue.add(
       {
         automation,
         event: { appId, timestamp: Date.now() },
@@ -101,7 +112,7 @@ exports.enableCronTrigger = async (appId, automation) => {
     // can't use getAppDB here as this is likely to be called from dev app,
     // but this call could be for dev app or prod app, need to just use what
     // was passed in
-    await doWithDB(appId, async db => {
+    await doWithDB(appId, async (db: any) => {
       const response = await db.put(automation)
       automation._id = response.id
       automation._rev = response.rev
@@ -118,13 +129,13 @@ exports.enableCronTrigger = async (appId, automation) => {
  * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
  * written to DB (this does not write to DB as it would be wasteful to repeat).
  */
-exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
+export async function checkForWebhooks({ oldAuto, newAuto }: any) {
   const appId = getAppId()
   const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
   const newTrigger = newAuto ? newAuto.definition.trigger : null
   const triggerChanged =
     oldTrigger && newTrigger && oldTrigger.id !== newTrigger.id
-  function isWebhookTrigger(auto) {
+  function isWebhookTrigger(auto: any) {
     return (
       auto &&
       auto.definition.trigger &&
@@ -160,7 +171,7 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
     (!isWebhookTrigger(oldAuto) || triggerChanged) &&
     isWebhookTrigger(newAuto)
   ) {
-    const ctx = {
+    const ctx: any = {
       appId,
       request: {
         body: new webhooks.Webhook(
@@ -190,6 +201,6 @@ exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
  * @param appId {string} the app that is being removed.
  * @return {Promise<void>} clean is complete if this succeeds.
  */
-exports.cleanupAutomations = async appId => {
-  await exports.disableAllCrons(appId)
+export async function cleanupAutomations(appId: any) {
+  await disableAllCrons(appId)
 }

package/src/constants/index.js

@@ -198,5 +198,11 @@ exports.WebhookType = {
   AUTOMATION: "automation",
 }
 
+exports.AutomationErrors = {
+  INCORRECT_TYPE: "INCORRECT_TYPE",
+  MAX_ITERATIONS: "MAX_ITERATIONS_REACHED",
+  FAILURE_CONDITION: "FAILURE_CONDITION_MET",
+}
+
 // pass through the list from the auth/core lib
 exports.ObjectStoreBuckets = ObjectStoreBuckets

package/src/definitions/common.ts

@@ -19,6 +19,7 @@ export interface FieldSchema {
   through?: string
   foreignKey?: string
   autocolumn?: boolean
+  subtype?: string
   throughFrom?: string
   throughTo?: string
   formula?: string

package/src/definitions/datasource.ts

@@ -49,6 +49,7 @@ export enum SourceNames {
   ORACLE = "ORACLE",
   GOOGLE_SHEETS = "GOOGLE_SHEETS",
   FIREBASE = "FIREBASE",
+  REDIS = "REDIS",
 }
 
 export enum IncludeRelationships {

package/src/environment.js

@@ -34,7 +34,7 @@ let inThread = false
 
 module.exports = {
   // important
-  PORT: process.env.PORT,
+  PORT: process.env.PORT || process.env.APP_PORT,
   JWT_SECRET: process.env.JWT_SECRET,
   COUCH_DB_URL: process.env.COUCH_DB_URL,
   MINIO_URL: process.env.MINIO_URL,
@@ -44,8 +44,6 @@ module.exports = {
   ENABLE_ANALYTICS: process.env.ENABLE_ANALYTICS,
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
-  USE_QUOTAS: process.env.USE_QUOTAS,
-  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,
@@ -65,6 +63,7 @@ module.exports = {
   LOG_LEVEL: process.env.LOG_LEVEL,
   AUTOMATION_DIRECTORY: process.env.AUTOMATION_DIRECTORY,
   AUTOMATION_BUCKET: process.env.AUTOMATION_BUCKET,
+  AUTOMATION_MAX_ITERATIONS: process.env.AUTOMATION_MAX_ITERATIONS,
   SENDGRID_API_KEY: process.env.SENDGRID_API_KEY,
   DYNAMO_ENDPOINT: process.env.DYNAMO_ENDPOINT,
   POSTHOG_TOKEN: process.env.POSTHOG_TOKEN,

package/src/index.ts

@@ -1,3 +1,4 @@
+import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
 const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
 
@@ -6,6 +7,8 @@ function runServer() {
   // will print an error explaining what to do
   checkDevelopmentEnvironment()
   fixPath()
+  // this will setup http and https proxies form env variables
+  bootstrap()
   require("./app")
 }
 

package/src/integrations/index.ts

@@ -11,6 +11,7 @@ const arangodb = require("./arangodb")
 const rest = require("./rest")
 const googlesheets = require("./googlesheets")
 const firebase = require("./firebase")
+const redis = require("./redis")
 const { SourceNames } = require("../definitions/datasource")
 const environment = require("../environment")
 
@@ -26,6 +27,8 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
+  [SourceNames.FIREBASE]: firebase.schema,
+  [SourceNames.REDIS]: redis.schema,
 }
 
 const INTEGRATIONS = {
@@ -42,6 +45,7 @@ const INTEGRATIONS = {
   [SourceNames.REST]: rest.integration,
   [SourceNames.FIREBASE]: firebase.integration,
   [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
+  [SourceNames.REDIS]: redis.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/redis.ts

@@ -0,0 +1,152 @@
+import {
+  DatasourceFieldTypes,
+  Integration,
+  QueryTypes,
+} from "../definitions/datasource"
+import Redis from "ioredis"
+
+module RedisModule {
+  interface RedisConfig {
+    host: string
+    port: number
+    username: string
+    password?: string
+  }
+
+  const SCHEMA: Integration = {
+    docs: "https://redis.io/docs/",
+    description: "",
+    friendlyName: "Redis",
+    datasource: {
+      host: {
+        type: "string",
+        required: true,
+        default: "localhost",
+      },
+      port: {
+        type: "number",
+        required: true,
+        default: 6379,
+      },
+      username: {
+        type: "string",
+        required: false,
+      },
+      password: {
+        type: "password",
+        required: false,
+      },
+    },
+    query: {
+      create: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          value: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+          ttl: {
+            type: DatasourceFieldTypes.NUMBER,
+          },
+        },
+      },
+      read: {
+        readable: true,
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      delete: {
+        type: QueryTypes.FIELDS,
+        fields: {
+          key: {
+            type: DatasourceFieldTypes.STRING,
+            required: true,
+          },
+        },
+      },
+      command: {
+        readable: true,
+        displayName: "Redis Command",
+        type: QueryTypes.JSON,
+      },
+    },
+  }
+
+  class RedisIntegration {
+    private readonly config: RedisConfig
+    private client: any
+
+    constructor(config: RedisConfig) {
+      this.config = config
+      this.client = new Redis({
+        host: this.config.host,
+        port: this.config.port,
+        username: this.config.username,
+        password: this.config.password,
+      })
+    }
+
+    async disconnect() {
+      this.client.disconnect()
+    }
+
+    async redisContext(query: Function) {
+      try {
+        return await query()
+      } catch (err) {
+        throw new Error(`Redis error: ${err}`)
+      } finally {
+        this.disconnect()
+      }
+    }
+
+    async create(query: { key: string; value: string; ttl: number }) {
+      return this.redisContext(async () => {
+        const response = await this.client.set(query.key, query.value)
+        if (query.ttl) {
+          await this.client.expire(query.key, query.ttl)
+        }
+        return response
+      })
+    }
+
+    async read(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.get(query.key)
+        return response
+      })
+    }
+
+    async delete(query: { key: string }) {
+      return this.redisContext(async () => {
+        const response = await this.client.del(query.key)
+        return response
+      })
+    }
+
+    async command(query: { json: string }) {
+      return this.redisContext(async () => {
+        const commands = query.json.trim().split(" ")
+        const pipeline = this.client.pipeline([commands])
+        const result = await pipeline.exec()
+        return {
+          response: result[0][1],
+        }
+      })
+    }
+  }
+
+  module.exports = {
+    schema: SCHEMA,
+    integration: RedisIntegration,
+  }
+}

package/src/integrations/tests/redis.spec.js

@@ -0,0 +1,60 @@
+const Redis = require("ioredis-mock")
+const RedisIntegration = require("../redis")
+
+class TestConfiguration {
+  constructor(config = {}) {
+    this.integration = new RedisIntegration.integration(config) 
+    this.redis = new Redis({
+      data: {
+        test: 'test',
+        result: "1"
+      },
+    })
+    this.integration.client = this.redis
+  }
+}
+
+describe("Redis Integration", () => {
+  let config 
+
+  beforeEach(() => {
+    config = new TestConfiguration()
+  })
+
+  it("calls the create method with the correct params", async () => {
+    const body = {
+      key: "key",
+      value: "value"
+    }
+    const response = await config.integration.create(body)
+    expect(await config.redis.get("key")).toEqual("value")
+  })
+
+  it("calls the read method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    const response = await config.integration.read(body)
+    expect(response).toEqual("test")
+  })
+
+  it("calls the delete method with the correct params", async () => {
+    const body = {
+      key: "test"
+    }
+    await config.integration.delete(body)
+    expect(await config.redis.get(body.key)).toEqual(null)
+  })
+
+  it("calls the command method with the correct params", async () => {
+    const body = {
+      json: "KEYS *"
+    }
+
+    // ioredis-mock doesn't support pipelines
+    config.integration.client.pipeline = jest.fn(() => ({ exec: jest.fn(() => [[]]) }))
+
+    await config.integration.command(body)
+    expect(config.integration.client.pipeline).toHaveBeenCalledWith([["KEYS", "*"]])
+  })
+})

package/src/middleware/{authorized.js → authorized.ts}

@@ -1,8 +1,8 @@
-const {
+import {
   getUserRoleHierarchy,
   getRequiredResourceRole,
   BUILTIN_ROLE_IDS,
-} = require("@budibase/backend-core/roles")
+} from "@budibase/backend-core/roles"
 const {
   PermissionTypes,
   PermissionLevels,
@@ -13,7 +13,7 @@ const { isWebhookEndpoint } = require("./utils")
 const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
 const { getAppId } = require("@budibase/backend-core/context")
 
-function hasResource(ctx) {
+function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
@@ -25,7 +25,12 @@ const csrf = buildCsrfMiddleware()
  * - Builders can access all resources.
  * - Otherwise the user must have the required role.
  */
-const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
+const checkAuthorized = async (
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
+) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
   const isBuilderApi = permType === PermissionTypes.BUILDER
@@ -40,10 +45,10 @@ const checkAuthorized = async (ctx, resourceRoles, permType, permLevel) => {
 }
 
 const checkAuthorizedResource = async (
-  ctx,
-  resourceRoles,
-  permType,
-  permLevel
+  ctx: any,
+  resourceRoles: any,
+  permType: any,
+  permLevel: any
 ) => {
   // get the user's roles
   const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
@@ -54,7 +59,9 @@ const checkAuthorizedResource = async (
   // check if the user has the required role
   if (resourceRoles.length > 0) {
     // deny access if the user doesn't have the required resource role
-    const found = userRoles.find(role => resourceRoles.indexOf(role._id) !== -1)
+    const found = userRoles.find(
+      (role: any) => resourceRoles.indexOf(role._id) !== -1
+    )
     if (!found) {
       ctx.throw(403, permError)
     }
@@ -64,9 +71,8 @@ const checkAuthorizedResource = async (
   }
 }
 
-module.exports =
-  (permType, permLevel = null, opts = { schema: false }) =>
-  async (ctx, next) => {
+export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
+  async (ctx: any, next: any) => {
     // webhooks don't need authentication, each webhook unique
     // also internal requests (between services) don't need authorized
     if (isWebhookEndpoint(ctx) || ctx.internal) {
@@ -82,8 +88,8 @@ module.exports =
     await builderMiddleware(ctx, permType)
 
     // get the resource roles
-    let resourceRoles = [],
-      otherLevelRoles
+    let resourceRoles: any = []
+    let otherLevelRoles: any = []
     const otherLevel =
       permLevel === PermissionLevels.READ
         ? PermissionLevels.WRITE

package/src/middleware/currentapp.js

@@ -4,7 +4,7 @@ const {
   getCookie,
   clearCookie,
 } = require("@budibase/backend-core/utils")
-const { Cookies } = require("@budibase/backend-core/constants")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
 const { getRole } = require("@budibase/backend-core/roles")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { generateUserMetadataID, isDevAppID } = require("../db/utils")
@@ -63,6 +63,21 @@ module.exports = async (ctx, next) => {
     appId = requestAppId
     // retrieving global user gets the right role
     roleId = globalUser.roleId || roleId
+
+    // Allow builders to specify their role via a header
+    const isBuilder =
+      globalUser && globalUser.builder && globalUser.builder.global
+    const isDevApp = appId && isDevAppID(appId)
+    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
+    if (isBuilder && isDevApp && roleHeader) {
+      // Ensure the role is valid by ensuring a definition exists
+      try {
+        await getRole(roleHeader)
+        roleId = roleHeader
+      } catch (error) {
+        // Swallow error and do nothing
+      }
+    }
   }
 
   // nothing more to do

package/src/middleware/tests/currentapp.spec.js

@@ -1,10 +1,14 @@
 mockAuthWithNoCookie()
 mockWorker()
 
-jest.mock("@budibase/backend-core/db", () => ({
-  ...jest.requireActual("@budibase/backend-core/db"),
-  dbExists: () => true,
-}))
+jest.mock("@budibase/backend-core/db", () => {
+  const coreDb = jest.requireActual("@budibase/backend-core/db")
+  coreDb.init()
+  return {
+    ...coreDb,
+    dbExists: () => true,
+  }
+})
 
 function mockWorker() {
   jest.mock("../../utilities/workerRequests", () => ({

package/src/migrations/functions/developerQuota.ts

@@ -0,0 +1,15 @@
+const { createUserBuildersView } = require("@budibase/backend-core/db")
+import * as syncDevelopers from "./usageQuotas/syncDevelopers"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Create the builder users view and sync the developer count
+ */
+
+export const run = async (db: any) => {
+  await createUserBuildersView(db)
+  await syncDevelopers.run()
+}

package/src/migrations/functions/publishedAppsQuota.ts

@@ -0,0 +1,13 @@
+import * as syncPublishedApps from "./usageQuotas/syncPublishedApps"
+
+/**
+ * Date:
+ * March 2022
+ *
+ * Description:
+ * Sync the published apps count
+ */
+
+export const run = async (db: any) => {
+  await syncPublishedApps.run()
+}

package/src/migrations/functions/tests/quotas1.spec.js

@@ -1,4 +1,3 @@
-const env = require("../../../environment")
 const TestConfig = require("../../../tests/utilities/TestConfiguration")
 
 const syncApps = jest.fn()
@@ -14,7 +13,6 @@ describe("run", () => {
 
   beforeEach(async () => {
     await config.init()
-    env._set("USE_QUOTAS", 1)
   })
 
   afterAll(config.end)  

package/src/migrations/functions/usageQuotas/index.ts

@@ -1,8 +1,3 @@
-const { useQuotas } = require("../../../utilities/usageQuota")
-
 export const runQuotaMigration = async (migration: Function) => {
-  if (!useQuotas()) {
-    return
-  }
   await migration()
 }

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,9 +1,8 @@
-import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
+import { getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
+import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
-  const db = getGlobalDB()
   // get app count
   // @ts-ignore
   const devApps = await getAllApps({ dev: true })
@@ -12,7 +11,5 @@ export const run = async () => {
   // sync app count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  const usageDoc = await getUsageQuotaDoc(db)
-  usageDoc.usageQuota.apps = appCount
-  await db.put(usageDoc)
+  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
 }