@budibase/server 1.0.123-alpha.0 → 1.0.123

package/src/migrations/functions/usageQuotas/syncApps.ts

@@ -1,8 +1,9 @@
-import { getTenantId } from "@budibase/backend-core/tenancy"
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
-import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 
 export const run = async () => {
+  const db = getGlobalDB()
   // get app count
   // @ts-ignore
   const devApps = await getAllApps({ dev: true })
@@ -11,5 +12,7 @@ export const run = async () => {
   // sync app count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`)
-  await quotas.setUsage(appCount, StaticQuotaName.APPS, QuotaUsageType.STATIC)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.apps = appCount
+  await db.put(usageDoc)
 }

package/src/migrations/functions/usageQuotas/syncRows.ts

@@ -1,9 +1,10 @@
-import { getTenantId } from "@budibase/backend-core/tenancy"
+import { getGlobalDB, getTenantId } from "@budibase/backend-core/tenancy"
 import { getAllApps } from "@budibase/backend-core/db"
+import { getUsageQuotaDoc } from "../../../utilities/usageQuota"
 import { getUniqueRows } from "../../../utilities/usageQuota/rows"
-import { quotas, QuotaUsageType, StaticQuotaName } from "@budibase/pro"
 
 export const run = async () => {
+  const db = getGlobalDB()
   // get all rows in all apps
   // @ts-ignore
   const allApps = await getAllApps({ all: true })
@@ -15,5 +16,7 @@ export const run = async () => {
   // sync row count
   const tenantId = getTenantId()
   console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`)
-  await quotas.setUsage(rowCount, StaticQuotaName.ROWS, QuotaUsageType.STATIC)
+  const usageDoc = await getUsageQuotaDoc(db)
+  usageDoc.usageQuota.rows = rowCount
+  await db.put(usageDoc)
 }

package/src/migrations/functions/usageQuotas/tests/syncApps.spec.js

@@ -0,0 +1,41 @@
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { TENANT_ID } = require("../../../../tests/utilities/structures")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
+const syncApps = require("../syncApps")
+const env = require("../../../../environment")
+
+describe("syncApps", () => {
+
+  let config = new TestConfig(false)
+
+    beforeEach(async () => {
+      await config.init()
+      env._set("USE_QUOTAS", 1)
+    })
+
+    afterAll(config.end)
+
+    it("runs successfully", async () => {
+      await doInTenant(TENANT_ID, async () => {
+        const db = getGlobalDB()
+        // create the usage quota doc and mock usages
+        await getUsageQuotaDoc(db)
+        await update(Properties.APPS, 3)
+
+        let usageDoc = await getUsageQuotaDoc(db)
+        expect(usageDoc.usageQuota.apps).toEqual(3)
+
+        // create an extra app to test the migration
+        await config.createApp("quota-test")
+
+        // migrate
+        await syncApps.run()
+
+        // assert the migration worked
+        usageDoc = await getUsageQuotaDoc(db)
+        expect(usageDoc.usageQuota.apps).toEqual(2)
+      })
+    })
+})
+

package/src/migrations/functions/usageQuotas/tests/syncRows.spec.js

@@ -0,0 +1,45 @@
+const { doInTenant, getGlobalDB } = require("@budibase/backend-core/tenancy")
+const TestConfig = require("../../../../tests/utilities/TestConfiguration")
+const { TENANT_ID } = require("../../../../tests/utilities/structures")
+const { getUsageQuotaDoc, update, Properties } = require("../../../../utilities/usageQuota")
+const syncRows = require("../syncRows")
+const env = require("../../../../environment")
+
+describe("syncRows", () => {
+  let config = new TestConfig(false)
+
+  beforeEach(async () => {
+    await config.init()
+    env._set("USE_QUOTAS", 1)
+  })
+
+  afterAll(config.end)  
+
+  it("runs successfully", async () => {
+    await doInTenant(TENANT_ID, async () => {
+      const db = getGlobalDB()
+      await getUsageQuotaDoc(db)
+      await update(Properties.ROW, 300)
+
+      let usageDoc = await getUsageQuotaDoc(db)
+      expect(usageDoc.usageQuota.rows).toEqual(300)
+
+      // app 1
+      await config.createTable()
+      await config.createRow()
+      // app 2
+      await config.createApp("second-app")
+      await config.createTable()
+      await config.createRow()
+      await config.createRow()
+
+      // migrate
+      await syncRows.run()
+
+      // assert the migration worked
+      usageDoc = await getUsageQuotaDoc(db)
+      expect(usageDoc.usageQuota.rows).toEqual(3)
+    })
+  })
+})
+

package/src/migrations/index.ts

@@ -7,8 +7,6 @@ const {
 import * as userEmailViewCasing from "./functions/userEmailViewCasing"
 import * as quota1 from "./functions/quotas1"
 import * as appUrls from "./functions/appUrls"
-import * as developerQuota from "./functions/developerQuota"
-import * as publishedAppsQuota from "./functions/publishedAppsQuota"
 
 export interface Migration {
   type: string
@@ -28,7 +26,7 @@ export interface Migration {
  */
 export interface MigrationOptions {
   tenantIds?: string[]
-  force?: {
+  forced?: {
     [type: string]: string[]
   }
 }
@@ -50,16 +48,6 @@ export const MIGRATIONS: Migration[] = [
     opts: { all: true },
     fn: appUrls.run,
   },
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "developer_quota",
-    fn: developerQuota.run,
-  },
-  {
-    type: MIGRATION_TYPES.GLOBAL,
-    name: "published_apps_quota",
-    fn: publishedAppsQuota.run,
-  },
 ]
 
 export const migrate = async (options?: MigrationOptions) => {

package/src/threads/automation.js

@@ -8,14 +8,10 @@ const { DocumentTypes } = require("../db/utils")
 const { doInTenant } = require("@budibase/backend-core/tenancy")
 const { definitions: triggerDefs } = require("../automations/triggerInfo")
 const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const { AutomationErrors } = require("../constants")
-const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
-const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 
+const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: false, status: "STOPPED" }
-const { cloneDeep } = require("lodash/fp")
-const env = require("../environment")
 
 /**
  * The automation orchestrator is a class responsible for executing automations.
@@ -78,208 +74,50 @@ class Orchestrator {
     this.executionOutput.steps.push(stepObj)
   }
 
-  updateContextAndOutput(loopStepNumber, step, output, result) {
-    this.executionOutput.steps.splice(loopStepNumber, 0, {
-      id: step.id,
-      stepId: step.stepId,
-      outputs: {
-        ...output,
-        success: result.success,
-        status: result.status,
-      },
-      inputs: step.inputs,
-    })
-    this._context.steps.splice(loopStepNumber, 0, {
-      ...output,
-      success: result.success,
-      status: result.status,
-    })
-  }
-
   async execute() {
     let automation = this._automation
     const app = await this.getApp()
     let stopped = false
-    let loopStep
-
-    let stepCount = 0
-    let loopStepNumber
-    let loopSteps = []
     for (let step of automation.definition.steps) {
-      stepCount++
-      let input
-      if (step.stepId === LOOP_STEP_ID) {
-        loopStep = step
-        loopStepNumber = stepCount
+      // execution stopped, record state for that
+      if (stopped) {
+        this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
         continue
       }
-
-      if (loopStep) {
-        input = await processObject(loopStep.inputs, this._context)
-      }
-      let iterations = loopStep ? input.binding.length : 1
-      let iterationCount = 0
-      for (let index = 0; index < iterations; index++) {
-        let originalStepInput = cloneDeep(step.inputs)
-
-        // Handle if the user has set a max iteration count or if it reaches the max limit set by us
-        if (loopStep) {
-          // lets first of all handle the input
-          // if the input is array then use it, if it is a string then split it on every new line
-          let newInput = await processObject(
-            loopStep.inputs,
-            cloneDeep(this._context)
-          )
-          newInput = automationUtils.cleanInputValues(
-            newInput,
-            loopStep.schema.inputs
-          )
-          this._context.steps[loopStepNumber] = {
-            currentItem: newInput.binding[index],
-          }
-
-          let tempOutput = { items: loopSteps, iterations: iterationCount }
-          if (
-            (loopStep.inputs.option === "Array" &&
-              !Array.isArray(newInput.binding)) ||
-            (loopStep.inputs.option === "String" &&
-              typeof newInput.binding !== "string")
-          ) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.INCORRECT_TYPE,
-              success: false,
-            })
-            loopSteps = null
-            loopStep = null
-            break
-          }
-
-          // The "Loop" binding in the front end is "fake", so replace it here so the context can understand it
-          // Pretty hacky because we need to account for the row object
-          for (let [key, value] of Object.entries(originalStepInput)) {
-            if (typeof value === "object") {
-              for (let [innerKey, innerValue] of Object.entries(
-                originalStepInput[key]
-              )) {
-                if (typeof innerValue === "string") {
-                  originalStepInput[key][innerKey] =
-                    automationUtils.substituteLoopStep(
-                      innerValue,
-                      `steps.${loopStepNumber}`
-                    )
-                }
-              }
-            } else {
-              if (typeof value === "string") {
-                originalStepInput[key] = automationUtils.substituteLoopStep(
-                  value,
-                  `steps.${loopStepNumber}`
-                )
-              }
-            }
-          }
-
-          if (
-            index === parseInt(env.AUTOMATION_MAX_ITERATIONS) ||
-            index === loopStep.inputs.iterations
-          ) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.MAX_ITERATIONS,
-              success: true,
-            })
-            loopSteps = null
-            loopStep = null
-            break
-          }
-
-          if (
-            this._context.steps[loopStepNumber]?.currentItem ===
-            loopStep.inputs.failure
-          ) {
-            this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
-              status: AutomationErrors.FAILURE_CONDITION,
-              success: false,
-            })
-            loopSteps = null
-            loopStep = null
-            break
-          }
-        }
-
-        // execution stopped, record state for that
-        if (stopped) {
-          this.updateExecutionOutput(step.id, step.stepId, {}, STOPPED_STATUS)
-          continue
-        }
-
-        // If it's a loop step, we need to manually add the bindings to the context
-        let stepFn = await this.getStepFunctionality(step.stepId)
-        let inputs = await processObject(originalStepInput, this._context)
-        inputs = automationUtils.cleanInputValues(inputs, step.schema.inputs)
-        try {
-          // appId is always passed
-          let tenantId = app.tenantId || DEFAULT_TENANT_ID
-          const outputs = await doInTenant(tenantId, () => {
-            return stepFn({
-              inputs: inputs,
-              appId: this._appId,
-              emitter: this._emitter,
-              context: this._context,
-            })
+      let stepFn = await this.getStepFunctionality(step.stepId)
+      step.inputs = await processObject(step.inputs, this._context)
+      step.inputs = automationUtils.cleanInputValues(
+        step.inputs,
+        step.schema.inputs
+      )
+      // appId is always passed
+      try {
+        let tenantId = app.tenantId || DEFAULT_TENANT_ID
+        const outputs = await doInTenant(tenantId, () => {
+          return stepFn({
+            inputs: step.inputs,
+            appId: this._appId,
+            emitter: this._emitter,
+            context: this._context,
           })
-          this._context.steps[stepCount] = outputs
-          // if filter causes us to stop execution don't break the loop, set a var
-          // so that we can finish iterating through the steps and record that it stopped
-          if (step.stepId === FILTER_STEP_ID && !outputs.success) {
-            stopped = true
-            this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
-              ...outputs,
-              ...STOPPED_STATUS,
-            })
-            continue
-          }
-          if (loopStep && loopSteps) {
-            loopSteps.push(outputs)
-          } else {
-            this.updateExecutionOutput(
-              step.id,
-              step.stepId,
-              step.inputs,
-              outputs
-            )
-          }
-        } catch (err) {
-          console.error(`Automation error - ${step.stepId} - ${err}`)
-          return err
-        }
-        if (loopStep) {
-          iterationCount++
-          if (index === iterations - 1) {
-            loopStep = null
-            this._context.steps.splice(loopStepNumber, 1)
-            break
-          }
-        }
-      }
-
-      if (loopSteps && loopSteps.length) {
-        let tempOutput = {
-          success: true,
-          items: loopSteps,
-          iterations: iterationCount,
-        }
-        this.executionOutput.steps.splice(loopStepNumber + 1, 0, {
-          id: step.id,
-          stepId: step.stepId,
-          outputs: tempOutput,
-          inputs: step.inputs,
         })
-
-        this._context.steps.splice(loopStepNumber, 0, tempOutput)
-        loopSteps = null
+        this._context.steps.push(outputs)
+        // if filter causes us to stop execution don't break the loop, set a var
+        // so that we can finish iterating through the steps and record that it stopped
+        if (step.stepId === FILTER_STEP_ID && !outputs.success) {
+          stopped = true
+          this.updateExecutionOutput(step.id, step.stepId, step.inputs, {
+            ...outputs,
+            ...STOPPED_STATUS,
+          })
+          continue
+        }
+        this.updateExecutionOutput(step.id, step.stepId, step.inputs, outputs)
+      } catch (err) {
+        console.error(`Automation error - ${step.stepId} - ${err}`)
+        return err
       }
     }
-
     return this.executionOutput
   }
 }

package/src/threads/{index.ts → index.js}

@@ -1,12 +1,12 @@
-import workerFarm from "worker-farm"
-import * as env from "../environment"
+const workerFarm = require("worker-farm")
+const env = require("../environment")
 
-export const ThreadType = {
+const ThreadType = {
   QUERY: "query",
   AUTOMATION: "automation",
 }
 
-function typeToFile(type: any) {
+function typeToFile(type) {
   let filename = null
   switch (type) {
     case ThreadType.QUERY:
@@ -21,14 +21,8 @@ function typeToFile(type: any) {
   return require.resolve(filename)
 }
 
-export class Thread {
-  type: any
-  count: any
-  disableThreading: any
-  workers: any
-  timeoutMs: any
-
-  constructor(type: any, opts: any = { timeoutMs: null, count: 1 }) {
+class Thread {
+  constructor(type, opts = { timeoutMs: null, count: 1 }) {
     this.type = type
     this.count = opts.count ? opts.count : 1
     this.disableThreading =
@@ -37,7 +31,7 @@ export class Thread {
       this.count === 0 ||
       env.isInThread()
     if (!this.disableThreading) {
-      const workerOpts: any = {
+      const workerOpts = {
         autoStart: true,
         maxConcurrentWorkers: this.count,
       }
@@ -49,7 +43,8 @@ export class Thread {
     }
   }
 
-  run(data: any) {
+  run(data) {
+    const timeoutMs = this.timeoutMs
     return new Promise((resolve, reject) => {
       let fncToCall
       // if in test then don't use threading
@@ -58,12 +53,10 @@ export class Thread {
       } else {
         fncToCall = this.workers
       }
-      fncToCall(data, (err: any, response: any) => {
+      fncToCall(data, (err, response) => {
         if (err && err.type === "TimeoutError") {
           reject(
-            new Error(
-              `Query response time exceeded ${this.timeoutMs}ms timeout.`
-            )
+            new Error(`Query response time exceeded ${timeoutMs}ms timeout.`)
           )
         } else if (err) {
           reject(err)
@@ -74,3 +67,6 @@ export class Thread {
     })
   }
 }
+
+module.exports.Thread = Thread
+module.exports.ThreadType = ThreadType

package/src/threads/utils.js

@@ -71,7 +71,6 @@ exports.hasExtraData = response => {
   return (
     typeof response === "object" &&
     !Array.isArray(response) &&
-    response &&
     response.data != null &&
     response.info != null
   )

package/src/utilities/queue/inMemoryQueue.js

@@ -66,8 +66,7 @@ class InMemoryQueue {
    * @param {object} msg A message to be transported over the queue, this should be
    * a JSON message as this is required by Bull.
    */
-  // eslint-disable-next-line no-unused-vars
-  add(msg, repeat) {
+  add(msg) {
     if (typeof msg !== "object") {
       throw "Queue only supports carrying JSON."
     }
@@ -91,11 +90,6 @@ class InMemoryQueue {
     return []
   }
 
-  // eslint-disable-next-line no-unused-vars
-  removeJobs(pattern) {
-    // no-op
-  }
-
   /**
    * Implemented for tests
    */

package/src/utilities/rowProcessor/index.js

@@ -2,7 +2,7 @@ const linkRows = require("../../db/linkedRows")
 const { cloneDeep } = require("lodash/fp")
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
 const { attachmentsRelativeURL } = require("../index")
-const { processFormulas } = require("./utils")
+const { processFormulas, fixAutoColumnSubType } = require("./utils")
 const { deleteFiles } = require("../../utilities/fileSystem/utilities")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
@@ -11,6 +11,7 @@ const {
   dbExists,
 } = require("@budibase/backend-core/db")
 const { getAppId } = require("@budibase/backend-core/context")
+const { InternalTables } = require("../../db/utils")
 
 const BASE_AUTO_ID = 1
 
@@ -137,21 +138,23 @@ function processAutoColumn(
   opts = { reprocessing: false, noAutoRelationships: false }
 ) {
   let noUser = !user || !user.userId
+  let isUserTable = table._id === InternalTables.USER_METADATA
   let now = new Date().toISOString()
   // if a row doesn't have a revision then it doesn't exist yet
   const creating = !row._rev
+  // check its not user table, or whether any of the processing options have been disabled
+  const shouldUpdateUserFields =
+    !isUserTable && !opts.reprocessing && !opts.noAutoRelationships && !noUser
   for (let [key, schema] of Object.entries(table.schema)) {
     if (!schema.autocolumn) {
       continue
     }
+    if (!schema.subtype) {
+      schema = fixAutoColumnSubType(schema)
+    }
     switch (schema.subtype) {
       case AutoFieldSubTypes.CREATED_BY:
-        if (
-          creating &&
-          !opts.reprocessing &&
-          !opts.noAutoRelationships &&
-          !noUser
-        ) {
+        if (creating && shouldUpdateUserFields) {
           row[key] = [user.userId]
         }
         break
@@ -161,7 +164,7 @@ function processAutoColumn(
         }
         break
       case AutoFieldSubTypes.UPDATED_BY:
-        if (!opts.reprocessing && !opts.noAutoRelationships && !noUser) {
+        if (shouldUpdateUserFields) {
           row[key] = [user.userId]
         }
         break
@@ -179,7 +182,7 @@ function processAutoColumn(
   return { table, row }
 }
 exports.processAutoColumn = processAutoColumn
-
+exports.fixAutoColumnSubType = fixAutoColumnSubType
 exports.processFormulas = processFormulas
 
 /**

package/src/utilities/rowProcessor/utils.js

@@ -1,6 +1,34 @@
-const { FieldTypes, FormulaTypes } = require("../../constants")
+const {
+  FieldTypes,
+  FormulaTypes,
+  AutoFieldDefaultNames,
+  AutoFieldSubTypes,
+} = require("../../constants")
 const { processStringSync } = require("@budibase/string-templates")
 
+/**
+ * If the subtype has been lost for any reason this works out what
+ * subtype the auto column should be.
+ */
+exports.fixAutoColumnSubType = column => {
+  if (!column.autocolumn || !column.name || column.subtype) {
+    return column
+  }
+  // the columns which get auto generated
+  if (column.name.endsWith(AutoFieldDefaultNames.CREATED_BY)) {
+    column.subtype = AutoFieldSubTypes.CREATED_BY
+  } else if (column.name.endsWith(AutoFieldDefaultNames.UPDATED_BY)) {
+    column.subtype = AutoFieldSubTypes.UPDATED_BY
+  } else if (column.name.endsWith(AutoFieldDefaultNames.CREATED_AT)) {
+    column.subtype = AutoFieldSubTypes.CREATED_AT
+  } else if (column.name.endsWith(AutoFieldDefaultNames.UPDATED_AT)) {
+    column.subtype = AutoFieldSubTypes.UPDATED_AT
+  } else if (column.name.endsWith(AutoFieldDefaultNames.AUTO_ID)) {
+    column.subtype = AutoFieldSubTypes.AUTO_ID
+  }
+  return column
+}
+
 /**
  * Looks through the rows provided and finds formulas - which it then processes.
  */