@budibase/server 2.1.45 → 2.1.46-alpha.1

package/src/tests/utilities/{structures.js → structures.ts}

@@ -1,14 +1,13 @@
-const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
-const { BUILTIN_PERMISSION_IDS } = require("@budibase/backend-core/permissions")
-const { createHomeScreen } = require("../../constants/screens")
-const { EMPTY_LAYOUT } = require("../../constants/layouts")
-const { cloneDeep } = require("lodash/fp")
+import { roles, permissions } from "@budibase/backend-core"
+import { createHomeScreen } from "../../constants/screens"
+import { EMPTY_LAYOUT } from "../../constants/layouts"
+import { cloneDeep } from "lodash/fp"
+import { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } from "../../automations"
 const { v4: uuidv4 } = require("uuid")
-const { TRIGGER_DEFINITIONS, ACTION_DEFINITIONS } = require("../../automations")
 
-exports.TENANT_ID = "default"
+export const TENANT_ID = "default"
 
-exports.basicTable = () => {
+export function basicTable() {
   return {
     name: "TestTable",
     type: "table",
@@ -30,16 +29,16 @@ exports.basicTable = () => {
   }
 }
 
-exports.basicView = tableId => {
+export function basicView(tableId: string) {
   return {
     tableId,
     name: "ViewTest",
   }
 }
 
-exports.filterView = tableId => {
+export function filterView(tableId: string) {
   return {
-    ...this.basicView(tableId),
+    ...basicView(tableId),
     filters: [
       {
         value: 0,
@@ -50,56 +49,58 @@ exports.filterView = tableId => {
   }
 }
 
-exports.calculationView = tableId => {
+export function calculationView(tableId: string) {
   return {
-    ...this.basicView(tableId),
+    ...basicView(tableId),
     field: "count",
     calculation: "sum",
   }
 }
 
-exports.view = tableId => {
+export function view(tableId: string) {
   return {
-    ...this.filterView(tableId),
-    ...this.calculationView(tableId),
+    ...filterView(tableId),
+    ...calculationView(tableId),
   }
 }
 
-exports.automationStep = (actionDefinition = ACTION_DEFINITIONS.CREATE_ROW) => {
+export function automationStep(
+  actionDefinition = ACTION_DEFINITIONS.CREATE_ROW
+) {
   return {
     id: uuidv4(),
     ...actionDefinition,
   }
 }
 
-exports.automationTrigger = (
+export function automationTrigger(
   triggerDefinition = TRIGGER_DEFINITIONS.ROW_SAVED
-) => {
+) {
   return {
     id: uuidv4(),
     ...triggerDefinition,
   }
 }
 
-exports.newAutomation = ({ steps, trigger } = {}) => {
-  const automation = exports.basicAutomation()
+export function newAutomation({ steps, trigger }: any = {}) {
+  const automation: any = basicAutomation()
 
   if (trigger) {
     automation.definition.trigger = trigger
   } else {
-    automation.definition.trigger = exports.automationTrigger()
+    automation.definition.trigger = automationTrigger()
   }
 
   if (steps) {
     automation.definition.steps = steps
   } else {
-    automation.definition.steps = [exports.automationStep()]
+    automation.definition.steps = [automationStep()]
   }
 
   return automation
 }
 
-exports.basicAutomation = () => {
+export function basicAutomation() {
   return {
     name: "My Automation",
     screenId: "kasdkfldsafkl",
@@ -115,7 +116,7 @@ exports.basicAutomation = () => {
   }
 }
 
-exports.basicRow = tableId => {
+export function basicRow(tableId: string) {
   return {
     name: "Test Contact",
     description: "original description",
@@ -123,23 +124,27 @@ exports.basicRow = tableId => {
   }
 }
 
-exports.basicLinkedRow = (tableId, linkedRowId, linkField = "link") => {
+export function basicLinkedRow(
+  tableId: string,
+  linkedRowId: string,
+  linkField: string = "link"
+) {
   // this is based on the basic linked tables you get from the test configuration
   return {
-    ...exports.basicRow(tableId),
+    ...basicRow(tableId),
     [linkField]: [linkedRowId],
   }
 }
 
-exports.basicRole = () => {
+export function basicRole() {
   return {
     name: "NewRole",
-    inherits: BUILTIN_ROLE_IDS.BASIC,
-    permissionId: BUILTIN_PERMISSION_IDS.READ_ONLY,
+    inherits: roles.BUILTIN_ROLE_IDS.BASIC,
+    permissionId: permissions.BuiltinPermissionID.READ_ONLY,
   }
 }
 
-exports.basicDatasource = () => {
+export function basicDatasource() {
   return {
     datasource: {
       type: "datasource",
@@ -150,7 +155,7 @@ exports.basicDatasource = () => {
   }
 }
 
-exports.basicQuery = datasourceId => {
+export function basicQuery(datasourceId: string) {
   return {
     datasourceId: datasourceId,
     name: "New Query",
@@ -161,7 +166,7 @@ exports.basicQuery = datasourceId => {
   }
 }
 
-exports.basicUser = role => {
+export function basicUser(role: string) {
   return {
     email: "bill@bill.com",
     password: "yeeooo",
@@ -169,15 +174,15 @@ exports.basicUser = role => {
   }
 }
 
-exports.basicScreen = () => {
+export function basicScreen() {
   return createHomeScreen()
 }
 
-exports.basicLayout = () => {
+export function basicLayout() {
   return cloneDeep(EMPTY_LAYOUT)
 }
 
-exports.basicWebhook = automationId => {
+export function basicWebhook(automationId: string) {
   return {
     live: true,
     name: "webhook",

package/src/threads/automation.ts

@@ -6,8 +6,8 @@ import {
   disableCronById,
   isErrorInOutput,
 } from "../automations/utils"
-import { default as actions } from "../automations/actions"
-import { default as automationUtils } from "../automations/automationUtils"
+import * as actions from "../automations/actions"
+import * as automationUtils from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
 import { generateAutomationMetadataID, isProdAppID } from "../db/utils"
 import { definitions as triggerDefs } from "../automations/triggerInfo"
@@ -23,15 +23,14 @@ import {
   AutomationMetadata,
 } from "../definitions/automations"
 import { WorkerCallback } from "./definitions"
-const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const { logAlertWithInfo, logWarn } = require("@budibase/backend-core/logging")
-const { processObject } = require("@budibase/string-templates")
+import { context, logging } from "@budibase/backend-core"
+import { processObject } from "@budibase/string-templates"
+import { cloneDeep } from "lodash/fp"
+import env from "../environment"
 const FILTER_STEP_ID = actions.ACTION_DEFINITIONS.FILTER.stepId
 const LOOP_STEP_ID = actions.ACTION_DEFINITIONS.LOOP.stepId
 const CRON_STEP_ID = triggerDefs.CRON.stepId
 const STOPPED_STATUS = { success: true, status: AutomationStatus.STOPPED }
-const { cloneDeep } = require("lodash/fp")
-const env = require("../environment")
 
 function typecastForLooping(loopStep: LoopStep, input: LoopInput) {
   if (!input || !input.binding) {
@@ -120,8 +119,8 @@ class Orchestrator {
   }
 
   async getMetadata(): Promise<AutomationMetadata> {
-    const metadataId = generateAutomationMetadataID(this._automation._id)
-    const db = getAppDB()
+    const metadataId = generateAutomationMetadataID(this._automation._id!)
+    const db = context.getAppDB()
     let metadata: AutomationMetadata
     try {
       metadata = await db.get(metadataId)
@@ -138,7 +137,7 @@ class Orchestrator {
     if (!this._job.opts.repeat) {
       return
     }
-    logWarn(
+    logging.logWarn(
       `CRON disabled reason=${reason} - ${this._appId}/${this._automation._id}`
     )
     const automation = this._automation
@@ -184,14 +183,14 @@ class Orchestrator {
     } else {
       metadata.errorCount = 0
     }
-    const db = getAppDB()
+    const db = context.getAppDB()
     try {
       await db.put(metadata)
     } catch (err) {
-      logAlertWithInfo(
+      logging.logAlertWithInfo(
         "Failed to write automation metadata",
         db.name,
-        automation._id,
+        automation._id!,
         err
       )
     }
@@ -265,7 +264,7 @@ class Orchestrator {
 
     for (let step of automation.definition.steps) {
       stepCount++
-      let input,
+      let input: any,
         iterations = 1,
         iterationCount = 0
 
@@ -283,7 +282,7 @@ class Orchestrator {
         let originalStepInput = cloneDeep(step.inputs)
         // Handle if the user has set a max iteration count or if it reaches the max limit set by us
         if (loopStep && input.binding) {
-          let newInput = await processObject(
+          let newInput: any = await processObject(
             loopStep.inputs,
             cloneDeep(this._context)
           )
@@ -336,7 +335,7 @@ class Orchestrator {
                   )) {
                     originalStepInput[key][innerKey][innerObject] =
                       automationUtils.substituteLoopStep(
-                        innerValue,
+                        innerValue as string,
                         `steps.${loopStepNumber}`
                       )
                   }
@@ -352,7 +351,7 @@ class Orchestrator {
             }
           }
           if (
-            index === env.AUTOMATION_MAX_ITERATION ||
+            index === env.AUTOMATION_MAX_ITERATIONS ||
             index === parseInt(loopStep.inputs.iterations)
           ) {
             this.updateContextAndOutput(loopStepNumber, step, tempOutput, {
@@ -481,7 +480,7 @@ export function execute(job: Job, callback: WorkerCallback) {
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
-  doInAppContext(appId, async () => {
+  return context.doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(job)
     try {
       const response = await automationOrchestrator.execute()
@@ -497,7 +496,7 @@ export const removeStalled = async (job: Job) => {
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
-  await doInAppContext(appId, async () => {
+  await context.doInAppContext(appId, async () => {
     const automationOrchestrator = new Orchestrator(job)
     await automationOrchestrator.stopCron("stalled")
   })

package/src/threads/query.ts

@@ -1,17 +1,12 @@
 import { default as threadUtils } from "./utils"
 threadUtils.threadSetup()
 import { WorkerCallback, QueryEvent, QueryVariable } from "./definitions"
-const ScriptRunner = require("../utilities/scriptRunner")
-const { getIntegration } = require("../integrations")
-const { processStringSync } = require("@budibase/string-templates")
-const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
-const {
-  refreshOAuthToken,
-  updateUserOAuth,
-} = require("@budibase/backend-core/auth")
-const { user: userCache } = require("@budibase/backend-core/cache")
-const { getGlobalIDFromUserMetadataID } = require("../db/utils")
-const { cloneDeep } = require("lodash/fp")
+import ScriptRunner from "../utilities/scriptRunner"
+import { getIntegration } from "../integrations"
+import { processStringSync } from "@budibase/string-templates"
+import { context, cache, auth } from "@budibase/backend-core"
+import { getGlobalIDFromUserMetadataID } from "../db/utils"
+import { cloneDeep } from "lodash/fp"
 
 const { isSQL } = require("../integrations/utils")
 const {
@@ -169,7 +164,7 @@ class QueryRunner {
   }
 
   async runAnotherQuery(queryId: string, parameters: any) {
-    const db = getAppDB()
+    const db = context.getAppDB()
     const query = await db.get(queryId)
     const datasource = await db.get(query.datasourceId)
     return new QueryRunner(
@@ -194,7 +189,7 @@ class QueryRunner {
       throw new Error("No refresh token found for authenticated user")
     }
 
-    const resp = await refreshOAuthToken(
+    const resp: any = await auth.refreshOAuthToken(
       oauth2.refreshToken,
       providerType,
       configId
@@ -204,8 +199,8 @@ class QueryRunner {
     // There are several other properties available in 'resp'
     if (!resp.err) {
       const globalUserId = getGlobalIDFromUserMetadataID(_id)
-      await updateUserOAuth(globalUserId, resp)
-      this.ctx.user = await userCache.getUser(globalUserId)
+      await auth.updateUserOAuth(globalUserId, resp)
+      this.ctx.user = await cache.user.getUser(globalUserId)
     } else {
       // In this event the user may have oAuth issues that
       // could require re-authenticating with their provider.
@@ -285,7 +280,7 @@ class QueryRunner {
 }
 
 export function execute(input: QueryEvent, callback: WorkerCallback) {
-  doInAppContext(input.appId, async () => {
+  context.doInAppContext(input.appId!, async () => {
     const Runner = new QueryRunner(input)
     try {
       const response = await Runner.execute()

package/src/threads/utils.ts

@@ -1,8 +1,7 @@
 import { QueryVariable } from "./definitions"
-const env = require("../environment")
-const db = require("../db")
-const redis = require("@budibase/backend-core/redis")
-const { SEPARATOR } = require("@budibase/backend-core/db")
+import env from "../environment"
+import * as db from "../db"
+import { redis, db as dbCore } from "@budibase/backend-core"
 
 const VARIABLE_TTL_SECONDS = 3600
 let client: any
@@ -21,7 +20,7 @@ process.on("exit", async () => {
 })
 
 function makeVariableKey(queryId: string, variable: string) {
-  return `${queryId}${SEPARATOR}${variable}`
+  return `${queryId}${dbCore.SEPARATOR}${variable}`
 }
 
 export function threadSetup() {

package/src/utilities/budibaseDir.ts

@@ -0,0 +1,3 @@
+import { objectStore } from "@budibase/backend-core"
+
+export const budibaseTempDir = objectStore.budibaseTempDir

package/src/utilities/{centralPath.js → centralPath.ts}

@@ -1,4 +1,4 @@
-const path = require("path")
+import path from "path"
 
 // this simply runs all of our path join and resolve functions through
 // a central location incase we need to add some protection to file paths
@@ -8,7 +8,7 @@ const path = require("path")
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-exports.join = function (...args) {
+export function join(...args: any) {
   return path.join(...args)
 }
 
@@ -17,6 +17,6 @@ exports.join = function (...args) {
  * @param args Any number of string arguments to add to a path
  * @returns {string} The final path ready to use
  */
-exports.resolve = function (...args) {
+export function resolve(...args: any) {
   return path.resolve(...args)
 }

package/src/utilities/{csvParser.js → csvParser.ts}

@@ -1,18 +1,25 @@
-const csv = require("csvtojson")
-const { FieldTypes } = require("../constants")
+import { FieldSchema, Table } from "@budibase/types"
+import csv from "csvtojson"
+import { FieldTypes } from "../constants"
 
-const VALIDATORS = {
+type CsvParseOpts = {
+  schema?: { [key: string]: any }
+  existingTable: Table
+  csvString?: string
+}
+
+const VALIDATORS: any = {
   [FieldTypes.STRING]: () => true,
   [FieldTypes.OPTIONS]: () => true,
   [FieldTypes.BARCODEQR]: () => true,
-  [FieldTypes.NUMBER]: attribute => {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
     // allow not to be present
     if (!attribute) {
       return true
     }
     return !isNaN(Number(attribute))
   },
-  [FieldTypes.DATETIME]: attribute => {
+  [FieldTypes.DATETIME]: (attribute?: string) => {
     // allow not to be present
     if (!attribute) {
       return true
@@ -21,14 +28,14 @@ const VALIDATORS = {
   },
 }
 
-const PARSERS = {
-  [FieldTypes.NUMBER]: attribute => {
+const PARSERS: any = {
+  [FieldTypes.NUMBER]: (attribute?: string) => {
     if (!attribute) {
       return attribute
     }
     return Number(attribute)
   },
-  [FieldTypes.DATETIME]: attribute => {
+  [FieldTypes.DATETIME]: (attribute?: string) => {
     if (!attribute) {
       return attribute
     }
@@ -36,10 +43,10 @@ const PARSERS = {
   },
 }
 
-function parse(csvString, parsers) {
+export function parse(csvString: string, parsers: any): Record<string, any> {
   const result = csv().fromString(csvString)
 
-  const schema = {}
+  const schema: Record<string, any> = {}
 
   return new Promise((resolve, reject) => {
     result.on("header", headers => {
@@ -77,16 +84,22 @@ function parse(csvString, parsers) {
   })
 }
 
-function updateSchema({ schema, existingTable }) {
+export function updateSchema({
+  schema,
+  existingTable,
+}: {
+  schema?: Record<string, any>
+  existingTable?: Table
+}) {
   if (!schema) {
     return schema
   }
-  const finalSchema = {}
-  const schemaKeyMap = {}
+  const finalSchema: Record<string, FieldSchema> = {}
+  const schemaKeyMap: Record<string, any> = {}
   Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
-  for (let [key, field] of Object.entries(existingTable.schema)) {
+  for (let [key, field] of Object.entries(existingTable?.schema || {})) {
     const lcKey = key.toLowerCase()
-    const foundKey = schemaKeyMap[lcKey]
+    const foundKey: string = schemaKeyMap[lcKey]
     if (foundKey) {
       finalSchema[key] = schema[foundKey]
       finalSchema[key].type = field.type
@@ -95,15 +108,22 @@ function updateSchema({ schema, existingTable }) {
   return finalSchema
 }
 
-async function transform({ schema, csvString, existingTable }) {
-  const colParser = {}
+export async function transform({
+  schema,
+  csvString,
+  existingTable,
+}: CsvParseOpts) {
+  if (!schema || !csvString) {
+    throw new Error("Unable to transform CSV without schema")
+  }
+  const colParser: any = {}
 
   // make sure the table has all the columns required for import
   if (existingTable) {
     schema = updateSchema({ schema, existingTable })
   }
 
-  for (let [key, field] of Object.entries(schema)) {
+  for (let [key, field] of Object.entries(schema || {})) {
     // don't import data to auto columns
     if (!field.autocolumn) {
       colParser[key] = PARSERS[field.type] || field.type
@@ -112,8 +132,10 @@ async function transform({ schema, csvString, existingTable }) {
 
   try {
     const data = await csv({ colParser }).fromString(csvString)
-    const schemaKeyMap = {}
-    Object.keys(schema).forEach(key => (schemaKeyMap[key.toLowerCase()] = key))
+    const schemaKeyMap: any = {}
+    Object.keys(schema || {}).forEach(
+      key => (schemaKeyMap[key.toLowerCase()] = key)
+    )
     for (let element of data) {
       if (!data) {
         continue
@@ -137,9 +159,3 @@ async function transform({ schema, csvString, existingTable }) {
     throw err
   }
 }
-
-module.exports = {
-  parse,
-  transform,
-  updateSchema,
-}

package/src/utilities/fileSystem/{clientLibrary.js → clientLibrary.ts}

@@ -1,9 +1,9 @@
-const { join } = require("path")
-const { ObjectStoreBuckets } = require("../../constants")
-const fs = require("fs")
-const { upload, retrieveToTmp, streamUpload } = require("./utilities")
-const { resolve } = require("../centralPath")
-const env = require("../../environment")
+import { join } from "path"
+import { ObjectStoreBuckets } from "../../constants"
+import fs from "fs"
+import { objectStore } from "@budibase/backend-core"
+import { resolve } from "../centralPath"
+import env from "../../environment"
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
 
 /**
@@ -33,18 +33,18 @@ const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..")
  * @param appId The app ID to backup
  * @returns {Promise<void>}
  */
-exports.backupClientLibrary = async appId => {
+export async function backupClientLibrary(appId: string) {
   // Copy existing manifest to tmp
   let tmpManifestPath
   try {
     // Try to load the manifest from the new file location
-    tmpManifestPath = await retrieveToTmp(
+    tmpManifestPath = await objectStore.retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(appId, "manifest.json")
     )
   } catch (error) {
     // Fallback to loading it from the old location for old apps
-    tmpManifestPath = await retrieveToTmp(
+    tmpManifestPath = await objectStore.retrieveToTmp(
       ObjectStoreBuckets.APPS,
       join(
         appId,
@@ -58,19 +58,19 @@ exports.backupClientLibrary = async appId => {
   }
 
   // Copy existing client lib to tmp
-  const tmpClientPath = await retrieveToTmp(
+  const tmpClientPath = await objectStore.retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js")
   )
 
   // Upload manifest and client library as backups
-  const manifestUpload = upload({
+  const manifestUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "manifest.json.bak"),
     path: tmpManifestPath,
     type: "application/json",
   })
-  const clientUpload = upload({
+  const clientUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "budibase-client.js.bak"),
     path: tmpClientPath,
@@ -85,7 +85,7 @@ exports.backupClientLibrary = async appId => {
  * @param appId The app ID to update
  * @returns {Promise<void>}
  */
-exports.updateClientLibrary = async appId => {
+export async function updateClientLibrary(appId: string) {
   let manifest, client
 
   if (env.isDev()) {
@@ -99,7 +99,7 @@ exports.updateClientLibrary = async appId => {
   }
 
   // Upload latest manifest and client library
-  const manifestUpload = streamUpload(
+  const manifestUpload = objectStore.streamUpload(
     ObjectStoreBuckets.APPS,
     join(appId, "manifest.json"),
     fs.createReadStream(manifest),
@@ -107,7 +107,7 @@ exports.updateClientLibrary = async appId => {
       ContentType: "application/json",
     }
   )
-  const clientUpload = streamUpload(
+  const clientUpload = objectStore.streamUpload(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js"),
     fs.createReadStream(client),
@@ -124,27 +124,27 @@ exports.updateClientLibrary = async appId => {
  * @param appId The app ID to revert
  * @returns {Promise<void>}
  */
-exports.revertClientLibrary = async appId => {
+export async function revertClientLibrary(appId: string) {
   // Copy backups manifest to tmp directory
-  const tmpManifestPath = await retrieveToTmp(
+  const tmpManifestPath = await objectStore.retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "manifest.json.bak")
   )
 
   // Copy backup client lib to tmp
-  const tmpClientPath = await retrieveToTmp(
+  const tmpClientPath = await objectStore.retrieveToTmp(
     ObjectStoreBuckets.APPS,
     join(appId, "budibase-client.js.bak")
   )
 
   // Upload backups as new versions
-  const manifestUpload = upload({
+  const manifestUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "manifest.json"),
     path: tmpManifestPath,
     type: "application/json",
   })
-  const clientUpload = upload({
+  const clientUpload = objectStore.upload({
     bucket: ObjectStoreBuckets.APPS,
     filename: join(appId, "budibase-client.js"),
     path: tmpClientPath,