@budibase/server 2.0.38 → 2.0.40-alpha.0

package/src/sdk/app/backups/imports.ts

@@ -0,0 +1,206 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { getAutomationParams, TABLE_ROW_PREFIX } from "../../../db/utils"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  upload,
+  uploadDirectory,
+} from "../../../utilities/fileSystem/utilities"
+import { downloadTemplate } from "../../../utilities/fileSystem"
+import { FieldTypes, ObjectStoreBuckets } from "../../../constants"
+import { join } from "path"
+import fs from "fs"
+import sdk from "../../"
+import {
+  Automation,
+  AutomationTriggerStepId,
+  CouchFindOptions,
+  RowAttachment,
+} from "@budibase/types"
+import PouchDB from "pouchdb"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+
+type TemplateType = {
+  file?: {
+    type: string
+    path: string
+  }
+  key?: string
+}
+
+async function updateAttachmentColumns(
+  prodAppId: string,
+  db: PouchDB.Database
+) {
+  // iterate through attachment documents and update them
+  const tables = await sdk.tables.getAllInternalTables(db)
+  for (let table of tables) {
+    const attachmentCols: string[] = []
+    for (let [key, column] of Object.entries(table.schema)) {
+      if (column.type === FieldTypes.ATTACHMENT) {
+        attachmentCols.push(key)
+      }
+    }
+    // no attachment columns, nothing to do
+    if (attachmentCols.length === 0) {
+      continue
+    }
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const params: CouchFindOptions = {
+      selector: {
+        _id: {
+          $regex: `^${TABLE_ROW_PREFIX}`,
+        },
+      },
+    }
+    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+    const { rows } = await dbCore.directCouchFind(db.name, params)
+    for (let row of rows) {
+      for (let column of attachmentCols) {
+        if (!Array.isArray(row[column])) {
+          continue
+        }
+        row[column] = row[column].map((attachment: RowAttachment) => {
+          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+          const urlParts = attachment.url.split("/")
+          // drop the first empty element
+          urlParts.shift()
+          // get the prefix
+          const prefix = urlParts.shift()
+          // remove the app ID
+          urlParts.shift()
+          // add new app ID
+          urlParts.unshift(prodAppId)
+          const key = urlParts.join("/")
+          return {
+            ...attachment,
+            key,
+            url: `/${prefix}/${key}`,
+          }
+        })
+      }
+    }
+    // write back the updated attachments
+    await db.bulkDocs(rows)
+  }
+}
+
+async function updateAutomations(prodAppId: string, db: PouchDB.Database) {
+  const automations = (
+    await db.allDocs(
+      getAutomationParams(null, {
+        include_docs: true,
+      })
+    )
+  ).rows.map(row => row.doc) as Automation[]
+  const devAppId = dbCore.getDevAppID(prodAppId)
+  let toSave: Automation[] = []
+  for (let automation of automations) {
+    const oldDevAppId = automation.appId,
+      oldProdAppId = dbCore.getProdAppID(automation.appId)
+    if (
+      automation.definition.trigger.stepId === AutomationTriggerStepId.WEBHOOK
+    ) {
+      const old = automation.definition.trigger.inputs
+      automation.definition.trigger.inputs = {
+        schemaUrl: old.schemaUrl.replace(oldDevAppId, devAppId),
+        triggerUrl: old.triggerUrl.replace(oldProdAppId, prodAppId),
+      }
+    }
+    automation.appId = devAppId
+    toSave.push(automation)
+  }
+  await db.bulkDocs(toSave)
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns a fs read stream which can be loaded into the database.
+ */
+async function getTemplateStream(template: TemplateType) {
+  if (template.file && template.file.type !== "text/plain") {
+    throw new Error("Cannot import a non-text based file.")
+  }
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else if (template.key) {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+export function untarFile(file: { path: string }) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  // extract the tarball
+  tar.extract({
+    sync: true,
+    cwd: tmpPath,
+    file: file.path,
+  })
+  return tmpPath
+}
+
+export function getGlobalDBFile(tmpPath: string) {
+  return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
+}
+
+export function getListOfAppsInMulti(tmpPath: string) {
+  return fs.readdirSync(tmpPath).filter(dir => dir !== GLOBAL_DB_EXPORT_FILE)
+}
+
+export async function importApp(
+  appId: string,
+  db: PouchDB.Database,
+  template: TemplateType
+) {
+  let prodAppId = dbCore.getProdAppID(appId)
+  let dbStream: any
+  const isTar = template.file && template?.file?.type?.endsWith("gzip")
+  const isDirectory =
+    template.file && fs.lstatSync(template.file.path).isDirectory()
+  if (template.file && (isTar || isDirectory)) {
+    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const contents = fs.readdirSync(tmpPath)
+    // have to handle object import
+    if (contents.length) {
+      let promises = []
+      let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
+      for (let filename of contents) {
+        const path = join(tmpPath, filename)
+        if (excludedFiles.includes(filename)) {
+          continue
+        }
+        filename = join(prodAppId, filename)
+        if (fs.lstatSync(path).isDirectory()) {
+          promises.push(
+            uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
+          )
+        } else {
+          promises.push(
+            upload({
+              bucket: ObjectStoreBuckets.APPS,
+              path,
+              filename,
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
+    }
+    dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
+  } else {
+    dbStream = await getTemplateStream(template)
+  }
+  // @ts-ignore
+  const { ok } = await db.load(dbStream)
+  if (!ok) {
+    throw "Error loading database dump from template."
+  }
+  await updateAttachmentColumns(prodAppId, db)
+  await updateAutomations(prodAppId, db)
+  return ok
+}

package/src/sdk/app/backups/index.ts

@@ -0,0 +1,9 @@
+import * as exportApps from "./exports"
+import * as importApps from "./imports"
+import * as statistics from "./statistics"
+
+export default {
+  ...exportApps,
+  ...importApps,
+  ...statistics,
+}

package/src/sdk/app/backups/statistics.ts

@@ -0,0 +1,78 @@
+import { context, db as dbCore } from "@budibase/backend-core"
+import {
+  getDatasourceParams,
+  getTableParams,
+  getAutomationParams,
+  getScreenParams,
+} from "../../../db/utils"
+import PouchDB from "pouchdb"
+
+async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+  if (db) {
+    return cb(db)
+  } else {
+    const devAppId = dbCore.getDevAppID(appId)
+    return context.doInAppContext(devAppId, () => {
+      const db = context.getAppDB()
+      return cb(db)
+    })
+  }
+}
+
+export async function calculateDatasourceCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const datasourceList = await db.allDocs(getDatasourceParams())
+      const tableList = await db.allDocs(getTableParams())
+      return datasourceList.rows.length + tableList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateAutomationCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const automationList = await db.allDocs(getAutomationParams())
+      return automationList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateScreenCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const screenList = await db.allDocs(getScreenParams())
+      return screenList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateBackupStats(appId: string) {
+  return runInContext(appId, async (db: PouchDB.Database) => {
+    const promises = []
+    promises.push(calculateDatasourceCount(appId, db))
+    promises.push(calculateAutomationCount(appId, db))
+    promises.push(calculateScreenCount(appId, db))
+    const responses = await Promise.all(promises)
+    return {
+      datasources: responses[0],
+      automations: responses[1],
+      screens: responses[2],
+    }
+  })
+}

package/src/sdk/app/tables/index.ts

@@ -0,0 +1,61 @@
+import { getAppDB } from "@budibase/backend-core/context"
+import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import { Table } from "@budibase/types"
+import PouchDB from "pouchdb"
+
+async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
+  if (!db) {
+    db = getAppDB() as PouchDB.Database
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+  const db = getAppDB()
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return datasource.entities
+}
+
+async function getExternalTable(
+  datasourceId: any,
+  tableName: any
+): Promise<Table> {
+  const entities = await getAllExternalTables(datasourceId)
+  return entities[tableName]
+}
+
+async function getTable(tableId: any): Promise<Table> {
+  const db = getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await db.get(datasourceId)
+    const table = await getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export default {
+  getAllInternalTables,
+  getAllExternalTables,
+  getExternalTable,
+  getTable,
+}

package/src/sdk/index.ts

@@ -0,0 +1,15 @@
+import { default as backups } from "./app/backups"
+import { default as tables } from "./app/tables"
+import { default as automations } from "./app/automations"
+
+const sdk = {
+  backups,
+  tables,
+  automations,
+}
+
+// default export for TS
+export default sdk
+
+// default export for JS
+module.exports = sdk

package/src/startup.ts

@@ -0,0 +1,138 @@
+import * as env from "./environment"
+import redis from "./utilities/redis"
+import {
+  createAdminUser,
+  generateApiKey,
+  getChecklist,
+} from "./utilities/workerRequests"
+import {
+  installation,
+  pinoSettings,
+  tenancy,
+  logging,
+} from "@budibase/backend-core"
+import fs from "fs"
+import { watch } from "./watch"
+import automations from "./automations"
+import fileSystem from "./utilities/fileSystem"
+import eventEmitter from "./events"
+import * as migrations from "./migrations"
+import bullboard from "./automations/bullboard"
+import * as pro from "@budibase/pro"
+import api from "./api"
+import sdk from "./sdk"
+const pino = require("koa-pino-logger")
+
+let STARTUP_RAN = false
+
+async function initRoutes(app: any) {
+  app.use(pino(pinoSettings()))
+
+  if (!env.isTest()) {
+    const plugin = await bullboard.init()
+    app.use(plugin)
+  }
+
+  app.context.eventEmitter = eventEmitter
+  app.context.auth = {}
+
+  // api routes
+  app.use(api.router.routes())
+}
+
+async function initPro() {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
+function shutdown(server?: any) {
+  server.close()
+  server.destroy()
+}
+
+export async function startup(app?: any, server?: any) {
+  if (STARTUP_RAN) {
+    return
+  }
+  STARTUP_RAN = true
+  if (server) {
+    console.log(`Budibase running on ${JSON.stringify(server.address())}`)
+    env._set("PORT", server.address().port)
+  }
+  eventEmitter.emitPort(env.PORT)
+  fileSystem.init()
+  await redis.init()
+
+  // run migrations on startup if not done via http
+  // not recommended in a clustered environment
+  if (!env.HTTP_MIGRATIONS && !env.isTest()) {
+    try {
+      await migrations.migrate()
+    } catch (e) {
+      logging.logAlert("Error performing migrations. Exiting.", e)
+      shutdown()
+    }
+  }
+
+  // check and create admin user if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.BB_ADMIN_USER_EMAIL &&
+    env.BB_ADMIN_USER_PASSWORD
+  ) {
+    const checklist = await getChecklist()
+    if (!checklist?.adminUser?.checked) {
+      try {
+        const tenantId = tenancy.getTenantId()
+        const user = await createAdminUser(
+          env.BB_ADMIN_USER_EMAIL,
+          env.BB_ADMIN_USER_PASSWORD,
+          tenantId
+        )
+        // Need to set up an API key for automated integration tests
+        if (env.isTest()) {
+          await generateApiKey(user._id)
+        }
+
+        console.log(
+          "Admin account automatically created for",
+          env.BB_ADMIN_USER_EMAIL
+        )
+      } catch (e) {
+        logging.logAlert("Error creating initial admin user. Exiting.", e)
+        shutdown()
+      }
+    }
+  }
+
+  // monitor plugin directory if required
+  if (
+    env.SELF_HOSTED &&
+    !env.MULTI_TENANCY &&
+    env.PLUGINS_DIR &&
+    fs.existsSync(env.PLUGINS_DIR)
+  ) {
+    watch()
+  }
+
+  // check for version updates
+  await installation.checkInstallVersion()
+
+  // get the references to the queue promises, don't await as
+  // they will never end, unless the processing stops
+  let queuePromises = []
+  queuePromises.push(automations.init())
+  queuePromises.push(initPro())
+  if (app) {
+    // bring routes online as final step once everything ready
+    await initRoutes(app)
+  }
+}

package/src/tests/utilities/TestConfiguration.js

@@ -25,7 +25,8 @@ const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, generateUserMetadataID } = require("../../db/utils")
+const { startup } = require("../../startup")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -41,6 +42,9 @@ class TestConfiguration {
       this.server = require("../../app")
       // we need the request for logging in, involves cookies, hard to fake
       this.request = supertest(this.server)
+      this.started = true
+    } else {
+      this.started = false
     }
     this.appId = null
     this.allApps = []
@@ -95,7 +99,13 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
-    await this.globalUser()
+    if (!this.started) {
+      await startup()
+    }
+    this.user = await this.globalUser()
+    this.globalUserId = this.user._id
+    this.userMetadataId = generateUserMetadataID(this.globalUserId)
+
     return this.createApp(appName)
   }
 

package/src/threads/automation.ts

@@ -1,6 +1,11 @@
 import { default as threadUtils } from "./utils"
+import { Job } from "bull"
 threadUtils.threadSetup()
-import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
+import {
+  isRecurring,
+  disableCronById,
+  isErrorInOutput,
+} from "../automations/utils"
 import { default as actions } from "../automations/actions"
 import { default as automationUtils } from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
@@ -13,7 +18,6 @@ import {
   LoopStep,
   LoopStepType,
   LoopInput,
-  AutomationEvent,
   TriggerOutput,
   AutomationContext,
   AutomationMetadata,
@@ -73,19 +77,16 @@ class Orchestrator {
   _automation: Automation
   _emitter: any
   _context: AutomationContext
-  _repeat?: { jobId: string; jobKey: string }
+  _job: Job
   executionOutput: AutomationContext
 
-  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
+  constructor(job: Job) {
+    let automation = job.data.automation,
+      triggerOutput = job.data.event
     const metadata = triggerOutput.metadata
     this._chainCount = metadata ? metadata.automationChainCount : 0
     this._appId = triggerOutput.appId as string
-    if (opts?.repeat) {
-      this._repeat = {
-        jobId: opts.repeat.jobId,
-        jobKey: opts.repeat.key,
-      }
-    }
+    this._job = job
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -134,7 +135,7 @@ class Orchestrator {
   }
 
   async stopCron(reason: string) {
-    if (!this._repeat) {
+    if (!this._job.opts.repeat) {
       return
     }
     logWarn(
@@ -142,7 +143,7 @@ class Orchestrator {
     )
     const automation = this._automation
     const trigger = automation.definition.trigger
-    await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
+    await disableCronById(this._job.id)
     this.updateExecutionOutput(
       trigger.id,
       trigger.stepId,
@@ -156,7 +157,7 @@ class Orchestrator {
   }
 
   async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this._repeat) {
+    if (!metadata.errorCount || !this._job.opts.repeat) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
@@ -475,17 +476,13 @@ class Orchestrator {
   }
 }
 
-export function execute(input: AutomationEvent, callback: WorkerCallback) {
-  const appId = input.data.event.appId
+export function execute(job: Job, callback: WorkerCallback) {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     try {
       const response = await automationOrchestrator.execute()
       callback(null, response)
@@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) {
   })
 }
 
-export const removeStalled = async (input: AutomationEvent) => {
-  const appId = input.data.event.appId
+export const removeStalled = async (job: Job) => {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     await automationOrchestrator.stopCron("stalled")
   })
 }

package/src/utilities/csvParser.js

@@ -4,6 +4,7 @@ const { FieldTypes } = require("../constants")
 const VALIDATORS = {
   [FieldTypes.STRING]: () => true,
   [FieldTypes.OPTIONS]: () => true,
+  [FieldTypes.BARCODEQR]: () => true,
   [FieldTypes.NUMBER]: attribute => {
     // allow not to be present
     if (!attribute) {