@budibase/server 2.0.30-alpha.14 → 2.0.30-alpha.15

package/src/sdk/app/backups/exports.ts

@@ -0,0 +1,171 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { retrieveDirectory } from "../../../utilities/fileSystem/utilities"
+import { streamFile, createTempFolder } from "../../../utilities/fileSystem"
+import { ObjectStoreBuckets } from "../../../constants"
+import {
+  LINK_USER_METADATA_PREFIX,
+  TABLE_ROW_PREFIX,
+  USER_METDATA_PREFIX,
+} from "../../../db/utils"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import fs from "fs"
+import { join } from "path"
+import env from "../../../environment"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+const MemoryStream = require("memorystream")
+
+type ExportOpts = {
+  filter?: any
+  exportPath?: string
+  tar?: boolean
+  excludeRows?: boolean
+}
+
+function tarFilesToTmp(tmpDir: string, files: string[]) {
+  const exportFile = join(budibaseTempDir(), `${uuid()}.tar.gz`)
+  tar.create(
+    {
+      sync: true,
+      gzip: true,
+      file: exportFile,
+      recursive: true,
+      cwd: tmpDir,
+    },
+    files
+  )
+  return exportFile
+}
+
+/**
+ * Exports a DB to either file or a variable (memory).
+ * @param {string} dbName the DB which is to be exported.
+ * @param {object} opts various options for the export, e.g. whether to stream,
+ * a filter function or the name of the export.
+ * @return {*} either a readable stream or a string
+ */
+export async function exportDB(dbName: string, opts: ExportOpts = {}) {
+  return dbCore.doWithDB(dbName, async (db: any) => {
+    // Write the dump to file if required
+    if (opts?.exportPath) {
+      const path = opts?.exportPath
+      const writeStream = fs.createWriteStream(path)
+      await db.dump(writeStream, { filter: opts?.filter })
+      return path
+    } else {
+      // Stringify the dump in memory if required
+      const memStream = new MemoryStream()
+      let appString = ""
+      memStream.on("data", (chunk: any) => {
+        appString += chunk.toString()
+      })
+      await db.dump(memStream, { filter: opts?.filter })
+      return appString
+    }
+  })
+}
+
+function defineFilter(excludeRows?: boolean) {
+  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
+  if (excludeRows) {
+    ids.push(TABLE_ROW_PREFIX)
+  }
+  return (doc: any) =>
+    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
+}
+
+/**
+ * Local utility to back up the database state for an app, excluding global user
+ * data or user relationships.
+ * @param {string} appId The app to back up
+ * @param {object} config Config to send to export DB/attachment export
+ * @returns {*} either a string or a stream of the backup
+ */
+export async function exportApp(appId: string, config?: ExportOpts) {
+  const prodAppId = dbCore.getProdAppID(appId)
+  const appPath = `${prodAppId}/`
+  // export bucket contents
+  let tmpPath
+  if (!env.isTest()) {
+    tmpPath = await retrieveDirectory(ObjectStoreBuckets.APPS, appPath)
+  } else {
+    tmpPath = createTempFolder(uuid())
+  }
+  const downloadedPath = join(tmpPath, appPath)
+  if (fs.existsSync(downloadedPath)) {
+    const allFiles = fs.readdirSync(downloadedPath)
+    for (let file of allFiles) {
+      const path = join(downloadedPath, file)
+      // move out of app directory, simplify structure
+      fs.renameSync(path, join(downloadedPath, "..", file))
+    }
+    // remove the old app directory created by object export
+    fs.rmdirSync(downloadedPath)
+  }
+  // enforce an export of app DB to the tmp path
+  const dbPath = join(tmpPath, DB_EXPORT_FILE)
+  await exportDB(appId, {
+    ...config,
+    filter: defineFilter(config?.excludeRows),
+    exportPath: dbPath,
+  })
+  // if tar requested, return where the tarball is
+  if (config?.tar) {
+    // now the tmpPath contains both the DB export and attachments, tar this
+    const tarPath = tarFilesToTmp(tmpPath, fs.readdirSync(tmpPath))
+    // cleanup the tmp export files as tarball returned
+    fs.rmSync(tmpPath, { recursive: true, force: true })
+    return tarPath
+  }
+  // tar not requested, turn the directory where export is
+  else {
+    return tmpPath
+  }
+}
+
+/**
+ * Export all apps + global DB (if supplied) to a single tarball, this includes
+ * the attachments for each app as well.
+ * @param {object[]} appMetadata The IDs and names of apps to export.
+ * @param {string} globalDbContents The contents of the global DB to export as well.
+ * @return {string} The path to the tarball.
+ */
+export async function exportMultipleApps(
+  appMetadata: { appId: string; name: string }[],
+  globalDbContents?: string
+) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  let exportPromises: Promise<void>[] = []
+  // export each app to a directory, then move it into the complete export
+  const exportAndMove = async (appId: string, appName: string) => {
+    const path = await exportApp(appId)
+    await fs.promises.rename(path, join(tmpPath, appName))
+  }
+  for (let metadata of appMetadata) {
+    exportPromises.push(exportAndMove(metadata.appId, metadata.name))
+  }
+  // wait for all exports to finish
+  await Promise.all(exportPromises)
+  // add the global DB contents
+  if (globalDbContents) {
+    fs.writeFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), globalDbContents)
+  }
+  const appNames = appMetadata.map(metadata => metadata.name)
+  const tarPath = tarFilesToTmp(tmpPath, [...appNames, GLOBAL_DB_EXPORT_FILE])
+  // clear up the tmp path now tarball generated
+  fs.rmSync(tmpPath, { recursive: true, force: true })
+  return tarPath
+}
+
+/**
+ * Streams a backup of the database state for an app
+ * @param {string} appId The ID of the app which is to be backed up.
+ * @param {boolean} excludeRows Flag to state whether the export should include data.
+ * @returns {*} a readable stream of the backup which is written in real time
+ */
+export async function streamExportApp(appId: string, excludeRows: boolean) {
+  const tmpPath = await exportApp(appId, { excludeRows, tar: true })
+  return streamFile(tmpPath)
+}

package/src/sdk/app/backups/imports.ts

@@ -0,0 +1,168 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  uploadDirectory,
+  upload,
+} from "../../../utilities/fileSystem/utilities"
+import { downloadTemplate } from "../../../utilities/fileSystem"
+import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { join } from "path"
+import fs from "fs"
+import sdk from "../../"
+import { CouchFindOptions, RowAttachment } from "@budibase/types"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+
+type TemplateType = {
+  file?: {
+    type: string
+    path: string
+  }
+  key?: string
+}
+
+async function updateAttachmentColumns(
+  prodAppId: string,
+  db: PouchDB.Database
+) {
+  // iterate through attachment documents and update them
+  const tables = await sdk.tables.getAllInternalTables(db)
+  for (let table of tables) {
+    const attachmentCols: string[] = []
+    for (let [key, column] of Object.entries(table.schema)) {
+      if (column.type === FieldTypes.ATTACHMENT) {
+        attachmentCols.push(key)
+      }
+    }
+    // no attachment columns, nothing to do
+    if (attachmentCols.length === 0) {
+      continue
+    }
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const params: CouchFindOptions = {
+      selector: {
+        _id: {
+          $regex: `^${TABLE_ROW_PREFIX}`,
+        },
+      },
+    }
+    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+    const { rows } = await dbCore.directCouchFind(db.name, params)
+    for (let row of rows) {
+      for (let column of attachmentCols) {
+        if (!Array.isArray(row[column])) {
+          continue
+        }
+        row[column] = row[column].map((attachment: RowAttachment) => {
+          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+          const urlParts = attachment.url.split("/")
+          // drop the first empty element
+          urlParts.shift()
+          // get the prefix
+          const prefix = urlParts.shift()
+          // remove the app ID
+          urlParts.shift()
+          // add new app ID
+          urlParts.unshift(prodAppId)
+          const key = urlParts.join("/")
+          return {
+            ...attachment,
+            key,
+            url: `/${prefix}/${key}`,
+          }
+        })
+      }
+    }
+    // write back the updated attachments
+    await db.bulkDocs(rows)
+  }
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns a fs read stream which can be loaded into the database.
+ */
+async function getTemplateStream(template: TemplateType) {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else if (template.key) {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+export function untarFile(file: { path: string }) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  // extract the tarball
+  tar.extract({
+    sync: true,
+    cwd: tmpPath,
+    file: file.path,
+  })
+  return tmpPath
+}
+
+export function getGlobalDBFile(tmpPath: string) {
+  return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
+}
+
+export function getListOfAppsInMulti(tmpPath: string) {
+  return fs.readdirSync(tmpPath).filter(dir => dir !== GLOBAL_DB_EXPORT_FILE)
+}
+
+export async function importApp(
+  appId: string,
+  db: PouchDB.Database,
+  template: TemplateType
+) {
+  let prodAppId = dbCore.getProdAppID(appId)
+  let dbStream: any
+  const isTar = template.file && template.file.type === "application/gzip"
+  const isDirectory =
+    template.file && fs.lstatSync(template.file.path).isDirectory()
+  if (template.file && (isTar || isDirectory)) {
+    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const contents = fs.readdirSync(tmpPath)
+    // have to handle object import
+    if (contents.length) {
+      let promises = []
+      let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
+      for (let filename of contents) {
+        const path = join(tmpPath, filename)
+        if (excludedFiles.includes(filename)) {
+          continue
+        }
+        filename = join(prodAppId, filename)
+        if (fs.lstatSync(path).isDirectory()) {
+          promises.push(
+            uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
+          )
+        } else {
+          promises.push(
+            upload({
+              bucket: ObjectStoreBuckets.APPS,
+              path,
+              filename,
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
+    }
+    dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
+  } else {
+    dbStream = await getTemplateStream(template)
+  }
+  // @ts-ignore
+  const { ok } = await db.load(dbStream)
+  if (!ok) {
+    throw "Error loading database dump from template."
+  }
+  await updateAttachmentColumns(prodAppId, db)
+  return ok
+}

package/src/sdk/app/backups/index.ts

@@ -0,0 +1,9 @@
+import * as exportApps from "./exports"
+import * as importApps from "./imports"
+import * as statistics from "./statistics"
+
+export default {
+  ...exportApps,
+  ...importApps,
+  ...statistics,
+}

package/src/sdk/app/backups/statistics.ts

@@ -0,0 +1,77 @@
+import { context, db as dbCore } from "@budibase/backend-core"
+import {
+  getDatasourceParams,
+  getTableParams,
+  getAutomationParams,
+  getScreenParams,
+} from "../../../db/utils"
+
+async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+  if (db) {
+    return cb(db)
+  } else {
+    const devAppId = dbCore.getDevAppID(appId)
+    return context.doInAppContext(devAppId, () => {
+      const db = context.getAppDB()
+      return cb(db)
+    })
+  }
+}
+
+export async function calculateDatasourceCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const datasourceList = await db.allDocs(getDatasourceParams())
+      const tableList = await db.allDocs(getTableParams())
+      return datasourceList.rows.length + tableList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateAutomationCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const automationList = await db.allDocs(getAutomationParams())
+      return automationList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateScreenCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const screenList = await db.allDocs(getScreenParams())
+      return screenList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateBackupStats(appId: string) {
+  return runInContext(appId, async (db: PouchDB.Database) => {
+    const promises = []
+    promises.push(calculateDatasourceCount(appId, db))
+    promises.push(calculateAutomationCount(appId, db))
+    promises.push(calculateScreenCount(appId, db))
+    const responses = await Promise.all(promises)
+    return {
+      datasources: responses[0],
+      automations: responses[1],
+      screens: responses[2],
+    }
+  })
+}

package/src/sdk/app/tables/index.ts

@@ -0,0 +1,60 @@
+import { getAppDB } from "@budibase/backend-core/context"
+import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import { Table } from "@budibase/types"
+
+async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
+  if (!db) {
+    db = getAppDB() as PouchDB.Database
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+  const db = getAppDB()
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return datasource.entities
+}
+
+async function getExternalTable(
+  datasourceId: any,
+  tableName: any
+): Promise<Table> {
+  const entities = await getAllExternalTables(datasourceId)
+  return entities[tableName]
+}
+
+async function getTable(tableId: any): Promise<Table> {
+  const db = getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await db.get(datasourceId)
+    const table = await getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export default {
+  getAllInternalTables,
+  getAllExternalTables,
+  getExternalTable,
+  getTable,
+}

package/src/sdk/index.ts

@@ -0,0 +1,13 @@
+import { default as backups } from "./app/backups"
+import { default as tables } from "./app/tables"
+
+const sdk = {
+  backups,
+  tables,
+}
+
+// default export for TS
+export default sdk
+
+// default export for JS
+module.exports = sdk

package/src/tests/utilities/TestConfiguration.js

@@ -25,7 +25,7 @@ const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, generateUserMetadataID } = require("../../db/utils")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -95,7 +95,10 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
-    await this.globalUser()
+    this.user = await this.globalUser()
+    this.globalUserId = this.user._id
+    this.userMetadataId = generateUserMetadataID(this.globalUserId)
+
     return this.createApp(appName)
   }
 

package/src/threads/automation.ts

@@ -1,6 +1,11 @@
 import { default as threadUtils } from "./utils"
+import { Job } from "bull"
 threadUtils.threadSetup()
-import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
+import {
+  isRecurring,
+  disableCronById,
+  isErrorInOutput,
+} from "../automations/utils"
 import { default as actions } from "../automations/actions"
 import { default as automationUtils } from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
@@ -13,7 +18,6 @@ import {
   LoopStep,
   LoopStepType,
   LoopInput,
-  AutomationEvent,
   TriggerOutput,
   AutomationContext,
   AutomationMetadata,
@@ -73,19 +77,16 @@ class Orchestrator {
   _automation: Automation
   _emitter: any
   _context: AutomationContext
-  _repeat?: { jobId: string; jobKey: string }
+  _job: Job
   executionOutput: AutomationContext
 
-  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
+  constructor(job: Job) {
+    let automation = job.data.automation,
+      triggerOutput = job.data.event
     const metadata = triggerOutput.metadata
     this._chainCount = metadata ? metadata.automationChainCount : 0
     this._appId = triggerOutput.appId as string
-    if (opts?.repeat) {
-      this._repeat = {
-        jobId: opts.repeat.jobId,
-        jobKey: opts.repeat.key,
-      }
-    }
+    this._job = job
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -134,7 +135,7 @@ class Orchestrator {
   }
 
   async stopCron(reason: string) {
-    if (!this._repeat) {
+    if (!this._job.opts.repeat) {
       return
     }
     logWarn(
@@ -142,7 +143,7 @@ class Orchestrator {
     )
     const automation = this._automation
     const trigger = automation.definition.trigger
-    await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
+    await disableCronById(this._job.id)
     this.updateExecutionOutput(
       trigger.id,
       trigger.stepId,
@@ -156,7 +157,7 @@ class Orchestrator {
   }
 
   async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this._repeat) {
+    if (!metadata.errorCount || !this._job.opts.repeat) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
@@ -475,17 +476,13 @@ class Orchestrator {
   }
 }
 
-export function execute(input: AutomationEvent, callback: WorkerCallback) {
-  const appId = input.data.event.appId
+export function execute(job: Job, callback: WorkerCallback) {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     try {
       const response = await automationOrchestrator.execute()
       callback(null, response)
@@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) {
   })
 }
 
-export const removeStalled = async (input: AutomationEvent) => {
-  const appId = input.data.event.appId
+export const removeStalled = async (job: Job) => {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     await automationOrchestrator.stopCron("stalled")
   })
 }