@budibase/server 2.0.33 → 2.0.34-alpha.0

package/src/sdk/app/backups/imports.ts

@@ -0,0 +1,169 @@
+import { db as dbCore } from "@budibase/backend-core"
+import { TABLE_ROW_PREFIX } from "../../../db/utils"
+import { budibaseTempDir } from "../../../utilities/budibaseDir"
+import { DB_EXPORT_FILE, GLOBAL_DB_EXPORT_FILE } from "./constants"
+import {
+  uploadDirectory,
+  upload,
+} from "../../../utilities/fileSystem/utilities"
+import { downloadTemplate } from "../../../utilities/fileSystem"
+import { ObjectStoreBuckets, FieldTypes } from "../../../constants"
+import { join } from "path"
+import fs from "fs"
+import sdk from "../../"
+import { CouchFindOptions, RowAttachment } from "@budibase/types"
+const uuid = require("uuid/v4")
+const tar = require("tar")
+import PouchDB from "pouchdb"
+
+type TemplateType = {
+  file?: {
+    type: string
+    path: string
+  }
+  key?: string
+}
+
+async function updateAttachmentColumns(
+  prodAppId: string,
+  db: PouchDB.Database
+) {
+  // iterate through attachment documents and update them
+  const tables = await sdk.tables.getAllInternalTables(db)
+  for (let table of tables) {
+    const attachmentCols: string[] = []
+    for (let [key, column] of Object.entries(table.schema)) {
+      if (column.type === FieldTypes.ATTACHMENT) {
+        attachmentCols.push(key)
+      }
+    }
+    // no attachment columns, nothing to do
+    if (attachmentCols.length === 0) {
+      continue
+    }
+    // use the CouchDB Mango query API to lookup rows that have attachments
+    const params: CouchFindOptions = {
+      selector: {
+        _id: {
+          $regex: `^${TABLE_ROW_PREFIX}`,
+        },
+      },
+    }
+    attachmentCols.forEach(col => (params.selector[col] = { $exists: true }))
+    const { rows } = await dbCore.directCouchFind(db.name, params)
+    for (let row of rows) {
+      for (let column of attachmentCols) {
+        if (!Array.isArray(row[column])) {
+          continue
+        }
+        row[column] = row[column].map((attachment: RowAttachment) => {
+          // URL looks like: /prod-budi-app-assets/appId/attachments/file.csv
+          const urlParts = attachment.url.split("/")
+          // drop the first empty element
+          urlParts.shift()
+          // get the prefix
+          const prefix = urlParts.shift()
+          // remove the app ID
+          urlParts.shift()
+          // add new app ID
+          urlParts.unshift(prodAppId)
+          const key = urlParts.join("/")
+          return {
+            ...attachment,
+            key,
+            url: `/${prefix}/${key}`,
+          }
+        })
+      }
+    }
+    // write back the updated attachments
+    await db.bulkDocs(rows)
+  }
+}
+
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns a fs read stream which can be loaded into the database.
+ */
+async function getTemplateStream(template: TemplateType) {
+  if (template.file) {
+    return fs.createReadStream(template.file.path)
+  } else if (template.key) {
+    const [type, name] = template.key.split("/")
+    const tmpPath = await downloadTemplate(type, name)
+    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
+  }
+}
+
+export function untarFile(file: { path: string }) {
+  const tmpPath = join(budibaseTempDir(), uuid())
+  fs.mkdirSync(tmpPath)
+  // extract the tarball
+  tar.extract({
+    sync: true,
+    cwd: tmpPath,
+    file: file.path,
+  })
+  return tmpPath
+}
+
+export function getGlobalDBFile(tmpPath: string) {
+  return fs.readFileSync(join(tmpPath, GLOBAL_DB_EXPORT_FILE), "utf8")
+}
+
+export function getListOfAppsInMulti(tmpPath: string) {
+  return fs.readdirSync(tmpPath).filter(dir => dir !== GLOBAL_DB_EXPORT_FILE)
+}
+
+export async function importApp(
+  appId: string,
+  db: PouchDB.Database,
+  template: TemplateType
+) {
+  let prodAppId = dbCore.getProdAppID(appId)
+  let dbStream: any
+  const isTar = template.file && template.file.type === "application/gzip"
+  const isDirectory =
+    template.file && fs.lstatSync(template.file.path).isDirectory()
+  if (template.file && (isTar || isDirectory)) {
+    const tmpPath = isTar ? untarFile(template.file) : template.file.path
+    const contents = fs.readdirSync(tmpPath)
+    // have to handle object import
+    if (contents.length) {
+      let promises = []
+      let excludedFiles = [GLOBAL_DB_EXPORT_FILE, DB_EXPORT_FILE]
+      for (let filename of contents) {
+        const path = join(tmpPath, filename)
+        if (excludedFiles.includes(filename)) {
+          continue
+        }
+        filename = join(prodAppId, filename)
+        if (fs.lstatSync(path).isDirectory()) {
+          promises.push(
+            uploadDirectory(ObjectStoreBuckets.APPS, path, filename)
+          )
+        } else {
+          promises.push(
+            upload({
+              bucket: ObjectStoreBuckets.APPS,
+              path,
+              filename,
+            })
+          )
+        }
+      }
+      await Promise.all(promises)
+    }
+    dbStream = fs.createReadStream(join(tmpPath, DB_EXPORT_FILE))
+  } else {
+    dbStream = await getTemplateStream(template)
+  }
+  // @ts-ignore
+  const { ok } = await db.load(dbStream)
+  if (!ok) {
+    throw "Error loading database dump from template."
+  }
+  await updateAttachmentColumns(prodAppId, db)
+  return ok
+}

package/src/sdk/app/backups/index.ts

@@ -0,0 +1,9 @@
+import * as exportApps from "./exports"
+import * as importApps from "./imports"
+import * as statistics from "./statistics"
+
+export default {
+  ...exportApps,
+  ...importApps,
+  ...statistics,
+}

package/src/sdk/app/backups/statistics.ts

@@ -0,0 +1,78 @@
+import { context, db as dbCore } from "@budibase/backend-core"
+import {
+  getDatasourceParams,
+  getTableParams,
+  getAutomationParams,
+  getScreenParams,
+} from "../../../db/utils"
+import PouchDB from "pouchdb"
+
+async function runInContext(appId: string, cb: any, db?: PouchDB.Database) {
+  if (db) {
+    return cb(db)
+  } else {
+    const devAppId = dbCore.getDevAppID(appId)
+    return context.doInAppContext(devAppId, () => {
+      const db = context.getAppDB()
+      return cb(db)
+    })
+  }
+}
+
+export async function calculateDatasourceCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const datasourceList = await db.allDocs(getDatasourceParams())
+      const tableList = await db.allDocs(getTableParams())
+      return datasourceList.rows.length + tableList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateAutomationCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const automationList = await db.allDocs(getAutomationParams())
+      return automationList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateScreenCount(
+  appId: string,
+  db?: PouchDB.Database
+) {
+  return runInContext(
+    appId,
+    async (db: PouchDB.Database) => {
+      const screenList = await db.allDocs(getScreenParams())
+      return screenList.rows.length
+    },
+    db
+  )
+}
+
+export async function calculateBackupStats(appId: string) {
+  return runInContext(appId, async (db: PouchDB.Database) => {
+    const promises = []
+    promises.push(calculateDatasourceCount(appId, db))
+    promises.push(calculateAutomationCount(appId, db))
+    promises.push(calculateScreenCount(appId, db))
+    const responses = await Promise.all(promises)
+    return {
+      datasources: responses[0],
+      automations: responses[1],
+      screens: responses[2],
+    }
+  })
+}

package/src/sdk/app/tables/index.ts

@@ -0,0 +1,61 @@
+import { getAppDB } from "@budibase/backend-core/context"
+import { BudibaseInternalDB, getTableParams } from "../../../db/utils"
+import {
+  breakExternalTableId,
+  isExternalTable,
+  isSQL,
+} from "../../../integrations/utils"
+import { Table } from "@budibase/types"
+import PouchDB from "pouchdb"
+
+async function getAllInternalTables(db?: PouchDB.Database): Promise<Table[]> {
+  if (!db) {
+    db = getAppDB() as PouchDB.Database
+  }
+  const internalTables = await db.allDocs(
+    getTableParams(null, {
+      include_docs: true,
+    })
+  )
+  return internalTables.rows.map((tableDoc: any) => ({
+    ...tableDoc.doc,
+    type: "internal",
+    sourceId: BudibaseInternalDB._id,
+  }))
+}
+
+async function getAllExternalTables(datasourceId: any): Promise<Table[]> {
+  const db = getAppDB()
+  const datasource = await db.get(datasourceId)
+  if (!datasource || !datasource.entities) {
+    throw "Datasource is not configured fully."
+  }
+  return datasource.entities
+}
+
+async function getExternalTable(
+  datasourceId: any,
+  tableName: any
+): Promise<Table> {
+  const entities = await getAllExternalTables(datasourceId)
+  return entities[tableName]
+}
+
+async function getTable(tableId: any): Promise<Table> {
+  const db = getAppDB()
+  if (isExternalTable(tableId)) {
+    let { datasourceId, tableName } = breakExternalTableId(tableId)
+    const datasource = await db.get(datasourceId)
+    const table = await getExternalTable(datasourceId, tableName)
+    return { ...table, sql: isSQL(datasource) }
+  } else {
+    return db.get(tableId)
+  }
+}
+
+export default {
+  getAllInternalTables,
+  getAllExternalTables,
+  getExternalTable,
+  getTable,
+}

package/src/sdk/index.ts

@@ -0,0 +1,13 @@
+import { default as backups } from "./app/backups"
+import { default as tables } from "./app/tables"
+
+const sdk = {
+  backups,
+  tables,
+}
+
+// default export for TS
+export default sdk
+
+// default export for JS
+module.exports = sdk

package/src/tests/utilities/TestConfiguration.js

@@ -25,7 +25,7 @@ const newid = require("../../db/newid")
 const context = require("@budibase/backend-core/context")
 const { generateDevInfoID, SEPARATOR } = require("@budibase/backend-core/db")
 const { encrypt } = require("@budibase/backend-core/encryption")
-const { DocumentType } = require("../../db/utils")
+const { DocumentType, generateUserMetadataID } = require("../../db/utils")
 
 const GLOBAL_USER_ID = "us_uuid1"
 const EMAIL = "babs@babs.com"
@@ -95,7 +95,10 @@ class TestConfiguration {
 
   // use a new id as the name to avoid name collisions
   async init(appName = newid()) {
-    await this.globalUser()
+    this.user = await this.globalUser()
+    this.globalUserId = this.user._id
+    this.userMetadataId = generateUserMetadataID(this.globalUserId)
+
     return this.createApp(appName)
   }
 

package/src/threads/automation.ts

@@ -1,6 +1,11 @@
 import { default as threadUtils } from "./utils"
+import { Job } from "bull"
 threadUtils.threadSetup()
-import { isRecurring, disableCron, isErrorInOutput } from "../automations/utils"
+import {
+  isRecurring,
+  disableCronById,
+  isErrorInOutput,
+} from "../automations/utils"
 import { default as actions } from "../automations/actions"
 import { default as automationUtils } from "../automations/automationUtils"
 import { default as AutomationEmitter } from "../events/AutomationEmitter"
@@ -13,7 +18,6 @@ import {
   LoopStep,
   LoopStepType,
   LoopInput,
-  AutomationEvent,
   TriggerOutput,
   AutomationContext,
   AutomationMetadata,
@@ -73,19 +77,16 @@ class Orchestrator {
   _automation: Automation
   _emitter: any
   _context: AutomationContext
-  _repeat?: { jobId: string; jobKey: string }
+  _job: Job
   executionOutput: AutomationContext
 
-  constructor(automation: Automation, triggerOutput: TriggerOutput, opts: any) {
+  constructor(job: Job) {
+    let automation = job.data.automation,
+      triggerOutput = job.data.event
     const metadata = triggerOutput.metadata
     this._chainCount = metadata ? metadata.automationChainCount : 0
     this._appId = triggerOutput.appId as string
-    if (opts?.repeat) {
-      this._repeat = {
-        jobId: opts.repeat.jobId,
-        jobKey: opts.repeat.key,
-      }
-    }
+    this._job = job
     const triggerStepId = automation.definition.trigger.stepId
     triggerOutput = this.cleanupTriggerOutputs(triggerStepId, triggerOutput)
     // remove from context
@@ -134,7 +135,7 @@ class Orchestrator {
   }
 
   async stopCron(reason: string) {
-    if (!this._repeat) {
+    if (!this._job.opts.repeat) {
       return
     }
     logWarn(
@@ -142,7 +143,7 @@ class Orchestrator {
     )
     const automation = this._automation
     const trigger = automation.definition.trigger
-    await disableCron(this._repeat?.jobId, this._repeat?.jobKey)
+    await disableCronById(this._job.id)
     this.updateExecutionOutput(
       trigger.id,
       trigger.stepId,
@@ -156,7 +157,7 @@ class Orchestrator {
   }
 
   async checkIfShouldStop(metadata: AutomationMetadata): Promise<boolean> {
-    if (!metadata.errorCount || !this._repeat) {
+    if (!metadata.errorCount || !this._job.opts.repeat) {
       return false
     }
     if (metadata.errorCount >= MAX_AUTOMATION_RECURRING_ERRORS) {
@@ -475,17 +476,13 @@ class Orchestrator {
   }
 }
 
-export function execute(input: AutomationEvent, callback: WorkerCallback) {
-  const appId = input.data.event.appId
+export function execute(job: Job, callback: WorkerCallback) {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     try {
       const response = await automationOrchestrator.execute()
       callback(null, response)
@@ -495,17 +492,13 @@ export function execute(input: AutomationEvent, callback: WorkerCallback) {
   })
 }
 
-export const removeStalled = async (input: AutomationEvent) => {
-  const appId = input.data.event.appId
+export const removeStalled = async (job: Job) => {
+  const appId = job.data.event.appId
   if (!appId) {
     throw new Error("Unable to execute, event doesn't contain app ID.")
   }
   await doInAppContext(appId, async () => {
-    const automationOrchestrator = new Orchestrator(
-      input.data.automation,
-      input.data.event,
-      input.opts
-    )
+    const automationOrchestrator = new Orchestrator(job)
     await automationOrchestrator.stopCron("stalled")
   })
 }

package/src/utilities/csvParser.js

@@ -4,6 +4,7 @@ const { FieldTypes } = require("../constants")
 const VALIDATORS = {
   [FieldTypes.STRING]: () => true,
   [FieldTypes.OPTIONS]: () => true,
+  [FieldTypes.BARCODEQR]: () => true,
   [FieldTypes.NUMBER]: attribute => {
     // allow not to be present
     if (!attribute) {

package/src/utilities/fileSystem/index.js

@@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const {
-  doWithDB,
-  dangerousGetDB,
-  closeDB,
-} = require("@budibase/backend-core/db")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
   retrieve,
   retrieveToTmp,
-  streamUpload,
   deleteFolder,
   downloadTarball,
   downloadTarballDirect,
@@ -21,12 +15,6 @@ const {
 const { updateClientLibrary } = require("./clientLibrary")
 const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
-const {
-  USER_METDATA_PREFIX,
-  LINK_USER_METADATA_PREFIX,
-  TABLE_ROW_PREFIX,
-} = require("../../db/utils")
-const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
 const fetch = require("node-fetch")
@@ -86,21 +74,6 @@ exports.checkDevelopmentEnvironment = () => {
   }
 }
 
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = async template => {
-  if (template.file) {
-    return fs.createReadStream(template.file.path)
-  } else {
-    const [type, name] = template.key.split("/")
-    const tmpPath = await exports.downloadTemplate(type, name)
-    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
-  }
-}
-
 /**
  * Used to retrieve a handlebars file from the system which will be used as a template.
  * This is allowable as the template handlebars files should be static and identical across
@@ -124,98 +97,8 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
-exports.defineFilter = excludeRows => {
-  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
-  if (excludeRows) {
-    ids.push(TABLE_ROW_PREFIX)
-  }
-  return doc =>
-    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
-}
-
-/**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} either a string or a stream of the backup
- */
-const backupAppData = async (appId, config, excludeRows) => {
-  return await exports.exportDB(appId, {
-    ...config,
-    filter: exports.defineFilter(excludeRows),
-  })
-}
-
-/**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-exports.performBackup = async (appId, backupName) => {
-  return await backupAppData(appId, { exportName: backupName })
-}
-
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = async (appId, excludeRows) => {
-  return await backupAppData(appId, { stream: true }, excludeRows)
-}
-
-/**
- * Exports a DB to either file or a variable (memory).
- * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
- * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
- */
-exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  // streaming a DB dump is a bit more complicated, can't close DB
-  if (stream) {
-    const db = dangerousGetDB(dbName)
-    const memStream = new MemoryStream()
-    memStream.on("end", async () => {
-      await closeDB(db)
-    })
-    db.dump(memStream, { filter })
-    return memStream
-  }
-
-  return doWithDB(dbName, async db => {
-    // Write the dump to file if required
-    if (exportName) {
-      const path = join(budibaseTempDir(), exportName)
-      const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter })
-
-      // Upload the dump to the object store if self hosted
-      if (env.SELF_HOSTED) {
-        await streamUpload(
-          ObjectStoreBuckets.BACKUPS,
-          join(dbName, exportName),
-          fs.createReadStream(path)
-        )
-      }
-
-      return fs.createReadStream(path)
-    }
-
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream()
-    let appString = ""
-    memStream.on("data", chunk => {
-      appString += chunk.toString()
-    })
-    await db.dump(memStream, { filter })
-    return appString
-  })
+exports.streamFile = path => {
+  return fs.createReadStream(path)
 }
 
 /**

package/src/utilities/fileSystem/utilities.js

@@ -6,6 +6,7 @@ const {
   streamUpload,
   retrieve,
   retrieveToTmp,
+  retrieveDirectory,
   deleteFolder,
   uploadDirectory,
   downloadTarball,
@@ -27,6 +28,7 @@ exports.upload = upload
 exports.streamUpload = streamUpload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
+exports.retrieveDirectory = retrieveDirectory
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball

package/src/utilities/rowProcessor/index.js

@@ -48,6 +48,11 @@ const TYPE_TRANSFORM_MAP = {
     [null]: "",
     [undefined]: undefined,
   },
+  [FieldTypes.BARCODEQR]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
   [FieldTypes.FORMULA]: {
     "": "",
     [null]: "",

package/src/utilities/usageQuota/rows.js

@@ -2,6 +2,7 @@ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const {
   isDevAppID,
   getDevelopmentAppID,
+  getProdAppID,
   doWithDB,
 } = require("@budibase/backend-core/db")
 
@@ -52,7 +53,8 @@ const getAppRows = async appId => {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  let uniqueRows = []
+  let uniqueRows = [],
+    rowsByApp = {}
   const pairs = getAppPairs(appIds)
 
   for (let pair of Object.values(pairs)) {
@@ -73,8 +75,10 @@ exports.getUniqueRows = async appIds => {
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
     // the array pre-concat is important to avoid stack overflow
-    uniqueRows = uniqueRows.concat([...new Set(appRows)])
+    const prodId = getProdAppID(pair.devId || pair.prodId)
+    rowsByApp[prodId] = [...new Set(appRows)]
+    uniqueRows = uniqueRows.concat(rowsByApp[prodId])
   }
 
-  return uniqueRows
+  return { rows: uniqueRows, appRows: rowsByApp }
 }