@budibase/server 2.0.39 → 2.0.40-alpha.0

package/src/utilities/fileSystem/index.js

@@ -2,17 +2,11 @@ const { budibaseTempDir } = require("../budibaseDir")
 const fs = require("fs")
 const { join } = require("path")
 const uuid = require("uuid/v4")
-const {
-  doWithDB,
-  dangerousGetDB,
-  closeDB,
-} = require("@budibase/backend-core/db")
 const { ObjectStoreBuckets } = require("../../constants")
 const {
   upload,
   retrieve,
   retrieveToTmp,
-  streamUpload,
   deleteFolder,
   downloadTarball,
   downloadTarballDirect,
@@ -21,12 +15,6 @@ const {
 const { updateClientLibrary } = require("./clientLibrary")
 const { checkSlashesInUrl } = require("../")
 const env = require("../../environment")
-const {
-  USER_METDATA_PREFIX,
-  LINK_USER_METADATA_PREFIX,
-  TABLE_ROW_PREFIX,
-} = require("../../db/utils")
-const MemoryStream = require("memorystream")
 const { getAppId } = require("@budibase/backend-core/context")
 const tar = require("tar")
 const fetch = require("node-fetch")
@@ -86,21 +74,6 @@ exports.checkDevelopmentEnvironment = () => {
   }
 }
 
-/**
- * This function manages temporary template files which are stored by Koa.
- * @param {Object} template The template object retrieved from the Koa context object.
- * @returns {Object} Returns an fs read stream which can be loaded into the database.
- */
-exports.getTemplateStream = async template => {
-  if (template.file) {
-    return fs.createReadStream(template.file.path)
-  } else {
-    const [type, name] = template.key.split("/")
-    const tmpPath = await exports.downloadTemplate(type, name)
-    return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"))
-  }
-}
-
 /**
  * Used to retrieve a handlebars file from the system which will be used as a template.
  * This is allowable as the template handlebars files should be static and identical across
@@ -124,98 +97,8 @@ exports.apiFileReturn = contents => {
   return fs.createReadStream(path)
 }
 
-exports.defineFilter = excludeRows => {
-  const ids = [USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX]
-  if (excludeRows) {
-    ids.push(TABLE_ROW_PREFIX)
-  }
-  return doc =>
-    !ids.map(key => doc._id.includes(key)).reduce((prev, curr) => prev || curr)
-}
-
-/**
- * Local utility to back up the database state for an app, excluding global user
- * data or user relationships.
- * @param {string} appId The app to backup
- * @param {object} config Config to send to export DB
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} either a string or a stream of the backup
- */
-const backupAppData = async (appId, config, excludeRows) => {
-  return await exports.exportDB(appId, {
-    ...config,
-    filter: exports.defineFilter(excludeRows),
-  })
-}
-
-/**
- * Takes a copy of the database state for an app to the object store.
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {string} backupName The name of the backup located in the object store.
- * @return {*} a readable stream to the completed backup file
- */
-exports.performBackup = async (appId, backupName) => {
-  return await backupAppData(appId, { exportName: backupName })
-}
-
-/**
- * Streams a backup of the database state for an app
- * @param {string} appId The ID of the app which is to be backed up.
- * @param {boolean} excludeRows Flag to state whether the export should include data.
- * @returns {*} a readable stream of the backup which is written in real time
- */
-exports.streamBackup = async (appId, excludeRows) => {
-  return await backupAppData(appId, { stream: true }, excludeRows)
-}
-
-/**
- * Exports a DB to either file or a variable (memory).
- * @param {string} dbName the DB which is to be exported.
- * @param {string} exportName optional - provide a filename to write the backup to a file
- * @param {boolean} stream optional - whether to perform a full backup
- * @param {function} filter optional - a filter function to clear out any un-wanted docs.
- * @return {*} either a readable stream or a string
- */
-exports.exportDB = async (dbName, { stream, filter, exportName } = {}) => {
-  // streaming a DB dump is a bit more complicated, can't close DB
-  if (stream) {
-    const db = dangerousGetDB(dbName)
-    const memStream = new MemoryStream()
-    memStream.on("end", async () => {
-      await closeDB(db)
-    })
-    db.dump(memStream, { filter })
-    return memStream
-  }
-
-  return doWithDB(dbName, async db => {
-    // Write the dump to file if required
-    if (exportName) {
-      const path = join(budibaseTempDir(), exportName)
-      const writeStream = fs.createWriteStream(path)
-      await db.dump(writeStream, { filter })
-
-      // Upload the dump to the object store if self hosted
-      if (env.SELF_HOSTED) {
-        await streamUpload(
-          ObjectStoreBuckets.BACKUPS,
-          join(dbName, exportName),
-          fs.createReadStream(path)
-        )
-      }
-
-      return fs.createReadStream(path)
-    }
-
-    // Stringify the dump in memory if required
-    const memStream = new MemoryStream()
-    let appString = ""
-    memStream.on("data", chunk => {
-      appString += chunk.toString()
-    })
-    await db.dump(memStream, { filter })
-    return appString
-  })
+exports.streamFile = path => {
+  return fs.createReadStream(path)
 }
 
 /**

package/src/utilities/fileSystem/utilities.js

@@ -6,6 +6,7 @@ const {
   streamUpload,
   retrieve,
   retrieveToTmp,
+  retrieveDirectory,
   deleteFolder,
   uploadDirectory,
   downloadTarball,
@@ -27,6 +28,7 @@ exports.upload = upload
 exports.streamUpload = streamUpload
 exports.retrieve = retrieve
 exports.retrieveToTmp = retrieveToTmp
+exports.retrieveDirectory = retrieveDirectory
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball

package/src/utilities/rowProcessor/index.js

@@ -48,6 +48,11 @@ const TYPE_TRANSFORM_MAP = {
     [null]: "",
     [undefined]: undefined,
   },
+  [FieldTypes.BARCODEQR]: {
+    "": "",
+    [null]: "",
+    [undefined]: undefined,
+  },
   [FieldTypes.FORMULA]: {
     "": "",
     [null]: "",

package/src/utilities/usageQuota/rows.js

@@ -2,6 +2,7 @@ const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils")
 const {
   isDevAppID,
   getDevelopmentAppID,
+  getProdAppID,
   doWithDB,
 } = require("@budibase/backend-core/db")
 
@@ -52,7 +53,8 @@ const getAppRows = async appId => {
  * Rows duplicates may exist across apps due to data import so they are not filtered out.
  */
 exports.getUniqueRows = async appIds => {
-  let uniqueRows = []
+  let uniqueRows = [],
+    rowsByApp = {}
   const pairs = getAppPairs(appIds)
 
   for (let pair of Object.values(pairs)) {
@@ -73,8 +75,10 @@ exports.getUniqueRows = async appIds => {
     // this can't be done on all rows because app import results in
     // duplicate row ids across apps
     // the array pre-concat is important to avoid stack overflow
-    uniqueRows = uniqueRows.concat([...new Set(appRows)])
+    const prodId = getProdAppID(pair.devId || pair.prodId)
+    rowsByApp[prodId] = [...new Set(appRows)]
+    uniqueRows = uniqueRows.concat(rowsByApp[prodId])
   }
 
-  return uniqueRows
+  return { rows: uniqueRows, appRows: rowsByApp }
 }

package/coverage/lcov-report/api/controllers/backup.js.html

@@ -1,142 +0,0 @@
-
-<!doctype html>
-<html lang="en">
-
-<head>
-    <title>Code coverage report for api/controllers/backup.js</title>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="../../prettify.css" />
-    <link rel="stylesheet" href="../../base.css" />
-    <link rel="shortcut icon" type="image/x-icon" href="../../favicon.png" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style type='text/css'>
-        .coverage-summary .sorter {
-            background-image: url(../../sort-arrow-sprite.png);
-        }
-    </style>
-</head>
-    
-<body>
-<div class='wrapper'>
-    <div class='pad1'>
-        <h1><a href="../../index.html">All files</a> / <a href="index.html">api/controllers</a> backup.js</h1>
-        <div class='clearfix'>
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Statements</span>
-                <span class='fraction'>15/15</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Branches</span>
-                <span class='fraction'>0/0</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Functions</span>
-                <span class='fraction'>2/2</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">100% </span>
-                <span class="quiet">Lines</span>
-                <span class='fraction'>15/15</span>
-            </div>
-        
-            
-        </div>
-        <p class="quiet">
-            Press <em>n</em> or <em>j</em> to go to the next uncovered block, <em>b</em>, <em>p</em> or <em>k</em> for the previous block.
-        </p>
-        <template id="filterTemplate">
-            <div class="quiet">
-                Filter:
-                <input oninput="onInput()" type="search" id="fileSearch">
-            </div>
-        </template>
-    </div>
-    <div class='status-line high'></div>
-    <pre><table class="coverage">
-<tr><td class="line-count quiet"><a name='L1'></a><a href='#L1'>1</a>
-<a name='L2'></a><a href='#L2'>2</a>
-<a name='L3'></a><a href='#L3'>3</a>
-<a name='L4'></a><a href='#L4'>4</a>
-<a name='L5'></a><a href='#L5'>5</a>
-<a name='L6'></a><a href='#L6'>6</a>
-<a name='L7'></a><a href='#L7'>7</a>
-<a name='L8'></a><a href='#L8'>8</a>
-<a name='L9'></a><a href='#L9'>9</a>
-<a name='L10'></a><a href='#L10'>10</a>
-<a name='L11'></a><a href='#L11'>11</a>
-<a name='L12'></a><a href='#L12'>12</a>
-<a name='L13'></a><a href='#L13'>13</a>
-<a name='L14'></a><a href='#L14'>14</a>
-<a name='L15'></a><a href='#L15'>15</a>
-<a name='L16'></a><a href='#L16'>16</a>
-<a name='L17'></a><a href='#L17'>17</a>
-<a name='L18'></a><a href='#L18'>18</a>
-<a name='L19'></a><a href='#L19'>19</a>
-<a name='L20'></a><a href='#L20'>20</a></td><td class="line-coverage quiet"><span class="cline-any cline-yes">28x</span>
-<span class="cline-any cline-yes">28x</span>
-<span class="cline-any cline-yes">28x</span>
-<span class="cline-any cline-yes">28x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">28x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span></td><td class="text"><pre class="prettyprint lang-js">const { streamBackup } = require("../../utilities/fileSystem")
-const { events, context } = require("@budibase/backend-core")
-const { DocumentType } = require("../../db/utils")
-const { isQsTrue } = require("../../utilities")
-&nbsp;
-exports.exportAppDump = async function (ctx) {
-  let { appId, excludeRows } = ctx.query
-  const appName = decodeURI(ctx.query.appname)
-  excludeRows = isQsTrue(excludeRows)
-  const backupIdentifier = `${appName}-export-${new Date().getTime()}.txt`
-  ctx.attachment(backupIdentifier)
-  ctx.body = await streamBackup(appId, excludeRows)
-&nbsp;
-  await context.doInAppContext(appId, async () =&gt; {
-    const appDb = context.getAppDB()
-    const app = await appDb.get(DocumentType.APP_METADATA)
-    await events.app.exported(app)
-  })
-}
-&nbsp;</pre></td></tr></table></pre>
-
-                <div class='push'></div><!-- for sticky footer -->
-            </div><!-- /wrapper -->
-            <div class='footer quiet pad2 space-top1 center small'>
-                Code coverage generated by
-                <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Mon Oct 31 2022 11:02:40 GMT+0000 (Coordinated Universal Time)
-            </div>
-        <script src="../../prettify.js"></script>
-        <script>
-            window.onload = function () {
-                prettyPrint();
-            };
-        </script>
-        <script src="../../sorter.js"></script>
-        <script src="../../block-navigation.js"></script>
-    </body>
-</html>
-