@budibase/server 1.0.27-alpha.7 → 1.0.27-alpha.8

package/dist/utilities/fileSystem/utilities.js

@@ -1,5 +1,5 @@
 "use strict";
-const { ObjectStore, makeSureBucketExists, upload, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, deleteFolder, uploadDirectory, downloadTarball, } = require("@budibase/auth").objectStore;
 /***********************************
  *              NOTE               *
  * This file purely exists so that *
@@ -17,3 +17,4 @@ exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFolder = deleteFolder;
 exports.uploadDirectory = uploadDirectory;
 exports.downloadTarball = downloadTarball;
+exports.deleteFiles = deleteFiles;

package/dist/utilities/rowProcessor/index.js

@@ -13,6 +13,10 @@ const { cloneDeep } = require("lodash/fp");
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants");
 const { attachmentsRelativeURL } = require("../index");
 const { processFormulas } = require("./utils");
+const { deleteFiles } = require("../../utilities/fileSystem/utilities");
+const { ObjectStoreBuckets } = require("../../constants");
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db");
+const CouchDB = require("../../db");
 const BASE_AUTO_ID = 1;
 /**
  * A map of how we convert various properties in rows to each other based on the row type.
@@ -103,6 +107,22 @@ const TYPE_TRANSFORM_MAP = {
         },
     },
 };
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+    if (!oldRow[attachmentKey]) {
+        return [];
+    }
+    const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key);
+    // no attachments in new row, all removed
+    if (!row[attachmentKey]) {
+        return oldKeys;
+    }
+    const newKeys = row[attachmentKey].map(attachment => attachment.key);
+    return oldKeys.filter(key => newKeys.indexOf(key) === -1);
+}
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -253,3 +273,46 @@ exports.outputProcessing = (ctx, table, rows, opts = { squash: true }) => __awai
     }
     return wasArray ? enriched : enriched[0];
 });
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = (appId, table, { row, rows, oldRow }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!isProdAppID(appId)) {
+        const prodAppId = getDeployedAppID(appId);
+        // if prod exists, then don't allow deleting
+        const exists = yield dbExists(CouchDB, prodAppId);
+        if (exists) {
+            return;
+        }
+    }
+    let files = [];
+    function addFiles(row, key) {
+        if (row[key]) {
+            files = files.concat(row[key].map(attachment => attachment.key));
+        }
+    }
+    for (let [key, schema] of Object.entries(table.schema)) {
+        if (schema.type !== FieldTypes.ATTACHMENT) {
+            continue;
+        }
+        // if updating, need to manage the differences
+        if (oldRow && row) {
+            files = files.concat(getRemovedAttachmentKeys(oldRow, row, key));
+        }
+        else if (row) {
+            addFiles(row, key);
+        }
+        else if (rows) {
+            rows.forEach(row => addFiles(row, key));
+        }
+    }
+    if (files.length > 0) {
+        return deleteFiles(ObjectStoreBuckets.APPS, files);
+    }
+});

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.27-alpha.7",
+  "version": "1.0.27-alpha.8",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/auth": "^1.0.27-alpha.7",
-    "@budibase/client": "^1.0.27-alpha.7",
-    "@budibase/string-templates": "^1.0.27-alpha.7",
+    "@budibase/auth": "^1.0.27-alpha.8",
+    "@budibase/client": "^1.0.27-alpha.8",
+    "@budibase/string-templates": "^1.0.27-alpha.8",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -165,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "495399ad0a598dfae8355b03e012c3db27a7a548"
+  "gitHead": "f74090d2eb0a4fefcd40e99aca1c53e45063511c"
 }

package/src/api/controllers/row/internal.js

@@ -11,6 +11,7 @@ const {
   inputProcessing,
   outputProcessing,
   processAutoColumn,
+  cleanupAttachments,
 } = require("../../../utilities/rowProcessor")
 const { FieldTypes } = require("../../../constants")
 const { isEqual } = require("lodash")
@@ -25,6 +26,7 @@ const {
   getFromDesignDoc,
   getFromMemoryDoc,
 } = require("../view/utils")
+const { cloneDeep } = require("lodash/fp")
 
 const CALCULATION_TYPES = {
   SUM: "sum",
@@ -109,14 +111,14 @@ exports.patch = async ctx => {
   const inputs = ctx.request.body
   const tableId = inputs.tableId
   const isUserTable = tableId === InternalTables.USER_METADATA
-  let dbRow
+  let oldRow
   try {
-    dbRow = await db.get(inputs._id)
+    oldRow = await db.get(inputs._id)
   } catch (err) {
     if (isUserTable) {
       // don't include the rev, it'll be the global rev
       // this time
-      dbRow = {
+      oldRow = {
         _id: inputs._id,
       }
     } else {
@@ -125,13 +127,14 @@ exports.patch = async ctx => {
   }
   let dbTable = await db.get(tableId)
   // need to build up full patch fields before coerce
+  let combinedRow = cloneDeep(oldRow)
   for (let key of Object.keys(inputs)) {
     if (!dbTable.schema[key]) continue
-    dbRow[key] = inputs[key]
+    combinedRow[key] = inputs[key]
   }
 
   // this returns the table and row incase they have been updated
-  let { table, row } = inputProcessing(ctx.user, dbTable, dbRow)
+  let { table, row } = inputProcessing(ctx.user, dbTable, combinedRow)
   const validateResult = await validate({
     row,
     table,
@@ -149,6 +152,8 @@ exports.patch = async ctx => {
     tableId: row.tableId,
     table,
   })
+  // check if any attachments removed
+  await cleanupAttachments(appId, table, { oldRow, row })
 
   if (isUserTable) {
     // the row has been updated, need to put it into the ctx
@@ -295,6 +300,8 @@ exports.destroy = async function (ctx) {
     row,
     tableId: row.tableId,
   })
+  // remove any attachments that were on the row from object storage
+  await cleanupAttachments(appId, table, { row })
 
   let response
   if (ctx.params.tableId === InternalTables.USER_METADATA) {
@@ -341,6 +348,8 @@ exports.bulkDestroy = async ctx => {
   } else {
     await db.bulkDocs(rows.map(row => ({ ...row, _deleted: true })))
   }
+  // remove any attachments that were on the rows from object storage
+  await cleanupAttachments(appId, table, { rows })
   await Promise.all(updates)
   return { response: { ok: true }, rows }
 }

package/src/integrations/base/sqlTable.ts

@@ -36,7 +36,7 @@ function generateSchema(
       case FieldTypes.STRING:
       case FieldTypes.OPTIONS:
       case FieldTypes.LONGFORM:
-        schema.string(key)
+        schema.text(key)
         break
       case FieldTypes.NUMBER:
         // if meta is specified then this is a junction table entry

package/src/utilities/fileSystem/utilities.js

@@ -2,6 +2,7 @@ const {
   ObjectStore,
   makeSureBucketExists,
   upload,
+  deleteFiles,
   streamUpload,
   retrieve,
   retrieveToTmp,
@@ -28,3 +29,4 @@ exports.retrieveToTmp = retrieveToTmp
 exports.deleteFolder = deleteFolder
 exports.uploadDirectory = uploadDirectory
 exports.downloadTarball = downloadTarball
+exports.deleteFiles = deleteFiles

package/src/utilities/rowProcessor/index.js

@@ -3,6 +3,10 @@ const { cloneDeep } = require("lodash/fp")
 const { FieldTypes, AutoFieldSubTypes } = require("../../constants")
 const { attachmentsRelativeURL } = require("../index")
 const { processFormulas } = require("./utils")
+const { deleteFiles } = require("../../utilities/fileSystem/utilities")
+const { ObjectStoreBuckets } = require("../../constants")
+const { isProdAppID, getDeployedAppID, dbExists } = require("@budibase/auth/db")
+const CouchDB = require("../../db")
 
 const BASE_AUTO_ID = 1
 
@@ -95,6 +99,23 @@ const TYPE_TRANSFORM_MAP = {
   },
 }
 
+/**
+ * Given the old state of the row and the new one after an update, this will
+ * find the keys that have been removed in the updated row.
+ */
+function getRemovedAttachmentKeys(oldRow, row, attachmentKey) {
+  if (!oldRow[attachmentKey]) {
+    return []
+  }
+  const oldKeys = oldRow[attachmentKey].map(attachment => attachment.key)
+  // no attachments in new row, all removed
+  if (!row[attachmentKey]) {
+    return oldKeys
+  }
+  const newKeys = row[attachmentKey].map(attachment => attachment.key)
+  return oldKeys.filter(key => newKeys.indexOf(key) === -1)
+}
+
 /**
  * This will update any auto columns that are found on the row/table with the correct information based on
  * time now and the current logged in user making the request.
@@ -272,3 +293,45 @@ exports.outputProcessing = async (
   }
   return wasArray ? enriched : enriched[0]
 }
+
+/**
+ * Clean up any attachments that were attached to a row.
+ * @param {string} appId The ID of the app from which a row is being deleted.
+ * @param {object} table The table from which a row is being removed.
+ * @param {any} row optional - the row being removed.
+ * @param {any} rows optional - if multiple rows being deleted can do this in bulk.
+ * @param {any} oldRow optional - if updating a row this will determine the difference.
+ * @return {Promise<void>} When all attachments have been removed this will return.
+ */
+exports.cleanupAttachments = async (appId, table, { row, rows, oldRow }) => {
+  if (!isProdAppID(appId)) {
+    const prodAppId = getDeployedAppID(appId)
+    // if prod exists, then don't allow deleting
+    const exists = await dbExists(CouchDB, prodAppId)
+    if (exists) {
+      return
+    }
+  }
+  let files = []
+  function addFiles(row, key) {
+    if (row[key]) {
+      files = files.concat(row[key].map(attachment => attachment.key))
+    }
+  }
+  for (let [key, schema] of Object.entries(table.schema)) {
+    if (schema.type !== FieldTypes.ATTACHMENT) {
+      continue
+    }
+    // if updating, need to manage the differences
+    if (oldRow && row) {
+      files = files.concat(getRemovedAttachmentKeys(oldRow, row, key))
+    } else if (row) {
+      addFiles(row, key)
+    } else if (rows) {
+      rows.forEach(row => addFiles(row, key))
+    }
+  }
+  if (files.length > 0) {
+    return deleteFiles(ObjectStoreBuckets.APPS, files)
+  }
+}