@budibase/server 2.2.11-alpha.1 → 2.2.11

package/src/utilities/index.ts

@@ -1,5 +1,6 @@
 import env from "../environment"
-import { context } from "@budibase/backend-core"
+import { OBJ_STORE_DIRECTORY } from "../constants"
+import { objectStore, context } from "@budibase/backend-core"
 import { generateMetadataID } from "../db/utils"
 import { Document } from "@budibase/types"
 import stream from "stream"
@@ -31,6 +32,49 @@ export function checkSlashesInUrl(url: string) {
   return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2")
 }
 
+/**
+ * Gets the address of the object store, depending on whether self hosted or in cloud.
+ * @return {string} The base URL of the object store (MinIO or S3).
+ */
+export function objectStoreUrl() {
+  if (env.SELF_HOSTED || env.MINIO_URL) {
+    // can use a relative url for this as all goes through the proxy (this is hosted in minio)
+    return OBJ_STORE_DIRECTORY
+  } else {
+    return env.CDN_URL
+  }
+}
+
+/**
+ * In production the client library is stored in the object store, however in development
+ * we use the symlinked version produced by lerna, located in node modules. We link to this
+ * via a specific endpoint (under /api/assets/client).
+ * @param {string} appId In production we need the appId to look up the correct bucket, as the
+ * version of the client lib may differ between apps.
+ * @param {string} version The version to retrieve.
+ * @return {string} The URL to be inserted into appPackage response or server rendered
+ * app index file.
+ */
+export function clientLibraryPath(appId: string, version: string) {
+  if (env.isProd()) {
+    let url = `${objectStoreUrl()}/${objectStore.sanitizeKey(
+      appId
+    )}/budibase-client.js`
+
+    // append app version to bust the cache
+    if (version) {
+      url += `?v=${version}`
+    }
+    return url
+  } else {
+    return `/api/assets/client`
+  }
+}
+
+export function attachmentsRelativeURL(attachmentKey: string) {
+  return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`)
+}
+
 export async function updateEntityMetadata(
   type: string,
   entityId: string,

package/src/utilities/plugins.ts

@@ -0,0 +1,22 @@
+import env from "../environment"
+import { plugins as ProPlugins } from "@budibase/pro"
+import { objectStore } from "@budibase/backend-core"
+import { Plugin } from "@budibase/types"
+
+export function enrichPluginURLs(plugins: Plugin[]) {
+  if (!plugins || !plugins.length) {
+    return []
+  }
+  return plugins.map(plugin => {
+    const cloud = !env.SELF_HOSTED
+    const bucket = objectStore.ObjectStoreBuckets.PLUGINS
+    const jsFileName = "plugin.min.js"
+
+    // In self host we need to prefix the path, as the bucket name is not part
+    // of the bucket path. In cloud, it's already part of the bucket path.
+    let jsUrl = cloud ? `${env.CDN_URL}/` : `/${bucket}/`
+    jsUrl += ProPlugins.getBucketPath(plugin.name)
+    jsUrl += jsFileName
+    return { ...plugin, jsUrl }
+  })
+}

package/src/utilities/rowProcessor/index.ts

@@ -1,11 +1,12 @@
 import * as linkRows from "../../db/linkedRows"
 import { FieldTypes, AutoFieldSubTypes } from "../../constants"
+import { attachmentsRelativeURL } from "../index"
 import { processFormulas, fixAutoColumnSubType } from "./utils"
 import { ObjectStoreBuckets } from "../../constants"
 import { context, db as dbCore, objectStore } from "@budibase/backend-core"
 import { InternalTables } from "../../db/utils"
 import { TYPE_TRANSFORM_MAP } from "./map"
-import { Row, RowAttachment, Table, ContextUser } from "@budibase/types"
+import { Row, Table, ContextUser } from "@budibase/types"
 const { cloneDeep } = require("lodash/fp")
 export * from "./utils"
 
@@ -34,7 +35,7 @@ function getRemovedAttachmentKeys(
     return oldKeys
   }
   const newKeys = row[attachmentKey].map((attachment: any) => attachment.key)
-  return oldKeys.filter((key: string) => newKeys.indexOf(key) === -1)
+  return oldKeys.filter((key: any) => newKeys.indexOf(key) === -1)
 }
 
 /**
@@ -104,7 +105,7 @@ export function processAutoColumn(
  * @param {object} type The type fo coerce to
  * @returns {object} The coerced value
  */
-export function coerce(row: any, type: string) {
+export function coerce(row: any, type: any) {
   // no coercion specified for type, skip it
   if (!TYPE_TRANSFORM_MAP[type]) {
     return row
@@ -157,16 +158,6 @@ export function inputProcessing(
     else {
       clonedRow[key] = coerce(value, field.type)
     }
-
-    // remove any attachment urls, they are generated on read
-    if (field.type === FieldTypes.ATTACHMENT) {
-      const attachments = clonedRow[key]
-      if (attachments?.length) {
-        attachments.forEach((attachment: RowAttachment) => {
-          delete attachment.url
-        })
-      }
-    }
   }
 
   if (!clonedRow._id || !clonedRow._rev) {
@@ -203,15 +194,15 @@ export async function outputProcessing(
   // process formulas
   enriched = processFormulas(table, enriched, { dynamic: true }) as Row[]
 
-  // set the attachments URLs
+  // update the attachments URL depending on hosting
   for (let [property, column] of Object.entries(table.schema)) {
     if (column.type === FieldTypes.ATTACHMENT) {
       for (let row of enriched) {
         if (row[property] == null || !Array.isArray(row[property])) {
           continue
         }
-        row[property].forEach((attachment: RowAttachment) => {
-          attachment.url = objectStore.getAppFileUrl(attachment.key)
+        row[property].forEach((attachment: any) => {
+          attachment.url = attachmentsRelativeURL(attachment.key)
         })
       }
     }
@@ -274,6 +265,6 @@ export async function cleanupAttachments(
     }
   }
   if (files.length > 0) {
-    await objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
+    return objectStore.deleteFiles(ObjectStoreBuckets.APPS, files)
   }
 }

package/tsconfig.build.json

@@ -19,8 +19,6 @@
     "node_modules",
     "dist",
     "src/tests",
-    "src/api/routes/tests/utilities",
-    "src/automations/tests/utilities",
     "**/*.spec.ts",
     "**/*.spec.js"
   ]

package/coverage/lcov-report/src/utilities/fileSystem/app.ts.html

@@ -1,343 +0,0 @@
-
-<!doctype html>
-<html lang="en">
-
-<head>
-    <title>Code coverage report for src/utilities/fileSystem/app.ts</title>
-    <meta charset="utf-8" />
-    <link rel="stylesheet" href="../../../prettify.css" />
-    <link rel="stylesheet" href="../../../base.css" />
-    <link rel="shortcut icon" type="image/x-icon" href="../../../favicon.png" />
-    <meta name="viewport" content="width=device-width, initial-scale=1" />
-    <style type='text/css'>
-        .coverage-summary .sorter {
-            background-image: url(../../../sort-arrow-sprite.png);
-        }
-    </style>
-</head>
-    
-<body>
-<div class='wrapper'>
-    <div class='pad1'>
-        <h1><a href="../../../index.html">All files</a> / <a href="index.html">src/utilities/fileSystem</a> app.ts</h1>
-        <div class='clearfix'>
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">58.97% </span>
-                <span class="quiet">Statements</span>
-                <span class='fraction'>23/39</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">0% </span>
-                <span class="quiet">Branches</span>
-                <span class='fraction'>0/3</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">42.85% </span>
-                <span class="quiet">Functions</span>
-                <span class='fraction'>3/7</span>
-            </div>
-        
-            
-            <div class='fl pad1y space-right2'>
-                <span class="strong">56.25% </span>
-                <span class="quiet">Lines</span>
-                <span class='fraction'>18/32</span>
-            </div>
-        
-            
-        </div>
-        <p class="quiet">
-            Press <em>n</em> or <em>j</em> to go to the next uncovered block, <em>b</em>, <em>p</em> or <em>k</em> for the previous block.
-        </p>
-        <template id="filterTemplate">
-            <div class="quiet">
-                Filter:
-                <input oninput="onInput()" type="search" id="fileSearch">
-            </div>
-        </template>
-    </div>
-    <div class='status-line medium'></div>
-    <pre><table class="coverage">
-<tr><td class="line-count quiet"><a name='L1'></a><a href='#L1'>1</a>
-<a name='L2'></a><a href='#L2'>2</a>
-<a name='L3'></a><a href='#L3'>3</a>
-<a name='L4'></a><a href='#L4'>4</a>
-<a name='L5'></a><a href='#L5'>5</a>
-<a name='L6'></a><a href='#L6'>6</a>
-<a name='L7'></a><a href='#L7'>7</a>
-<a name='L8'></a><a href='#L8'>8</a>
-<a name='L9'></a><a href='#L9'>9</a>
-<a name='L10'></a><a href='#L10'>10</a>
-<a name='L11'></a><a href='#L11'>11</a>
-<a name='L12'></a><a href='#L12'>12</a>
-<a name='L13'></a><a href='#L13'>13</a>
-<a name='L14'></a><a href='#L14'>14</a>
-<a name='L15'></a><a href='#L15'>15</a>
-<a name='L16'></a><a href='#L16'>16</a>
-<a name='L17'></a><a href='#L17'>17</a>
-<a name='L18'></a><a href='#L18'>18</a>
-<a name='L19'></a><a href='#L19'>19</a>
-<a name='L20'></a><a href='#L20'>20</a>
-<a name='L21'></a><a href='#L21'>21</a>
-<a name='L22'></a><a href='#L22'>22</a>
-<a name='L23'></a><a href='#L23'>23</a>
-<a name='L24'></a><a href='#L24'>24</a>
-<a name='L25'></a><a href='#L25'>25</a>
-<a name='L26'></a><a href='#L26'>26</a>
-<a name='L27'></a><a href='#L27'>27</a>
-<a name='L28'></a><a href='#L28'>28</a>
-<a name='L29'></a><a href='#L29'>29</a>
-<a name='L30'></a><a href='#L30'>30</a>
-<a name='L31'></a><a href='#L31'>31</a>
-<a name='L32'></a><a href='#L32'>32</a>
-<a name='L33'></a><a href='#L33'>33</a>
-<a name='L34'></a><a href='#L34'>34</a>
-<a name='L35'></a><a href='#L35'>35</a>
-<a name='L36'></a><a href='#L36'>36</a>
-<a name='L37'></a><a href='#L37'>37</a>
-<a name='L38'></a><a href='#L38'>38</a>
-<a name='L39'></a><a href='#L39'>39</a>
-<a name='L40'></a><a href='#L40'>40</a>
-<a name='L41'></a><a href='#L41'>41</a>
-<a name='L42'></a><a href='#L42'>42</a>
-<a name='L43'></a><a href='#L43'>43</a>
-<a name='L44'></a><a href='#L44'>44</a>
-<a name='L45'></a><a href='#L45'>45</a>
-<a name='L46'></a><a href='#L46'>46</a>
-<a name='L47'></a><a href='#L47'>47</a>
-<a name='L48'></a><a href='#L48'>48</a>
-<a name='L49'></a><a href='#L49'>49</a>
-<a name='L50'></a><a href='#L50'>50</a>
-<a name='L51'></a><a href='#L51'>51</a>
-<a name='L52'></a><a href='#L52'>52</a>
-<a name='L53'></a><a href='#L53'>53</a>
-<a name='L54'></a><a href='#L54'>54</a>
-<a name='L55'></a><a href='#L55'>55</a>
-<a name='L56'></a><a href='#L56'>56</a>
-<a name='L57'></a><a href='#L57'>57</a>
-<a name='L58'></a><a href='#L58'>58</a>
-<a name='L59'></a><a href='#L59'>59</a>
-<a name='L60'></a><a href='#L60'>60</a>
-<a name='L61'></a><a href='#L61'>61</a>
-<a name='L62'></a><a href='#L62'>62</a>
-<a name='L63'></a><a href='#L63'>63</a>
-<a name='L64'></a><a href='#L64'>64</a>
-<a name='L65'></a><a href='#L65'>65</a>
-<a name='L66'></a><a href='#L66'>66</a>
-<a name='L67'></a><a href='#L67'>67</a>
-<a name='L68'></a><a href='#L68'>68</a>
-<a name='L69'></a><a href='#L69'>69</a>
-<a name='L70'></a><a href='#L70'>70</a>
-<a name='L71'></a><a href='#L71'>71</a>
-<a name='L72'></a><a href='#L72'>72</a>
-<a name='L73'></a><a href='#L73'>73</a>
-<a name='L74'></a><a href='#L74'>74</a>
-<a name='L75'></a><a href='#L75'>75</a>
-<a name='L76'></a><a href='#L76'>76</a>
-<a name='L77'></a><a href='#L77'>77</a>
-<a name='L78'></a><a href='#L78'>78</a>
-<a name='L79'></a><a href='#L79'>79</a>
-<a name='L80'></a><a href='#L80'>80</a>
-<a name='L81'></a><a href='#L81'>81</a>
-<a name='L82'></a><a href='#L82'>82</a>
-<a name='L83'></a><a href='#L83'>83</a>
-<a name='L84'></a><a href='#L84'>84</a>
-<a name='L85'></a><a href='#L85'>85</a>
-<a name='L86'></a><a href='#L86'>86</a>
-<a name='L87'></a><a href='#L87'>87</a></td><td class="line-coverage quiet"><span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-yes">1x</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-yes">58x</span>
-<span class="cline-any cline-yes">43x</span>
-<span class="cline-any cline-yes">508x</span>
-<span class="cline-any cline-yes">508x</span>
-<span class="cline-any cline-no">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span>
-<span class="cline-any cline-neutral">&nbsp;</span></td><td class="text"><pre class="prettyprint lang-js">import { budibaseTempDir } from "../budibaseDir"
-import fs from "fs"
-import { join } from "path"
-import { ObjectStoreBuckets } from "../../constants"
-import { updateClientLibrary } from "./clientLibrary"
-import env from "../../environment"
-import { objectStore, context } from "@budibase/backend-core"
-import { TOP_LEVEL_PATH } from "./filesystem"
-&nbsp;
-export const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules")
-&nbsp;
-/**
- * Uploads the latest client library to the object store.
- * @param {string} appId The ID of the app which is being created.
- * @return {Promise&lt;void&gt;} once promise completes app resources should be ready in object store.
- */
-export const createApp = <span class="fstat-no" title="function not covered" >async (a</span>ppId: string) =&gt;<span class="cstat-no" title="statement not covered" ><span class="fstat-no" title="function not covered" > {</span></span>
-<span class="cstat-no" title="statement not covered" >  await updateClientLibrary(appId)</span>
-}
-&nbsp;
-/**
- * Removes all of the assets created for an app in the object store.
- * @param {string} appId The ID of the app which is being deleted.
- * @return {Promise&lt;void&gt;} once promise completes the app resources will be removed from object store.
- */
-export const deleteApp = <span class="fstat-no" title="function not covered" >async (a</span>ppId: string) =&gt;<span class="cstat-no" title="statement not covered" ><span class="fstat-no" title="function not covered" > {</span></span>
-<span class="cstat-no" title="statement not covered" >  await objectStore.deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`)</span>
-}
-&nbsp;
-/**
- * Retrieves component libraries from object store (or tmp symlink if in local)
- */
-export const getComponentLibraryManifest = async (library: string) =&gt; {
-  const appId = context.getAppId()
-  const filename = "manifest.json"
-  /* istanbul ignore next */
-  // when testing in cypress and so on we need to get the package
-  // as the environment may not be fully fleshed out for dev or prod
-  if (env.isTest()) {
-    library = library.replace("standard-components", "client")
-    const lib = library.split("/")[1]
-    const path = require.resolve(library).split(lib)[0]
-    return require(join(path, lib, filename))
-  } else if (env.isDev()) {
-    const path = join(NODE_MODULES_PATH, "@budibase", "client", filename)
-    // always load from new so that updates are refreshed
-    delete require.cache[require.resolve(path)]
-    return require(path)
-  }
-&nbsp;
-<span class="cstat-no" title="statement not covered" >  <span class="missing-if-branch" title="if path not taken" >I</span>if (!appId) {</span>
-<span class="cstat-no" title="statement not covered" >    throw new Error("No app ID found - cannot get component libraries")</span>
-  }
-&nbsp;
-  let resp
-  let path
-<span class="cstat-no" title="statement not covered" >  try {</span>
-    // Try to load the manifest from the new file location
-<span class="cstat-no" title="statement not covered" >    path = join(appId, filename)</span>
-<span class="cstat-no" title="statement not covered" >    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)</span>
-  } catch (error) {
-<span class="cstat-no" title="statement not covered" >    console.error(</span>
-      `component-manifest-objectstore=failed appId=${appId} path=${path}`,
-      error
-    )
-    // Fallback to loading it from the old location for old apps
-<span class="cstat-no" title="statement not covered" >    path = join(appId, "node_modules", library, "package", filename)</span>
-<span class="cstat-no" title="statement not covered" >    resp = await objectStore.retrieve(ObjectStoreBuckets.APPS, path)</span>
-  }
-<span class="cstat-no" title="statement not covered" >  <span class="missing-if-branch" title="if path not taken" >I</span>if (typeof resp !== "string") {</span>
-<span class="cstat-no" title="statement not covered" >    resp = resp.toString("utf8")</span>
-  }
-<span class="cstat-no" title="statement not covered" >  return JSON.parse(resp)</span>
-}
-&nbsp;
-/**
- * Given a set of app IDs makes sure file system is cleared of any of their temp info.
- */
-export const cleanup = (appIds: string[]) =&gt; {
-  for (let appId of appIds) {
-    const path = join(budibaseTempDir(), appId)
-    <span class="missing-if-branch" title="if path not taken" >I</span>if (fs.existsSync(path)) {
-<span class="cstat-no" title="statement not covered" >      fs.rmdirSync(path, { recursive: true })</span>
-    }
-  }
-}
-&nbsp;</pre></td></tr></table></pre>
-
-                <div class='push'></div><!-- for sticky footer -->
-            </div><!-- /wrapper -->
-            <div class='footer quiet pad2 space-top1 center small'>
-                Code coverage generated by
-                <a href="https://istanbul.js.org/" target="_blank" rel="noopener noreferrer">istanbul</a>
-                at Fri Jan 06 2023 11:09:36 GMT+0000 (Coordinated Universal Time)
-            </div>
-        <script src="../../../prettify.js"></script>
-        <script>
-            window.onload = function () {
-                prettyPrint();
-            };
-        </script>
-        <script src="../../../sorter.js"></script>
-        <script src="../../../block-navigation.js"></script>
-    </body>
-</html>
-