@budibase/server 2.2.3 → 2.2.4-alpha.1

package/dist/utilities/fileSystem/plugin.js

@@ -0,0 +1,63 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getDatasourcePlugin = exports.getPluginMetadata = void 0;
+const { budibaseTempDir } = require("../budibaseDir");
+const fs = require("fs");
+const { join } = require("path");
+const backend_core_1 = require("@budibase/backend-core");
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
+    let metadata = {};
+    try {
+        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
+        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
+        metadata.schema = JSON.parse(schema);
+        metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
+        }
+    }
+    catch (err) {
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
+    }
+    return { metadata, directory: path };
+});
+exports.getPluginMetadata = getPluginMetadata;
+const getDatasourcePlugin = (plugin) => __awaiter(void 0, void 0, void 0, function* () {
+    var _a;
+    const hash = (_a = plugin.schema) === null || _a === void 0 ? void 0 : _a.hash;
+    if (!fs.existsSync(DATASOURCE_PATH)) {
+        fs.mkdirSync(DATASOURCE_PATH);
+    }
+    const filename = join(DATASOURCE_PATH, plugin.name);
+    const metadataName = `${filename}.bbmetadata`;
+    if (fs.existsSync(filename)) {
+        const currentHash = fs.readFileSync(metadataName, "utf8");
+        // if hash is the same return the file, otherwise remove it and re-download
+        if (currentHash === hash) {
+            return require(filename);
+        }
+        else {
+            console.log(`Updating plugin: ${plugin.name}`);
+            delete require.cache[require.resolve(filename)];
+            fs.unlinkSync(filename);
+        }
+    }
+    const pluginKey = backend_core_1.objectStore.getPluginJSKey(plugin);
+    const pluginJs = yield backend_core_1.objectStore.retrieve(backend_core_1.objectStore.ObjectStoreBuckets.PLUGINS, pluginKey);
+    fs.writeFileSync(filename, pluginJs);
+    fs.writeFileSync(metadataName, hash);
+    return require(filename);
+});
+exports.getDatasourcePlugin = getDatasourcePlugin;

package/dist/utilities/fileSystem/template.js

@@ -0,0 +1,44 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.downloadTemplate = exports.getTemplateStream = void 0;
+const fs = require("fs");
+const { join } = require("path");
+const constants_1 = require("../../constants");
+const backend_core_1 = require("@budibase/backend-core");
+/**
+ * This function manages temporary template files which are stored by Koa.
+ * @param {Object} template The template object retrieved from the Koa context object.
+ * @returns {Object} Returns an fs read stream which can be loaded into the database.
+ */
+const getTemplateStream = (template) => __awaiter(void 0, void 0, void 0, function* () {
+    if (template.file) {
+        return fs.createReadStream(template.file.path);
+    }
+    else {
+        const [type, name] = template.key.split("/");
+        const tmpPath = yield (0, exports.downloadTemplate)(type, name);
+        return fs.createReadStream(join(tmpPath, name, "db", "dump.txt"));
+    }
+});
+exports.getTemplateStream = getTemplateStream;
+/**
+ * Retrieves a template and pipes it to minio as well as making it available temporarily.
+ * @param {string} type The type of template which is to be retrieved.
+ * @param name
+ * @return {Promise<*>}
+ */
+const downloadTemplate = (type, name) => __awaiter(void 0, void 0, void 0, function* () {
+    const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com";
+    const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`;
+    return backend_core_1.objectStore.downloadTarball(templateUrl, constants_1.ObjectStoreBuckets.TEMPLATES, type);
+});
+exports.downloadTemplate = downloadTemplate;

package/dist/utilities/index.js

@@ -12,9 +12,8 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.isQsTrue = exports.convertBookmark = exports.formatBytes = exports.stringToReadStream = exports.escapeDangerousCharacters = exports.deleteEntityMetadata = exports.saveEntityMetadata = exports.updateEntityMetadata = exports.attachmentsRelativeURL = exports.clientLibraryPath = exports.objectStoreUrl = exports.checkSlashesInUrl = exports.removeFromArray = exports.NUMBER_REGEX = exports.isDev = exports.wait = void 0;
+exports.isQsTrue = exports.convertBookmark = exports.formatBytes = exports.stringToReadStream = exports.escapeDangerousCharacters = exports.deleteEntityMetadata = exports.saveEntityMetadata = exports.updateEntityMetadata = exports.checkSlashesInUrl = exports.removeFromArray = exports.NUMBER_REGEX = exports.isDev = exports.wait = void 0;
 const environment_1 = __importDefault(require("../environment"));
-const constants_1 = require("../constants");
 const backend_core_1 = require("@budibase/backend-core");
 const utils_1 = require("../db/utils");
 const stream_1 = __importDefault(require("stream"));
@@ -43,48 +42,6 @@ function checkSlashesInUrl(url) {
     return url.replace(/(https?:\/\/)|(\/)+/g, "$1$2");
 }
 exports.checkSlashesInUrl = checkSlashesInUrl;
-/**
- * Gets the address of the object store, depending on whether self hosted or in cloud.
- * @return {string} The base URL of the object store (MinIO or S3).
- */
-function objectStoreUrl() {
-    if (environment_1.default.SELF_HOSTED || environment_1.default.MINIO_URL) {
-        // can use a relative url for this as all goes through the proxy (this is hosted in minio)
-        return constants_1.OBJ_STORE_DIRECTORY;
-    }
-    else {
-        return environment_1.default.CDN_URL;
-    }
-}
-exports.objectStoreUrl = objectStoreUrl;
-/**
- * In production the client library is stored in the object store, however in development
- * we use the symlinked version produced by lerna, located in node modules. We link to this
- * via a specific endpoint (under /api/assets/client).
- * @param {string} appId In production we need the appId to look up the correct bucket, as the
- * version of the client lib may differ between apps.
- * @param {string} version The version to retrieve.
- * @return {string} The URL to be inserted into appPackage response or server rendered
- * app index file.
- */
-function clientLibraryPath(appId, version) {
-    if (environment_1.default.isProd()) {
-        let url = `${objectStoreUrl()}/${backend_core_1.objectStore.sanitizeKey(appId)}/budibase-client.js`;
-        // append app version to bust the cache
-        if (version) {
-            url += `?v=${version}`;
-        }
-        return url;
-    }
-    else {
-        return `/api/assets/client`;
-    }
-}
-exports.clientLibraryPath = clientLibraryPath;
-function attachmentsRelativeURL(attachmentKey) {
-    return checkSlashesInUrl(`${objectStoreUrl()}/${attachmentKey}`);
-}
-exports.attachmentsRelativeURL = attachmentsRelativeURL;
 function updateEntityMetadata(type, entityId, updateFn) {
     return __awaiter(this, void 0, void 0, function* () {
         const db = backend_core_1.context.getAppDB();

package/dist/utilities/rowProcessor/index.js

@@ -38,7 +38,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 exports.cleanupAttachments = exports.outputProcessing = exports.inputProcessing = exports.coerce = exports.processAutoColumn = void 0;
 const linkRows = __importStar(require("../../db/linkedRows"));
 const constants_1 = require("../../constants");
-const index_1 = require("../index");
 const utils_1 = require("./utils");
 const constants_2 = require("../../constants");
 const backend_core_1 = require("@budibase/backend-core");
@@ -172,6 +171,15 @@ function inputProcessing(user, table, row, opts) {
         else {
             clonedRow[key] = coerce(value, field.type);
         }
+        // remove any attachment urls, they are generated on read
+        if (field.type === constants_1.FieldTypes.ATTACHMENT) {
+            const attachments = clonedRow[key];
+            if (attachments === null || attachments === void 0 ? void 0 : attachments.length) {
+                attachments.forEach((attachment) => {
+                    delete attachment.url;
+                });
+            }
+        }
     }
     if (!clonedRow._id || !clonedRow._rev) {
         clonedRow._id = row._id;
@@ -201,7 +209,7 @@ function outputProcessing(table, rows, opts = { squash: true }) {
         let enriched = yield linkRows.attachFullLinkedDocs(table, rows);
         // process formulas
         enriched = (0, utils_1.processFormulas)(table, enriched, { dynamic: true });
-        // update the attachments URL depending on hosting
+        // set the attachments URLs
         for (let [property, column] of Object.entries(table.schema)) {
             if (column.type === constants_1.FieldTypes.ATTACHMENT) {
                 for (let row of enriched) {
@@ -209,7 +217,7 @@ function outputProcessing(table, rows, opts = { squash: true }) {
                         continue;
                     }
                     row[property].forEach((attachment) => {
-                        attachment.url = (0, index_1.attachmentsRelativeURL)(attachment.key);
+                        attachment.url = backend_core_1.objectStore.getAppFileUrl(attachment.key);
                     });
                 }
             }
@@ -269,7 +277,7 @@ function cleanupAttachments(table, { row, rows, oldRow, oldTable, }) {
             }
         }
         if (files.length > 0) {
-            return backend_core_1.objectStore.deleteFiles(constants_2.ObjectStoreBuckets.APPS, files);
+            yield backend_core_1.objectStore.deleteFiles(constants_2.ObjectStoreBuckets.APPS, files);
         }
     });
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.3",
+  "version": "2.2.4-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^2.2.3",
-    "@budibase/client": "^2.2.3",
-    "@budibase/pro": "2.2.2",
-    "@budibase/string-templates": "^2.2.3",
-    "@budibase/types": "^2.2.3",
+    "@budibase/backend-core": "2.2.4-alpha.1",
+    "@budibase/client": "2.2.4-alpha.1",
+    "@budibase/pro": "2.2.3",
+    "@budibase/string-templates": "2.2.4-alpha.1",
+    "@budibase/types": "2.2.4-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -130,13 +130,15 @@
     "@types/ioredis": "4.28.10",
     "@types/jest": "27.5.1",
     "@types/koa": "2.13.4",
-    "@types/koa__router": "8.0.11",
+    "@types/koa__router": "8.0.8",
     "@types/lodash": "4.14.180",
     "@types/node": "14.18.20",
     "@types/node-fetch": "2.6.1",
     "@types/oracledb": "5.2.2",
     "@types/pouchdb": "6.4.0",
     "@types/redis": "4.0.11",
+    "@types/server-destroy": "1.0.1",
+    "@types/tar": "6.1.3",
     "@typescript-eslint/parser": "5.45.0",
     "apidoc": "0.50.4",
     "babel-jest": "27.5.1",
@@ -165,5 +167,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "1fee98510b81d27546acc7fa1dbcfb89784c8802"
+  "gitHead": "b785582a227ce0c3c8d68c37b46b47a0d43adbfe"
 }

package/src/api/controllers/application.ts

@@ -23,21 +23,18 @@ import {
   errors,
   events,
   migrations,
+  objectStore,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
-import {
-  clientLibraryPath,
-  stringToReadStream,
-  isQsTrue,
-} from "../../utilities"
+import { stringToReadStream, isQsTrue } from "../../utilities"
 import { getLocksById } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
   revertClientLibrary,
-} from "../../utilities/fileSystem/clientLibrary"
+} from "../../utilities/fileSystem"
 import { cleanupAutomations } from "../../automations/utils"
 import { checkAppMetadata } from "../../automations/logging"
 import { getUniqueRows } from "../../utilities/usageQuota/rows"
@@ -49,9 +46,9 @@ import {
   MigrationType,
   BBContext,
   Database,
+  UserCtx,
 } from "@budibase/types"
 import { BASE_LAYOUT_PROP_IDS } from "../../constants/layouts"
-import { enrichPluginURLs } from "../../utilities/plugins"
 import sdk from "../../sdk"
 
 // utility function, need to do away with this
@@ -204,27 +201,34 @@ export async function fetchAppDefinition(ctx: BBContext) {
   }
 }
 
-export async function fetchAppPackage(ctx: BBContext) {
+export async function fetchAppPackage(ctx: UserCtx) {
   const db = context.getAppDB()
   let application = await db.get(DocumentType.APP_METADATA)
   const layouts = await getLayouts()
   let screens = await getScreens()
 
   // Enrich plugin URLs
-  application.usedPlugins = enrichPluginURLs(application.usedPlugins)
+  application.usedPlugins = objectStore.enrichPluginURLs(
+    application.usedPlugins
+  )
 
   // Only filter screens if the user is not a builder
-  if (!(ctx.user?.builder && ctx.user.builder.global)) {
+  if (!(ctx.user.builder && ctx.user.builder.global)) {
     const userRoleId = getUserRoleId(ctx)
     const accessController = new roles.AccessController()
     screens = await accessController.checkScreensAccess(screens, userRoleId)
   }
 
+  const clientLibPath = objectStore.clientLibraryUrl(
+    ctx.params.appId,
+    application.version
+  )
+
   ctx.body = {
     application,
     screens,
     layouts,
-    clientLibPath: clientLibraryPath(ctx.params.appId, application.version),
+    clientLibPath,
   }
 }
 
@@ -370,7 +374,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
         if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
           // delete the app
-          // skip pre- and post-steps as no rows have been added to quotas yet
+          // skip pre and post-steps as no rows have been added to quotas yet
           ctx.params.appId = app.appId
           await destroyApp(ctx)
         }

package/src/api/controllers/auth.ts

@@ -3,7 +3,7 @@ import { InternalTables } from "../../db/utils"
 import { getFullUser } from "../../utilities/users"
 import { roles, context } from "@budibase/backend-core"
 import { groups } from "@budibase/pro"
-import { ContextUser, User, Row } from "@budibase/types"
+import { ContextUser, User, Row, UserCtx } from "@budibase/types"
 
 const PUBLIC_ROLE = roles.BUILTIN_ROLE_IDS.PUBLIC
 
@@ -16,7 +16,7 @@ const addSessionAttributesToUser = (ctx: any) => {
   }
 }
 
-export async function fetchSelf(ctx: any) {
+export async function fetchSelf(ctx: UserCtx) {
   let userId = ctx.user.userId || ctx.user._id
   /* istanbul ignore next */
   if (!userId || !ctx.isAuthenticated) {

package/src/api/controllers/cloud.ts

@@ -5,8 +5,8 @@ import { stringToReadStream } from "../../utilities"
 import { getDocParams, DocumentType, isDevAppID } from "../../db/utils"
 import { create } from "./application"
 import { join } from "path"
-import { App, BBContext, Database } from "@budibase/types"
 import sdk from "../../sdk"
+import { App, Ctx, Database } from "@budibase/types"
 
 async function createApp(appName: string, appDirectory: string) {
   const ctx = {
@@ -35,7 +35,7 @@ async function getAllDocType(db: Database, docType: string) {
   return response.rows.map(row => row.doc)
 }
 
-export async function exportApps(ctx: BBContext) {
+export async function exportApps(ctx: Ctx) {
   if (env.SELF_HOSTED || !env.MULTI_TENANCY) {
     ctx.throw(400, "Exporting only allowed in multi-tenant cloud environments.")
   }
@@ -65,13 +65,13 @@ async function checkHasBeenImported() {
   return apps.length !== 0
 }
 
-export async function hasBeenImported(ctx: BBContext) {
+export async function hasBeenImported(ctx: Ctx) {
   ctx.body = {
     imported: await checkHasBeenImported(),
   }
 }
 
-export async function importApps(ctx: BBContext) {
+export async function importApps(ctx: Ctx) {
   if (!env.SELF_HOSTED || env.MULTI_TENANCY) {
     ctx.throw(400, "Importing only allowed in self hosted environments.")
   }
@@ -82,12 +82,16 @@ export async function importApps(ctx: BBContext) {
       "Import file is required and environment must be fresh to import apps."
     )
   }
-  if (ctx.request.files.importFile.type !== "application/gzip") {
+  const file = ctx.request.files.importFile
+  if (Array.isArray(file)) {
+    ctx.throw(400, "Single file is required")
+  }
+  if (file.type !== "application/gzip") {
     ctx.throw(400, "Import file must be a gzipped tarball.")
   }
 
   // initially get all the app databases out of the tarball
-  const tmpPath = sdk.backups.untarFile(ctx.request.files.importFile)
+  const tmpPath = sdk.backups.untarFile(file)
   const globalDbImport = sdk.backups.getGlobalDBFile(tmpPath)
   const appNames = sdk.backups.getListOfAppsInMulti(tmpPath)
 

package/src/api/controllers/plugin/index.ts

@@ -1,10 +1,14 @@
 import { npmUpload, urlUpload, githubUpload, fileUpload } from "./uploaders"
-import { plugins as pluginCore, tenancy } from "@budibase/backend-core"
-import { PluginType, FileType, PluginSource } from "@budibase/types"
+import {
+  plugins as pluginCore,
+  db as dbCore,
+  tenancy,
+  objectStore,
+} from "@budibase/backend-core"
+import { PluginType, FileType, PluginSource, Plugin } from "@budibase/types"
 import env from "../../../environment"
 import { ClientAppSocket } from "../../../websocket"
-import { db as dbCore } from "@budibase/backend-core"
-import { plugins } from "@budibase/pro"
+import { sdk as pro } from "@budibase/pro"
 
 export async function getPlugins(type?: PluginType) {
   const db = tenancy.getGlobalDB()
@@ -13,9 +17,10 @@ export async function getPlugins(type?: PluginType) {
       include_docs: true,
     })
   )
-  const plugins = response.rows.map((row: any) => row.doc)
+  let plugins = response.rows.map((row: any) => row.doc) as Plugin[]
+  plugins = objectStore.enrichPluginURLs(plugins)
   if (type) {
-    return plugins.filter((plugin: any) => plugin.schema?.type === type)
+    return plugins.filter((plugin: Plugin) => plugin.schema?.type === type)
   } else {
     return plugins
   }
@@ -84,7 +89,7 @@ export async function create(ctx: any) {
       )
     }
 
-    const doc = await plugins.storePlugin(metadata, directory, source)
+    const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
     ClientAppSocket.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
@@ -107,7 +112,7 @@ export async function destroy(ctx: any) {
   const { pluginId } = ctx.params
 
   try {
-    await plugins.deletePlugin(pluginId)
+    await pro.plugins.deletePlugin(pluginId)
 
     ctx.body = { message: `Plugin ${ctx.params.pluginId} deleted.` }
   } catch (err: any) {
@@ -127,7 +132,7 @@ export async function processUploadedPlugin(
     throw new Error("Only component plugins are supported outside of self-host")
   }
 
-  const doc = await plugins.storePlugin(metadata, directory, source)
+  const doc = await pro.plugins.storePlugin(metadata, directory, source)
   ClientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
   return doc
 }

package/src/api/controllers/row/ExternalRequest.ts

@@ -21,10 +21,10 @@ import { getDatasourceAndQuery } from "./utils"
 import { FieldTypes, RelationshipTypes } from "../../../constants"
 import { breakExternalTableId, isSQL } from "../../../integrations/utils"
 import { processObjectSync } from "@budibase/string-templates"
-// @ts-ignore
 import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
+import { removeKeyNumbering } from "./utils"
 
 export interface ManyRelationship {
   tableId?: string
@@ -55,15 +55,21 @@ function buildFilters(
   let idCopy: undefined | string | any[] = cloneDeep(id)
   if (filters) {
     // need to map over the filters and make sure the _id field isn't present
-    for (let filter of Object.values(filters)) {
-      if (filter._id && primary) {
-        const parts = breakRowIdField(filter._id)
-        for (let field of primary) {
-          filter[field] = parts.shift()
+    let prefix = 1
+    for (let operator of Object.values(filters)) {
+      for (let field of Object.keys(operator || {})) {
+        if (removeKeyNumbering(field) === "_id") {
+          if (primary) {
+            const parts = breakRowIdField(operator[field])
+            for (let field of primary) {
+              operator[`${prefix}:${field}`] = parts.shift()
+            }
+            prefix++
+          }
+          // make sure this field doesn't exist on any filter
+          delete operator[field]
         }
       }
-      // make sure this field doesn't exist on any filter
-      delete filter._id
     }
   }
   // there is no id, just use the user provided filters