@budibase/server 2.1.43-alpha.2 → 2.1.43

package/dist/utilities/fileSystem/index.js

@@ -8,25 +8,21 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.deleteFolderFileSystem = exports.findFileRec = exports.getDatasourcePlugin = exports.getPluginMetadata = exports.extractTarball = exports.createTempFolder = exports.cleanup = exports.readFileSync = exports.getComponentLibraryManifest = exports.downloadTemplate = exports.deleteApp = exports.createApp = exports.sendTempFile = exports.stringToFileStream = exports.storeTempFile = exports.streamFile = exports.apiFileReturn = exports.loadHandlebarsFile = exports.init = exports.NODE_MODULES_PATH = exports.TOP_LEVEL_PATH = void 0;
-const budibaseDir_1 = require("../budibaseDir");
-const fs_1 = __importDefault(require("fs"));
-const path_1 = require("path");
-const backend_core_1 = require("@budibase/backend-core");
-const constants_1 = require("../../constants");
-const clientLibrary_1 = require("./clientLibrary");
-const __1 = require("../");
-const environment_1 = __importDefault(require("../../environment"));
-const node_fetch_1 = __importDefault(require("node-fetch"));
+const { budibaseTempDir } = require("../budibaseDir");
+const fs = require("fs");
+const { join } = require("path");
 const uuid = require("uuid/v4");
+const { ObjectStoreBuckets } = require("../../constants");
+const { upload, retrieve, retrieveToTmp, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
+const { updateClientLibrary } = require("./clientLibrary");
+const { checkSlashesInUrl } = require("../");
+const env = require("../../environment");
+const { getAppId } = require("@budibase/backend-core/context");
 const tar = require("tar");
-exports.TOP_LEVEL_PATH = (0, path_1.join)(__dirname, "..", "..", "..");
-exports.NODE_MODULES_PATH = (0, path_1.join)(exports.TOP_LEVEL_PATH, "node_modules");
-const DATASOURCE_PATH = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), "datasource");
+const fetch = require("node-fetch");
+const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
+const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -37,13 +33,13 @@ const DATASOURCE_PATH = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), "
 /**
  * Upon first startup of instance there may not be everything we need in tmp directory, set it up.
  */
-function init() {
-    const tempDir = (0, budibaseDir_1.budibaseTempDir)();
-    if (!fs_1.default.existsSync(tempDir)) {
+exports.init = () => {
+    const tempDir = budibaseTempDir();
+    if (!fs.existsSync(tempDir)) {
         // some test cases fire this quickly enough that
         // synchronous cases can end up here at the same time
         try {
-            fs_1.default.mkdirSync(tempDir);
+            fs.mkdirSync(tempDir);
         }
         catch (err) {
             if (!err || err.code !== "EEXIST") {
@@ -51,25 +47,24 @@ function init() {
             }
         }
     }
-    const clientLibPath = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), "budibase-client.js");
-    if (environment_1.default.isTest() && !fs_1.default.existsSync(clientLibPath)) {
-        fs_1.default.copyFileSync(require.resolve("@budibase/client"), clientLibPath);
+    const clientLibPath = join(budibaseTempDir(), "budibase-client.js");
+    if (env.isTest() && !fs.existsSync(clientLibPath)) {
+        fs.copyFileSync(require.resolve("@budibase/client"), clientLibPath);
     }
-}
-exports.init = init;
+};
 /**
  * Checks if the system is currently in development mode and if it is makes sure
  * everything required to function is ready.
  */
 exports.checkDevelopmentEnvironment = () => {
-    if (!environment_1.default.isDev() || environment_1.default.isTest()) {
+    if (!env.isDev() || env.isTest()) {
         return;
     }
-    if (!fs_1.default.existsSync((0, budibaseDir_1.budibaseTempDir)())) {
-        fs_1.default.mkdirSync((0, budibaseDir_1.budibaseTempDir)());
+    if (!fs.existsSync(budibaseTempDir())) {
+        fs.mkdirSync(budibaseTempDir());
     }
     let error;
-    if (!fs_1.default.existsSync((0, path_1.join)(process.cwd(), ".env"))) {
+    if (!fs.existsSync(join(process.cwd(), ".env"))) {
         error = "Must run via yarn once to generate environment.";
     }
     if (error) {
@@ -84,246 +79,215 @@ exports.checkDevelopmentEnvironment = () => {
  * @param {string} path The path to the handlebars file which is to be loaded.
  * @returns {string} The loaded handlebars file as a string - loaded as utf8.
  */
-function loadHandlebarsFile(path) {
-    return fs_1.default.readFileSync(path, "utf8");
-}
-exports.loadHandlebarsFile = loadHandlebarsFile;
+exports.loadHandlebarsFile = path => {
+    return fs.readFileSync(path, "utf8");
+};
 /**
  * When return a file from the API need to write the file to the system temporarily so we
  * can create a read stream to send.
  * @param {string} contents the contents of the file which is to be returned from the API.
- * @param {string} encoding the encoding of the file to return (utf8 default)
  * @return {Object} the read stream which can be put into the koa context body.
  */
-function apiFileReturn(contents, encoding = "utf8") {
-    const path = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), uuid());
-    fs_1.default.writeFileSync(path, contents, { encoding });
-    return fs_1.default.createReadStream(path, { encoding });
-}
-exports.apiFileReturn = apiFileReturn;
-function streamFile(path) {
-    return fs_1.default.createReadStream(path);
-}
-exports.streamFile = streamFile;
+exports.apiFileReturn = contents => {
+    const path = join(budibaseTempDir(), uuid());
+    fs.writeFileSync(path, contents);
+    return fs.createReadStream(path);
+};
+exports.streamFile = path => {
+    return fs.createReadStream(path);
+};
 /**
  * Writes the provided contents to a temporary file, which can be used briefly.
  * @param {string} fileContents contents which will be written to a temp file.
  * @return {string} the path to the temp file.
  */
-function storeTempFile(fileContents) {
-    const path = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), uuid());
-    fs_1.default.writeFileSync(path, fileContents);
+exports.storeTempFile = fileContents => {
+    const path = join(budibaseTempDir(), uuid());
+    fs.writeFileSync(path, fileContents);
     return path;
-}
-exports.storeTempFile = storeTempFile;
+};
 /**
  * Utility function for getting a file read stream - a simple in memory buffered read
  * stream doesn't work for pouchdb.
  */
-function stringToFileStream(contents) {
+exports.stringToFileStream = contents => {
     const path = exports.storeTempFile(contents);
-    return fs_1.default.createReadStream(path);
-}
-exports.stringToFileStream = stringToFileStream;
+    return fs.createReadStream(path);
+};
 /**
  * Creates a temp file and returns it from the API.
  * @param {string} fileContents the contents to be returned in file.
  */
-function sendTempFile(fileContents) {
+exports.sendTempFile = fileContents => {
     const path = exports.storeTempFile(fileContents);
-    return fs_1.default.createReadStream(path);
-}
-exports.sendTempFile = sendTempFile;
+    return fs.createReadStream(path);
+};
 /**
  * Uploads the latest client library to the object store.
  * @param {string} appId The ID of the app which is being created.
  * @return {Promise<void>} once promise completes app resources should be ready in object store.
  */
-function createApp(appId) {
-    return __awaiter(this, void 0, void 0, function* () {
-        yield (0, clientLibrary_1.updateClientLibrary)(appId);
-    });
-}
-exports.createApp = createApp;
+exports.createApp = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    yield updateClientLibrary(appId);
+});
 /**
  * Removes all of the assets created for an app in the object store.
  * @param {string} appId The ID of the app which is being deleted.
  * @return {Promise<void>} once promise completes the app resources will be removed from object store.
  */
-function deleteApp(appId) {
-    return __awaiter(this, void 0, void 0, function* () {
-        yield backend_core_1.objectStore.deleteFolder(constants_1.ObjectStoreBuckets.APPS, `${appId}/`);
-    });
-}
-exports.deleteApp = deleteApp;
+exports.deleteApp = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    yield deleteFolder(ObjectStoreBuckets.APPS, `${appId}/`);
+});
 /**
  * Retrieves a template and pipes it to minio as well as making it available temporarily.
  * @param {string} type The type of template which is to be retrieved.
  * @param name
  * @return {Promise<*>}
  */
-function downloadTemplate(type, name) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com";
-        const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`;
-        return backend_core_1.objectStore.downloadTarball(templateUrl, constants_1.ObjectStoreBuckets.TEMPLATES, type);
-    });
-}
-exports.downloadTemplate = downloadTemplate;
+exports.downloadTemplate = (type, name) => __awaiter(void 0, void 0, void 0, function* () {
+    const DEFAULT_TEMPLATES_BUCKET = "prod-budi-templates.s3-eu-west-1.amazonaws.com";
+    const templateUrl = `https://${DEFAULT_TEMPLATES_BUCKET}/templates/${type}/${name}.tar.gz`;
+    return downloadTarball(templateUrl, ObjectStoreBuckets.TEMPLATES, type);
+});
 /**
  * Retrieves component libraries from object store (or tmp symlink if in local)
  */
-function getComponentLibraryManifest(library) {
-    return __awaiter(this, void 0, void 0, function* () {
-        const appId = backend_core_1.context.getAppId();
-        const filename = "manifest.json";
-        /* istanbul ignore next */
-        // when testing in cypress and so on we need to get the package
-        // as the environment may not be fully fleshed out for dev or prod
-        if (environment_1.default.isTest()) {
-            library = library.replace("standard-components", "client");
-            const lib = library.split("/")[1];
-            const path = require.resolve(library).split(lib)[0];
-            return require((0, path_1.join)(path, lib, filename));
-        }
-        else if (environment_1.default.isDev()) {
-            const path = (0, path_1.join)(exports.NODE_MODULES_PATH, "@budibase", "client", filename);
-            // always load from new so that updates are refreshed
-            delete require.cache[require.resolve(path)];
-            return require(path);
-        }
-        if (!appId) {
-            throw new Error("No app ID found - cannot get component libraries");
-        }
-        let resp;
-        let path;
-        try {
-            // Try to load the manifest from the new file location
-            path = (0, path_1.join)(appId, filename);
-            resp = yield backend_core_1.objectStore.retrieve(constants_1.ObjectStoreBuckets.APPS, path);
-        }
-        catch (error) {
-            console.error(`component-manifest-objectstore=failed appId=${appId} path=${path}`, error);
-            // Fallback to loading it from the old location for old apps
-            path = (0, path_1.join)(appId, "node_modules", library, "package", filename);
-            resp = yield backend_core_1.objectStore.retrieve(constants_1.ObjectStoreBuckets.APPS, path);
-        }
-        if (typeof resp !== "string") {
-            resp = resp.toString("utf8");
-        }
-        return JSON.parse(resp);
-    });
-}
-exports.getComponentLibraryManifest = getComponentLibraryManifest;
+exports.getComponentLibraryManifest = (library) => __awaiter(void 0, void 0, void 0, function* () {
+    const appId = getAppId();
+    const filename = "manifest.json";
+    /* istanbul ignore next */
+    // when testing in cypress and so on we need to get the package
+    // as the environment may not be fully fleshed out for dev or prod
+    if (env.isTest()) {
+        library = library.replace("standard-components", "client");
+        const lib = library.split("/")[1];
+        const path = require.resolve(library).split(lib)[0];
+        return require(join(path, lib, filename));
+    }
+    else if (env.isDev()) {
+        const path = join(NODE_MODULES_PATH, "@budibase", "client", filename);
+        // always load from new so that updates are refreshed
+        delete require.cache[require.resolve(path)];
+        return require(path);
+    }
+    let resp;
+    let path;
+    try {
+        // Try to load the manifest from the new file location
+        path = join(appId, filename);
+        resp = yield retrieve(ObjectStoreBuckets.APPS, path);
+    }
+    catch (error) {
+        console.error(`component-manifest-objectstore=failed appId=${appId} path=${path}`, error);
+        // Fallback to loading it from the old location for old apps
+        path = join(appId, "node_modules", library, "package", filename);
+        resp = yield retrieve(ObjectStoreBuckets.APPS, path);
+    }
+    if (typeof resp !== "string") {
+        resp = resp.toString("utf8");
+    }
+    return JSON.parse(resp);
+});
 /**
  * All file reads come through here just to make sure all of them make sense
  * allows a centralised location to check logic is all good.
  */
-function readFileSync(filepath, options = "utf8") {
-    return fs_1.default.readFileSync(filepath, { encoding: options });
-}
-exports.readFileSync = readFileSync;
+exports.readFileSync = (filepath, options = "utf8") => {
+    return fs.readFileSync(filepath, options);
+};
 /**
  * Given a set of app IDs makes sure file system is cleared of any of their temp info.
  */
-function cleanup(appIds) {
+exports.cleanup = appIds => {
     for (let appId of appIds) {
-        const path = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), appId);
-        if (fs_1.default.existsSync(path)) {
-            fs_1.default.rmdirSync(path, { recursive: true });
+        const path = join(budibaseTempDir(), appId);
+        if (fs.existsSync(path)) {
+            fs.rmdirSync(path, { recursive: true });
         }
     }
-}
-exports.cleanup = cleanup;
-function createTempFolder(item) {
-    const path = (0, path_1.join)((0, budibaseDir_1.budibaseTempDir)(), item);
+};
+const createTempFolder = item => {
+    const path = join(budibaseTempDir(), item);
     try {
         // remove old tmp directories automatically - don't combine
-        if (fs_1.default.existsSync(path)) {
-            fs_1.default.rmSync(path, { recursive: true, force: true });
+        if (fs.existsSync(path)) {
+            fs.rmSync(path, { recursive: true, force: true });
         }
-        fs_1.default.mkdirSync(path);
+        fs.mkdirSync(path);
     }
     catch (err) {
         throw new Error(`Path cannot be created: ${err.message}`);
     }
     return path;
-}
+};
 exports.createTempFolder = createTempFolder;
-function extractTarball(fromFilePath, toPath) {
-    return __awaiter(this, void 0, void 0, function* () {
-        yield tar.extract({
-            file: fromFilePath,
-            C: toPath,
-        });
+const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
+    yield tar.extract({
+        file: fromFilePath,
+        C: toPath,
     });
-}
+});
 exports.extractTarball = extractTarball;
-function getPluginMetadata(path) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let metadata = {};
-        try {
-            const pkg = fs_1.default.readFileSync((0, path_1.join)(path, "package.json"), "utf8");
-            const schema = fs_1.default.readFileSync((0, path_1.join)(path, "schema.json"), "utf8");
-            metadata.schema = JSON.parse(schema);
-            metadata.package = JSON.parse(pkg);
-            if (!metadata.package.name ||
-                !metadata.package.version ||
-                !metadata.package.description) {
-                throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
-            }
-        }
-        catch (err) {
-            throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
+    let metadata = {};
+    try {
+        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
+        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
+        metadata.schema = JSON.parse(schema);
+        metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
         }
-        return { metadata, directory: path };
-    });
-}
+    }
+    catch (err) {
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
+    }
+    return { metadata, directory: path };
+});
 exports.getPluginMetadata = getPluginMetadata;
-function getDatasourcePlugin(name, url, hash) {
-    return __awaiter(this, void 0, void 0, function* () {
-        if (!fs_1.default.existsSync(DATASOURCE_PATH)) {
-            fs_1.default.mkdirSync(DATASOURCE_PATH);
-        }
-        const filename = (0, path_1.join)(DATASOURCE_PATH, name);
-        const metadataName = `${filename}.bbmetadata`;
-        if (fs_1.default.existsSync(filename)) {
-            const currentHash = fs_1.default.readFileSync(metadataName, "utf8");
-            // if hash is the same return the file, otherwise remove it and re-download
-            if (currentHash === hash) {
-                return require(filename);
-            }
-            else {
-                console.log(`Updating plugin: ${name}`);
-                delete require.cache[require.resolve(filename)];
-                fs_1.default.unlinkSync(filename);
-            }
-        }
-        const fullUrl = (0, __1.checkSlashesInUrl)(`${environment_1.default.MINIO_URL}/${constants_1.ObjectStoreBuckets.PLUGINS}/${url}`);
-        const response = yield (0, node_fetch_1.default)(fullUrl);
-        if (response.status === 200) {
-            const content = yield response.text();
-            fs_1.default.writeFileSync(filename, content);
-            fs_1.default.writeFileSync(metadataName, hash);
+exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!fs.existsSync(DATASOURCE_PATH)) {
+        fs.mkdirSync(DATASOURCE_PATH);
+    }
+    const filename = join(DATASOURCE_PATH, name);
+    const metadataName = `${filename}.bbmetadata`;
+    if (fs.existsSync(filename)) {
+        const currentHash = fs.readFileSync(metadataName, "utf8");
+        // if hash is the same return the file, otherwise remove it and re-download
+        if (currentHash === hash) {
             return require(filename);
         }
         else {
-            throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
+            console.log(`Updating plugin: ${name}`);
+            delete require.cache[require.resolve(filename)];
+            fs.unlinkSync(filename);
         }
-    });
-}
-exports.getDatasourcePlugin = getDatasourcePlugin;
+    }
+    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
+    const response = yield fetch(fullUrl);
+    if (response.status === 200) {
+        const content = yield response.text();
+        fs.writeFileSync(filename, content);
+        fs.writeFileSync(metadataName, hash);
+        return require(filename);
+    }
+    else {
+        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
+    }
+});
 /**
  * Find for a file recursively from start path applying filter, return first match
  */
-function findFileRec(startPath, filter) {
-    if (!fs_1.default.existsSync(startPath)) {
+exports.findFileRec = (startPath, filter) => {
+    if (!fs.existsSync(startPath)) {
         return;
     }
-    const files = fs_1.default.readdirSync(startPath);
+    const files = fs.readdirSync(startPath);
     for (let i = 0, len = files.length; i < len; i++) {
-        const filename = (0, path_1.join)(startPath, files[i]);
-        const stat = fs_1.default.lstatSync(filename);
+        const filename = join(startPath, files[i]);
+        const stat = fs.lstatSync(filename);
         if (stat.isDirectory()) {
             return exports.findFileRec(filename, filter);
         }
@@ -331,15 +295,23 @@ function findFileRec(startPath, filter) {
             return filename;
         }
     }
-}
-exports.findFileRec = findFileRec;
+};
 /**
  * Remove a folder which is not empty from the file system
  */
-function deleteFolderFileSystem(path) {
-    if (!fs_1.default.existsSync(path)) {
+exports.deleteFolderFileSystem = path => {
+    if (!fs.existsSync(path)) {
         return;
     }
-    fs_1.default.rmSync(path, { recursive: true, force: true });
-}
-exports.deleteFolderFileSystem = deleteFolderFileSystem;
+    fs.rmSync(path, { recursive: true, force: true });
+};
+/**
+ * Full function definition for below can be found in the utilities.
+ */
+exports.upload = upload;
+exports.retrieve = retrieve;
+exports.retrieveToTmp = retrieveToTmp;
+exports.deleteFiles = deleteFiles;
+exports.downloadTarballDirect = downloadTarballDirect;
+exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
+exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/dist/utilities/fileSystem/processor.js

@@ -8,19 +8,14 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __importDefault = (this && this.__importDefault) || function (mod) {
-    return (mod && mod.__esModule) ? mod : { "default": mod };
-};
-Object.defineProperty(exports, "__esModule", { value: true });
-exports.process = void 0;
-const jimp_1 = __importDefault(require("jimp"));
+const jimp = require("jimp");
 const FORMATS = {
     IMAGES: ["png", "jpg", "jpeg", "gif", "bmp", "tiff"],
 };
 function processImage(file) {
     // this will overwrite the temp file
-    return jimp_1.default.read(file.path).then(img => {
-        return img.resize(300, jimp_1.default.AUTO).write(file.path);
+    return jimp.read(file.path).then(img => {
+        return img.resize(300, jimp.AUTO).write(file.path);
     });
 }
 function process(file) {

package/dist/utilities/fileSystem/utilities.js

@@ -0,0 +1,22 @@
+"use strict";
+const { ObjectStore, makeSureBucketExists, upload, deleteFiles, streamUpload, retrieve, retrieveToTmp, retrieveDirectory, deleteFolder, uploadDirectory, downloadTarball, downloadTarballDirect, } = require("@budibase/backend-core/objectStore");
+/***********************************
+ *              NOTE               *
+ * This file purely exists so that *
+ *  the object store functionality *
+ *   can easily be mocked out of   *
+ *  the server without mocking the *
+ *      entire core library.       *
+ ***********************************/
+exports.ObjectStore = ObjectStore;
+exports.makeSureBucketExists = makeSureBucketExists;
+exports.upload = upload;
+exports.streamUpload = streamUpload;
+exports.retrieve = retrieve;
+exports.retrieveToTmp = retrieveToTmp;
+exports.retrieveDirectory = retrieveDirectory;
+exports.deleteFolder = deleteFolder;
+exports.uploadDirectory = uploadDirectory;
+exports.downloadTarball = downloadTarball;
+exports.downloadTarballDirect = downloadTarballDirect;
+exports.deleteFiles = deleteFiles;