@budibase/server 1.3.21 → 1.3.22-alpha.1

package/dist/utilities/fileSystem/index.js

@@ -14,14 +14,18 @@ const { join } = require("path");
 const uuid = require("uuid/v4");
 const { doWithDB, dangerousGetDB, closeDB, } = require("@budibase/backend-core/db");
 const { ObjectStoreBuckets } = require("../../constants");
-const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, deleteFiles, } = require("./utilities");
+const { upload, retrieve, retrieveToTmp, streamUpload, deleteFolder, downloadTarball, downloadTarballDirect, deleteFiles, } = require("./utilities");
 const { updateClientLibrary } = require("./clientLibrary");
+const { checkSlashesInUrl } = require("../");
 const env = require("../../environment");
 const { USER_METDATA_PREFIX, LINK_USER_METADATA_PREFIX, TABLE_ROW_PREFIX, } = require("../../db/utils");
 const MemoryStream = require("memorystream");
 const { getAppId } = require("@budibase/backend-core/context");
+const tar = require("tar");
+const fetch = require("node-fetch");
 const TOP_LEVEL_PATH = join(__dirname, "..", "..", "..");
 const NODE_MODULES_PATH = join(TOP_LEVEL_PATH, "node_modules");
+const DATASOURCE_PATH = join(budibaseTempDir(), "datasource");
 /**
  * The single stack system (Cloud and Builder) should not make use of the file system where possible,
  * this file handles all of the file access for the system with the intention of limiting it all to one
@@ -294,6 +298,105 @@ exports.cleanup = appIds => {
         }
     }
 };
+const createTempFolder = item => {
+    const path = join(budibaseTempDir(), item);
+    try {
+        // remove old tmp directories automatically - don't combine
+        if (fs.existsSync(path)) {
+            fs.rmSync(path, { recursive: true, force: true });
+        }
+        fs.mkdirSync(path);
+    }
+    catch (err) {
+        throw new Error(`Path cannot be created: ${err.message}`);
+    }
+    return path;
+};
+exports.createTempFolder = createTempFolder;
+const extractTarball = (fromFilePath, toPath) => __awaiter(void 0, void 0, void 0, function* () {
+    yield tar.extract({
+        file: fromFilePath,
+        C: toPath,
+    });
+});
+exports.extractTarball = extractTarball;
+const getPluginMetadata = (path) => __awaiter(void 0, void 0, void 0, function* () {
+    let metadata = {};
+    try {
+        const pkg = fs.readFileSync(join(path, "package.json"), "utf8");
+        const schema = fs.readFileSync(join(path, "schema.json"), "utf8");
+        metadata.schema = JSON.parse(schema);
+        metadata.package = JSON.parse(pkg);
+        if (!metadata.package.name ||
+            !metadata.package.version ||
+            !metadata.package.description) {
+            throw new Error("package.json is missing one of 'name', 'version' or 'description'.");
+        }
+    }
+    catch (err) {
+        throw new Error(`Unable to process schema.json/package.json in plugin. ${err.message}`);
+    }
+    return { metadata, directory: path };
+});
+exports.getPluginMetadata = getPluginMetadata;
+exports.getDatasourcePlugin = (name, url, hash) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!fs.existsSync(DATASOURCE_PATH)) {
+        fs.mkdirSync(DATASOURCE_PATH);
+    }
+    const filename = join(DATASOURCE_PATH, name);
+    const metadataName = `${filename}.bbmetadata`;
+    if (fs.existsSync(filename)) {
+        const currentHash = fs.readFileSync(metadataName, "utf8");
+        // if hash is the same return the file, otherwise remove it and re-download
+        if (currentHash === hash) {
+            return require(filename);
+        }
+        else {
+            console.log(`Updating plugin: ${name}`);
+            delete require.cache[require.resolve(filename)];
+            fs.unlinkSync(filename);
+        }
+    }
+    const fullUrl = checkSlashesInUrl(`${env.MINIO_URL}/${ObjectStoreBuckets.PLUGINS}/${url}`);
+    const response = yield fetch(fullUrl);
+    if (response.status === 200) {
+        const content = yield response.text();
+        fs.writeFileSync(filename, content);
+        fs.writeFileSync(metadataName, hash);
+        return require(filename);
+    }
+    else {
+        throw new Error(`Unable to retrieve plugin - reason: ${yield response.text()}`);
+    }
+});
+/**
+ * Find for a file recursively from start path applying filter, return first match
+ */
+exports.findFileRec = (startPath, filter) => {
+    if (!fs.existsSync(startPath)) {
+        return;
+    }
+    const files = fs.readdirSync(startPath);
+    for (let i = 0, len = files.length; i < len; i++) {
+        const filename = join(startPath, files[i]);
+        const stat = fs.lstatSync(filename);
+        if (stat.isDirectory()) {
+            return exports.findFileRec(filename, filter);
+        }
+        else if (filename.endsWith(filter)) {
+            return filename;
+        }
+    }
+};
+/**
+ * Remove a folder which is not empty from the file system
+ */
+exports.deleteFolderFileSystem = path => {
+    if (!fs.existsSync(path)) {
+        return;
+    }
+    fs.rmSync(path, { recursive: true, force: true });
+};
 /**
  * Full function definition for below can be found in the utilities.
  */
@@ -301,5 +404,6 @@ exports.upload = upload;
 exports.retrieve = retrieve;
 exports.retrieveToTmp = retrieveToTmp;
 exports.deleteFiles = deleteFiles;
+exports.downloadTarballDirect = downloadTarballDirect;
 exports.TOP_LEVEL_PATH = TOP_LEVEL_PATH;
 exports.NODE_MODULES_PATH = NODE_MODULES_PATH;

package/dist/utilities/workerRequests.js

@@ -128,3 +128,7 @@ exports.getChecklist = () => __awaiter(void 0, void 0, void 0, function* () {
     const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + "/api/global/configs/checklist"), request(null, { method: "GET" }));
     return checkResponse(response, "get checklist");
 });
+exports.generateApiKey = (userId) => __awaiter(void 0, void 0, void 0, function* () {
+    const response = yield fetch(checkSlashesInUrl(env.WORKER_URL + "/api/global/self/api_key"), request(null, { method: "POST", body: { userId } }));
+    return checkResponse(response, "generate API key");
+});

package/dist/watch.js

@@ -0,0 +1,77 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.watch = void 0;
+const path_1 = __importDefault(require("path"));
+const env = __importStar(require("./environment"));
+const chokidar_1 = __importDefault(require("chokidar"));
+const fs_1 = __importDefault(require("fs"));
+const backend_core_1 = require("@budibase/backend-core");
+const constants_1 = require("@budibase/backend-core/constants");
+const plugin_1 = require("./api/controllers/plugin");
+function watch() {
+    const watchPath = path_1.default.join(env.PLUGINS_DIR, "./**/*.tar.gz");
+    chokidar_1.default
+        .watch(watchPath, {
+        ignored: "**/node_modules",
+        awaitWriteFinish: {
+            pollInterval: 100,
+            stabilityThreshold: 250,
+        },
+        usePolling: true,
+        interval: 250,
+    })
+        .on("all", (event, path) => __awaiter(this, void 0, void 0, function* () {
+        // Sanity checks
+        if (!(path === null || path === void 0 ? void 0 : path.endsWith(".tar.gz")) || !fs_1.default.existsSync(path)) {
+            return;
+        }
+        yield backend_core_1.tenancy.doInTenant(constants_1.DEFAULT_TENANT_ID, () => __awaiter(this, void 0, void 0, function* () {
+            try {
+                const split = path.split("/");
+                const name = split[split.length - 1];
+                console.log("Importing plugin:", path);
+                yield (0, plugin_1.processUploadedPlugin)({ name, path });
+            }
+            catch (err) {
+                const message = (err === null || err === void 0 ? void 0 : err.message) ? err === null || err === void 0 ? void 0 : err.message : err;
+                console.error("Failed to import plugin:", message);
+            }
+        }));
+    }));
+}
+exports.watch = watch;

package/dist/websocket.js

@@ -0,0 +1,22 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.ClientAppSocket = exports.initialise = void 0;
+const socket_io_1 = require("socket.io");
+class Websocket {
+    constructor(server, path) {
+        this.socketServer = new socket_io_1.Server(server, {
+            path,
+        });
+    }
+    // Emit an event to all sockets
+    emit(event, payload) {
+        this.socketServer.sockets.emit(event, payload);
+    }
+}
+// Likely to be more socket instances in future
+let ClientAppSocket;
+exports.ClientAppSocket = ClientAppSocket;
+const initialise = (server) => {
+    exports.ClientAppSocket = ClientAppSocket = new Websocket(server, "/socket/client");
+};
+exports.initialise = initialise;

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.3.21",
+  "version": "1.3.22-alpha.1",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -77,11 +77,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "^1.3.21",
-    "@budibase/client": "^1.3.21",
-    "@budibase/pro": "1.3.20",
-    "@budibase/string-templates": "^1.3.21",
-    "@budibase/types": "^1.3.21",
+    "@budibase/backend-core": "1.3.22-alpha.1",
+    "@budibase/client": "1.3.22-alpha.1",
+    "@budibase/pro": "1.3.22-alpha.0",
+    "@budibase/string-templates": "1.3.22-alpha.1",
+    "@budibase/types": "1.3.22-alpha.1",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -95,6 +95,7 @@
     "bcryptjs": "2.4.3",
     "bull": "4.8.5",
     "chmodr": "1.2.0",
+    "chokidar": "3.5.3",
     "csvtojson": "2.0.10",
     "curlconverter": "3.21.0",
     "dotenv": "8.2.0",
@@ -138,8 +139,10 @@
     "redis": "4",
     "server-destroy": "1.0.1",
     "snowflake-promise": "^4.5.0",
+    "socket.io": "^4.5.1",
     "svelte": "3.49.0",
     "swagger-parser": "10.0.3",
+    "tar": "6.1.11",
     "to-json-schema": "0.2.5",
     "uuid": "3.3.2",
     "validate.js": "0.13.1",
@@ -196,5 +199,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "19ed43cb817a650c859e9d237c5053cd3ee54526"
+  "gitHead": "db762e7c2d25e6b39b73d0c99ce883bda863ed39"
 }

package/scripts/dev/manage.js

@@ -58,6 +58,7 @@ async function init() {
       DEPLOYMENT_ENVIRONMENT: "development",
       BB_ADMIN_USER_EMAIL: "",
       BB_ADMIN_USER_PASSWORD: "",
+      PLUGINS_DIR: "",
     }
     let envFile = ""
     Object.keys(envFileJson).forEach(key => {
@@ -71,6 +72,9 @@ async function up() {
   console.log("Spinning up your budibase dev environment... 🔧✨")
   await init()
   await compose.upAll(CONFIG)
+
+  // We always ensure to restart the proxy service in case of nginx conf changes
+  await compose.restartOne("proxy-service", CONFIG)
 }
 
 async function down() {

package/scripts/localdomain.js

@@ -17,6 +17,8 @@ const arg = process.argv.slice(2)[0]
  */
 updateDotEnv({
   ACCOUNT_PORTAL_URL:
-    arg === "enable" ? "http://local.com:10001" : "http://localhost:10001",
+    arg === "enable"
+      ? "http://account.local.com:10001"
+      : "http://localhost:10001",
   COOKIE_DOMAIN: arg === "enable" ? ".local.com" : "",
 }).then(() => console.log("Updated worker!"))

package/specs/openapi.json

@@ -1327,7 +1327,7 @@
             "type": "string"
           },
           "datasourceId": {
-            "description": "The ID of the data source the query belongs to.",
+            "description": "The ID of the datasource the query belongs to.",
             "type": "string"
           },
           "parameters": {
@@ -1386,7 +1386,7 @@
                   "type": "string"
                 },
                 "datasourceId": {
-                  "description": "The ID of the data source the query belongs to.",
+                  "description": "The ID of the datasource the query belongs to.",
                   "type": "string"
                 },
                 "parameters": {
@@ -1659,6 +1659,117 @@
           "data"
         ]
       },
+      "rowSearch": {
+        "type": "object",
+        "properties": {
+          "query": {
+            "type": "object",
+            "properties": {
+              "allOr": {
+                "type": "boolean",
+                "description": "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used."
+              },
+              "string": {
+                "type": "object",
+                "example": {
+                  "columnName1": "value",
+                  "columnName2": "value"
+                },
+                "description": "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
+                "additionalProperties": {
+                  "type": "string",
+                  "description": "The value to search for in the column."
+                }
+              },
+              "fuzzy": {
+                "type": "object",
+                "description": "A fuzzy search, only supported by internal tables."
+              },
+              "range": {
+                "type": "object",
+                "description": "Searches within a range, the format of this must be in the format of an object with a \"low\" and \"high\" property.",
+                "example": {
+                  "columnName1": {
+                    "low": 10,
+                    "high": 20
+                  }
+                }
+              },
+              "equal": {
+                "type": "object",
+                "description": "Searches for rows that have a column value that is exactly the value set."
+              },
+              "notEqual": {
+                "type": "object",
+                "description": "Searches for any row which does not contain the specified column value."
+              },
+              "empty": {
+                "type": "object",
+                "description": "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
+                "example": {
+                  "columnName1": ""
+                }
+              },
+              "notEmpty": {
+                "type": "object",
+                "description": "Searches for rows which have the specified column."
+              },
+              "oneOf": {
+                "type": "object",
+                "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]."
+              }
+            }
+          },
+          "paginate": {
+            "type": "boolean",
+            "description": "Enables pagination, by default this is disabled."
+          },
+          "bookmark": {
+            "oneOf": [
+              {
+                "type": "string"
+              },
+              {
+                "type": "integer"
+              }
+            ],
+            "description": "If retrieving another page, the bookmark from the previous request must be supplied."
+          },
+          "limit": {
+            "type": "integer",
+            "description": "The maximum number of rows to return, useful when paginating, for internal tables this will be limited to 1000, for SQL tables it will be 5000."
+          },
+          "sort": {
+            "type": "object",
+            "description": "A set of parameters describing the sort behaviour of the search.",
+            "properties": {
+              "order": {
+                "type": "string",
+                "enum": [
+                  "ascending",
+                  "descending"
+                ],
+                "description": "The order of the sort, by default this is ascending."
+              },
+              "column": {
+                "type": "string",
+                "description": "The name of the column by which the rows will be sorted."
+              },
+              "type": {
+                "type": "string",
+                "enum": [
+                  "string",
+                  "number"
+                ],
+                "description": "Defines whether the column should be treated as a string or as numbers when sorting."
+              }
+            }
+          }
+        },
+        "required": [
+          "query"
+        ]
+      },
       "nameSearch": {
         "type": "object",
         "properties": {
@@ -2129,115 +2240,7 @@
           "content": {
             "application/json": {
               "schema": {
-                "type": "object",
-                "required": [
-                  "query"
-                ],
-                "properties": {
-                  "query": {
-                    "type": "object",
-                    "properties": {
-                      "allOr": {
-                        "type": "boolean",
-                        "description": "Specifies that a row should be returned if it satisfies any of the specified options, rather than requiring it to fulfill all the search parameters. This defaults to false, meaning AND logic will be used."
-                      },
-                      "string": {
-                        "type": "object",
-                        "example": {
-                          "columnName1": "value",
-                          "columnName2": "value"
-                        },
-                        "description": "A map of field name to the string to search for, this will look for rows that have a value starting with the string value.",
-                        "additionalProperties": {
-                          "type": "string",
-                          "description": "The value to search for in the column."
-                        }
-                      },
-                      "fuzzy": {
-                        "type": "object",
-                        "description": "A fuzzy search, only supported by internal tables."
-                      },
-                      "range": {
-                        "type": "object",
-                        "description": "Searches within a range, the format of this must be in the format of an object with a \"low\" and \"high\" property.",
-                        "example": {
-                          "columnName1": {
-                            "low": 10,
-                            "high": 20
-                          }
-                        }
-                      },
-                      "equal": {
-                        "type": "object",
-                        "description": "Searches for rows that have a column value that is exactly the value set."
-                      },
-                      "notEqual": {
-                        "type": "object",
-                        "description": "Searches for any row which does not contain the specified column value."
-                      },
-                      "empty": {
-                        "type": "object",
-                        "description": "Searches for rows which do not contain the specified column. The object should simply contain keys of the column names, these can map to any value.",
-                        "example": {
-                          "columnName1": ""
-                        }
-                      },
-                      "notEmpty": {
-                        "type": "object",
-                        "description": "Searches for rows which have the specified column."
-                      },
-                      "oneOf": {
-                        "type": "object",
-                        "description": "Searches for rows which have a column value that is any of the specified values. The format of this must be columnName -> [value1, value2]."
-                      }
-                    }
-                  },
-                  "paginate": {
-                    "type": "boolean",
-                    "description": "Enables pagination, by default this is disabled."
-                  },
-                  "bookmark": {
-                    "oneOf": [
-                      {
-                        "type": "string"
-                      },
-                      {
-                        "type": "integer"
-                      }
-                    ],
-                    "description": "If retrieving another page, the bookmark from the previous request must be supplied."
-                  },
-                  "limit": {
-                    "type": "integer",
-                    "description": "The maximum number of rows to return, useful when paginating, for internal tables this will be limited to 1000, for SQL tables it will be 5000."
-                  },
-                  "sort": {
-                    "type": "object",
-                    "description": "A set of parameters describing the sort behaviour of the search.",
-                    "properties": {
-                      "order": {
-                        "type": "string",
-                        "enum": [
-                          "ascending",
-                          "descending"
-                        ],
-                        "description": "The order of the sort, by default this is ascending."
-                      },
-                      "column": {
-                        "type": "string",
-                        "description": "The name of the column by which the rows will be sorted."
-                      },
-                      "type": {
-                        "type": "string",
-                        "enum": [
-                          "string",
-                          "number"
-                        ],
-                        "description": "Defines whether the column should be treated as a string or as numbers when sorting."
-                      }
-                    }
-                  }
-                }
+                "$ref": "#/components/schemas/rowSearch"
               }
             }
           }
@@ -2289,7 +2292,7 @@
         },
         "responses": {
           "200": {
-            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external data sources.",
+            "description": "Returns the created table, including the ID which has been generated for it. This can be internal or external datasources.",
             "content": {
               "application/json": {
                 "schema": {