@budibase/server 2.0.30-alpha.7 → 2.0.31

package/dist/integrations/mongodb.js

@@ -8,13 +8,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-var __asyncValues = (this && this.__asyncValues) || function (o) {
-    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
-    var m = o[Symbol.asyncIterator], i;
-    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
-    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
-    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
-};
 Object.defineProperty(exports, "__esModule", { value: true });
 const types_1 = require("@budibase/types");
 const mongodb_1 = require("mongodb");
@@ -47,232 +40,6 @@ const SCHEMA = {
         delete: {
             type: types_1.QueryType.JSON,
         },
-        aggregate: {
-            type: types_1.QueryType.JSON,
-            steps: [
-                {
-                    key: "$addFields",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$bucket",
-                    template: `{
-  "groupBy": "",
-  "boundaries": [],
-  "default": "",
-  "output": {}
-}`,
-                },
-                {
-                    key: "$bucketAuto",
-                    template: `{
-  "groupBy": "",
-  "buckets": 1,
-  "output": {},
-  "granularity": "R5"
-}`,
-                },
-                {
-                    key: "$changeStream",
-                    template: `{
-  "allChangesForCluster": true,
-  "fullDocument": "",
-  "fullDocumentBeforeChange": "",
-  "resumeAfter": 1,
-  "showExpandedEvents": true,
-  "startAfter": {},
-  "startAtOperationTime": ""
-}`,
-                },
-                {
-                    key: "$collStats",
-                    template: `{
-  "latencyStats": { "histograms": true } },
-  "storageStats": { "scale": 1 } },
-  "count": {},
-  "queryExecStats": {}
-}`,
-                },
-                {
-                    key: "$count",
-                    template: ``,
-                },
-                {
-                    key: "$densify",
-                    template: `{
-  "field": "",
-  "partitionByFields": [],
-  "range": {
-    "step": 1,
-    "unit": 1,
-    "bounds": "full"
-  }
-}`,
-                },
-                {
-                    key: "$documents",
-                    template: `[]`,
-                },
-                {
-                    key: "$facet",
-                    template: `{\n\t\n}`,
-                },
-                {
-                    key: "$fill",
-                    template: `{
-  "partitionBy": "",
-  "partitionByFields": [],
-  "sortBy": {},
-  "output": {}
-}`,
-                },
-                {
-                    key: "$geoNear",
-                    template: `{
-  "near": { 
-    "type": "Point", 
-    "coordinates": [ 
-      -73.98142, 40.71782
-    ] 
-  },
-  "key": "location",
-  "distanceField": "dist.calculated",
-  "query": { "category": "Parks" }
-}`,
-                },
-                {
-                    key: "$graphLookup",
-                    template: `{
-  "from": "",
-  "startWith": "",
-  "connectFromField": "",
-  "connectToField": "",
-  "as": "",
-  "maxDepth": 1,
-  "depthField": "",
-  "restrictSearchWithMatch": {}
-}`,
-                },
-                {
-                    key: "$group",
-                    template: `{
-  "_id": ""
-}`,
-                },
-                {
-                    key: "$indexStats",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$limit",
-                    template: `1`,
-                },
-                {
-                    key: "$listLocalSessions",
-                    template: `{\n\t\n}`,
-                },
-                {
-                    key: "$listSessions",
-                    template: `{\n\t\n}`,
-                },
-                {
-                    key: "$lookup",
-                    template: `{
-  "from": "",
-  "localField": "",
-  "foreignField": "",
-  "as": ""
-}`,
-                },
-                {
-                    key: "$match",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$merge",
-                    template: `{
-  "into": {},
-  "on": "_id",
-  "whenMatched": "replace",
-  "whenNotMatched": "insert"
-}`,
-                },
-                {
-                    key: "$out",
-                    template: `{
-  "db": "",
-  "coll": ""
-}`,
-                },
-                {
-                    key: "$planCacheStats",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$project",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$redact",
-                    template: "",
-                },
-                {
-                    key: "$replaceRoot",
-                    template: `{ "newRoot": "" }`,
-                },
-                {
-                    key: "$replaceWith",
-                    template: ``,
-                },
-                {
-                    key: "$sample",
-                    template: `{ "size": 3 }`,
-                },
-                {
-                    key: "$set",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$setWindowFields",
-                    template: `{
-  "partitionBy": "",
-  "sortBy": {},
-  "output": {}
-}`,
-                },
-                {
-                    key: "$skip",
-                    template: `1`,
-                },
-                {
-                    key: "$sort",
-                    template: "{\n\t\n}",
-                },
-                {
-                    key: "$sortByCount",
-                    template: "",
-                },
-                {
-                    key: "$unionWith",
-                    template: `{
-  "coll": "",
-  "pipeline": []
-}`,
-                },
-                {
-                    key: "$unset",
-                    template: "",
-                },
-                {
-                    key: "$unwind",
-                    template: `{
-  "path": "",
-  "includeArrayIndex": "",
-  "preserveNullAndEmptyArrays": true
-}`,
-                },
-            ],
-        },
     },
     extra: {
         collection: {
@@ -280,8 +47,8 @@ const SCHEMA = {
             type: types_1.DatasourceFieldType.STRING,
             required: true,
         },
-        actionType: {
-            displayName: "Query Type",
+        actionTypes: {
+            displayName: "Action Types",
             type: types_1.DatasourceFieldType.LIST,
             required: true,
             data: {
@@ -289,7 +56,6 @@ const SCHEMA = {
                 create: ["insertOne", "insertMany"],
                 update: ["updateOne", "updateMany"],
                 delete: ["deleteOne", "deleteMany"],
-                aggregate: ["json", "pipeline"],
             },
         },
     },
@@ -316,7 +82,7 @@ class MongoIntegration {
                     json[field].toLowerCase().startsWith("objectid")) {
                     const id = (_a = json[field].match(/(?<=objectid\(['"]).*(?=['"]\))/gi)) === null || _a === void 0 ? void 0 : _a[0];
                     if (id) {
-                        json[field] = mongodb_1.ObjectId.createFromHexString(id);
+                        json[field] = mongodb_1.ObjectID.createFromHexString(id);
                     }
                 }
             }
@@ -379,7 +145,7 @@ class MongoIntegration {
                 let json = this.createObjectIds(query.json);
                 // For mongodb we add an extra actionType to specify
                 // which method we want to call on the collection
-                switch (query.extra.actionType) {
+                switch (query.extra.actionTypes) {
                     case "insertOne": {
                         return yield collection.insertOne(json);
                     }
@@ -387,7 +153,7 @@ class MongoIntegration {
                         return yield collection.insertMany(json);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for create`);
+                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for create`);
                     }
                 }
             }
@@ -407,7 +173,7 @@ class MongoIntegration {
                 const db = this.client.db(this.config.db);
                 const collection = db.collection(query.extra.collection);
                 let json = this.createObjectIds(query.json);
-                switch (query.extra.actionType) {
+                switch (query.extra.actionTypes) {
                     case "find": {
                         return yield collection.find(json).toArray();
                     }
@@ -428,7 +194,7 @@ class MongoIntegration {
                         return yield collection.distinct(json);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for read`);
+                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for read`);
                     }
                 }
             }
@@ -452,7 +218,7 @@ class MongoIntegration {
                     queryJson = this.parseQueryParams(queryJson, "update");
                 }
                 let json = this.createObjectIds(queryJson);
-                switch (query.extra.actionType) {
+                switch (query.extra.actionTypes) {
                     case "updateOne": {
                         return yield collection.updateOne(json.filter, json.update, json.options);
                     }
@@ -460,7 +226,7 @@ class MongoIntegration {
                         return yield collection.updateMany(json.filter, json.update, json.options);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for update`);
+                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for update`);
                     }
                 }
             }
@@ -490,7 +256,7 @@ class MongoIntegration {
                         options: {},
                     };
                 }
-                switch (query.extra.actionType) {
+                switch (query.extra.actionTypes) {
                     case "deleteOne": {
                         return yield collection.deleteOne(json.filter, json.options);
                     }
@@ -498,64 +264,9 @@ class MongoIntegration {
                         return yield collection.deleteMany(json.filter, json.options);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for delete`);
-                    }
-                }
-            }
-            catch (err) {
-                console.error("Error writing to mongodb", err);
-                throw err;
-            }
-            finally {
-                yield this.client.close();
-            }
-        });
-    }
-    aggregate(query) {
-        var e_1, _a, e_2, _b;
-        var _c;
-        return __awaiter(this, void 0, void 0, function* () {
-            try {
-                yield this.connect();
-                const db = this.client.db(this.config.db);
-                const collection = db.collection(query.extra.collection);
-                let response = [];
-                if (((_c = query.extra) === null || _c === void 0 ? void 0 : _c.actionType) === "pipeline") {
-                    try {
-                        for (var _d = __asyncValues(collection.aggregate(query.steps.map(({ key, value }) => {
-                            let temp = {};
-                            temp[key] = JSON.parse(value.value);
-                            return this.createObjectIds(temp);
-                        }))), _e; _e = yield _d.next(), !_e.done;) {
-                            const doc = _e.value;
-                            response.push(doc);
-                        }
-                    }
-                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
-                    finally {
-                        try {
-                            if (_e && !_e.done && (_a = _d.return)) yield _a.call(_d);
-                        }
-                        finally { if (e_1) throw e_1.error; }
-                    }
-                }
-                else {
-                    const stages = query.json;
-                    try {
-                        for (var _f = __asyncValues(collection.aggregate(stages ? this.createObjectIds(stages) : [])), _g; _g = yield _f.next(), !_g.done;) {
-                            const doc = _g.value;
-                            response.push(doc);
-                        }
-                    }
-                    catch (e_2_1) { e_2 = { error: e_2_1 }; }
-                    finally {
-                        try {
-                            if (_g && !_g.done && (_b = _f.return)) yield _b.call(_f);
-                        }
-                        finally { if (e_2) throw e_2.error; }
+                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for delete`);
                     }
                 }
-                return response;
             }
             catch (err) {
                 console.error("Error writing to mongodb", err);

package/dist/integrations/rest.js

@@ -1,27 +1,4 @@
 "use strict";
-var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    var desc = Object.getOwnPropertyDescriptor(m, k);
-    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
-      desc = { enumerable: true, get: function() { return m[k]; } };
-    }
-    Object.defineProperty(o, k2, desc);
-}) : (function(o, m, k, k2) {
-    if (k2 === undefined) k2 = k;
-    o[k2] = m[k];
-}));
-var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
-    Object.defineProperty(o, "default", { enumerable: true, value: v });
-}) : function(o, v) {
-    o["default"] = v;
-});
-var __importStar = (this && this.__importStar) || function (mod) {
-    if (mod && mod.__esModule) return mod;
-    var result = {};
-    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
-    __setModuleDefault(result, mod);
-    return result;
-};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -38,7 +15,6 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const types_1 = require("@budibase/types");
 const datasource_1 = require("../definitions/datasource");
 const lodash_1 = require("lodash");
-const https = __importStar(require("https"));
 const querystring_1 = __importDefault(require("querystring"));
 const fetch = require("node-fetch");
 const { formatBytes } = require("../utilities");
@@ -96,11 +72,11 @@ const SCHEMA = {
             required: false,
             default: {},
         },
-        rejectUnauthorized: {
-            display: "Reject Unauthorized",
+        legacyHttpParser: {
+            display: "Legacy HTTP Support",
             type: types_1.DatasourceFieldType.BOOLEAN,
-            default: true,
             required: false,
+            default: false,
         },
     },
     query: {
@@ -221,11 +197,8 @@ class RestIntegration {
                 queryString = `${paginationString}&${queryString}`;
             }
         }
-        if (queryString) {
-            // make sure the query string is fully encoded
-            queryString = "?" + querystring_1.default.encode(querystring_1.default.decode(queryString));
-        }
-        const main = `${path}${queryString}`;
+        // make sure the query string is fully encoded
+        const main = `${path}?${querystring_1.default.encode(querystring_1.default.decode(queryString))}`;
         let complete = main;
         if (this.config.url && !main.startsWith("http")) {
             complete = !this.config.url ? main : `${this.config.url}/${main}`;
@@ -345,12 +318,6 @@ class RestIntegration {
             }
             let input = { method, headers: this.headers };
             input = this.addBody(bodyType, requestBody, input, pagination, paginationValues);
-            if (this.config.rejectUnauthorized == false) {
-                input.agent = new https.Agent({
-                    rejectUnauthorized: false,
-                });
-            }
-            // Deprecated by rejectUnauthorized
             if (this.config.legacyHttpParser) {
                 // https://github.com/nodejs/node/issues/43798
                 input.extraHttpOptions = { insecureHTTPParser: true };

package/dist/migrations/functions/{usageQuotas/syncPlugins.js → pluginCount.js}

@@ -14,7 +14,9 @@ const backend_core_1 = require("@budibase/backend-core");
 const pro_1 = require("@budibase/pro");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        yield pro_1.plugins.checkPluginQuotas();
+        yield backend_core_1.tenancy.doInTenant(backend_core_1.tenancy.DEFAULT_TENANT_ID, () => __awaiter(void 0, void 0, void 0, function* () {
+            yield pro_1.plugins.checkPluginQuotas();
+        }));
     }
     catch (err) {
         backend_core_1.logging.logAlert("Failed to update plugin quotas", err);

package/dist/migrations/functions/{syncQuotas.js → quotas1.js}

@@ -36,15 +36,19 @@ exports.run = void 0;
 const usageQuotas_1 = require("./usageQuotas");
 const syncApps = __importStar(require("./usageQuotas/syncApps"));
 const syncRows = __importStar(require("./usageQuotas/syncRows"));
-const syncPlugins = __importStar(require("./usageQuotas/syncPlugins"));
 /**
- * Synchronise quotas to the state of the db.
+ * Date:
+ * January 2022
+ *
+ * Description:
+ * Synchronise the app and row quotas to the state of the db after it was
+ * discovered that the quota resets were still in place and the row quotas
+ * weren't being decremented correctly.
  */
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     yield (0, usageQuotas_1.runQuotaMigration)(() => __awaiter(void 0, void 0, void 0, function* () {
         yield syncApps.run();
         yield syncRows.run();
-        yield syncPlugins.run();
     }));
 });
 exports.run = run;

package/dist/migrations/functions/usageQuotas/syncApps.js

@@ -16,6 +16,7 @@ const pro_1 = require("@budibase/pro");
 const types_1 = require("@budibase/types");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     // get app count
+    // @ts-ignore
     const devApps = yield (0, db_1.getAllApps)({ dev: true });
     const appCount = devApps ? devApps.length : 0;
     // sync app count

package/dist/migrations/functions/usageQuotas/syncRows.js

@@ -17,19 +17,15 @@ const pro_1 = require("@budibase/pro");
 const types_1 = require("@budibase/types");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     // get all rows in all apps
+    // @ts-ignore
     const allApps = yield (0, db_1.getAllApps)({ all: true });
+    // @ts-ignore
     const appIds = allApps ? allApps.map((app) => app.appId) : [];
-    const { appRows } = yield (0, rows_1.getUniqueRows)(appIds);
-    // get the counts per app
-    const counts = {};
-    let rowCount = 0;
-    Object.entries(appRows).forEach(([appId, rows]) => {
-        counts[appId] = rows.length;
-        rowCount += rows.length;
-    });
+    const rows = yield (0, rows_1.getUniqueRows)(appIds);
+    const rowCount = rows ? rows.length : 0;
     // sync row count
     const tenantId = (0, tenancy_1.getTenantId)();
     console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`);
-    yield pro_1.quotas.setUsagePerApp(counts, types_1.StaticQuotaName.ROWS, types_1.QuotaUsageType.STATIC);
+    yield pro_1.quotas.setUsage(rowCount, types_1.StaticQuotaName.ROWS, types_1.QuotaUsageType.STATIC);
 });
 exports.run = run;

package/dist/migrations/index.js

@@ -41,9 +41,10 @@ const types_1 = require("@budibase/types");
 const environment_1 = __importDefault(require("../environment"));
 // migration functions
 const userEmailViewCasing = __importStar(require("./functions/userEmailViewCasing"));
-const syncQuotas = __importStar(require("./functions/syncQuotas"));
+const quota1 = __importStar(require("./functions/quotas1"));
 const appUrls = __importStar(require("./functions/appUrls"));
 const backfill = __importStar(require("./functions/backfill"));
+const pluginCount = __importStar(require("./functions/pluginCount"));
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -57,8 +58,8 @@ const buildMigrations = () => {
                 serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: userEmailViewCasing.run }));
                 break;
             }
-            case types_1.MigrationName.SYNC_QUOTAS: {
-                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: syncQuotas.run }));
+            case types_1.MigrationName.QUOTAS_1: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: quota1.run }));
                 break;
             }
             case types_1.MigrationName.APP_URLS: {
@@ -77,6 +78,11 @@ const buildMigrations = () => {
                 serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: backfill.installation.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: !!environment_1.default.SELF_HOSTED }));
                 break;
             }
+            case types_1.MigrationName.PLUGIN_COUNT: {
+                if (environment_1.default.SELF_HOSTED) {
+                    serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: pluginCount.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: false }));
+                }
+            }
         }
     }
     return serverMigrations;
@@ -95,11 +101,34 @@ const migrate = (options) => __awaiter(void 0, void 0, void 0, function* () {
 });
 exports.migrate = migrate;
 const migrateWithLock = (options) => __awaiter(void 0, void 0, void 0, function* () {
-    yield backend_core_1.locks.doWithLock({
-        type: types_1.LockType.TRY_ONCE,
-        name: types_1.LockName.MIGRATIONS,
-        ttl: 1000 * 60 * 15, // auto expire the migration lock after 15 minutes
-    }, () => __awaiter(void 0, void 0, void 0, function* () {
+    // get a new lock client
+    const redlock = yield backend_core_1.redis.clients.getMigrationsRedlock();
+    // lock for 15 minutes
+    const ttl = 1000 * 60 * 15;
+    let migrationLock;
+    // acquire lock
+    try {
+        migrationLock = yield redlock.lock("migrations", ttl);
+    }
+    catch (e) {
+        if (e.name === "LockError") {
+            return;
+        }
+        else {
+            throw e;
+        }
+    }
+    // run migrations
+    try {
         yield backend_core_1.migrations.runMigrations(exports.MIGRATIONS, options);
-    }));
+    }
+    finally {
+        // release lock
+        try {
+            yield migrationLock.unlock();
+        }
+        catch (e) {
+            console.error("unable to release migration lock");
+        }
+    }
 });

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "2.0.30-alpha.6",
+    "version": "2.0.30",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -77,11 +77,11 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "10.0.3",
-        "@budibase/backend-core": "2.0.30-alpha.6",
-        "@budibase/client": "2.0.30-alpha.6",
-        "@budibase/pro": "2.0.30-alpha.6",
-        "@budibase/string-templates": "2.0.30-alpha.6",
-        "@budibase/types": "2.0.30-alpha.6",
+        "@budibase/backend-core": "^2.0.30",
+        "@budibase/client": "^2.0.30",
+        "@budibase/pro": "2.0.30",
+        "@budibase/string-templates": "^2.0.30",
+        "@budibase/types": "^2.0.30",
         "@bull-board/api": "3.7.0",
         "@bull-board/koa": "3.9.4",
         "@elastic/elasticsearch": "7.10.0",
@@ -123,7 +123,7 @@
         "koa2-ratelimit": "1.1.1",
         "lodash": "4.17.21",
         "memorystream": "0.3.1",
-        "mongodb": "4.9",
+        "mongodb": "3.6.3",
         "mssql": "6.2.3",
         "mysql2": "2.3.3",
         "node-fetch": "2.6.7",
@@ -146,7 +146,7 @@
         "to-json-schema": "0.2.5",
         "uuid": "3.3.2",
         "validate.js": "0.13.1",
-        "vm2": "3.9.11",
+        "vm2": "3.9.6",
         "worker-farm": "1.7.0",
         "xml2js": "0.4.23",
         "yargs": "13.2.4",
@@ -166,6 +166,7 @@
         "@types/koa": "2.13.4",
         "@types/koa__router": "8.0.0",
         "@types/lodash": "4.14.180",
+        "@types/mongodb": "3.6.3",
         "@types/node": "14.18.20",
         "@types/node-fetch": "2.6.1",
         "@types/oracledb": "5.2.2",