@budibase/server 2.0.39 → 2.0.40-alpha.0

package/dist/integrations/mongodb.js

@@ -8,6 +8,13 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 const types_1 = require("@budibase/types");
 const mongodb_1 = require("mongodb");
@@ -40,6 +47,232 @@ const SCHEMA = {
         delete: {
             type: types_1.QueryType.JSON,
         },
+        aggregate: {
+            type: types_1.QueryType.JSON,
+            steps: [
+                {
+                    key: "$addFields",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$bucket",
+                    template: `{
+  "groupBy": "",
+  "boundaries": [],
+  "default": "",
+  "output": {}
+}`,
+                },
+                {
+                    key: "$bucketAuto",
+                    template: `{
+  "groupBy": "",
+  "buckets": 1,
+  "output": {},
+  "granularity": "R5"
+}`,
+                },
+                {
+                    key: "$changeStream",
+                    template: `{
+  "allChangesForCluster": true,
+  "fullDocument": "",
+  "fullDocumentBeforeChange": "",
+  "resumeAfter": 1,
+  "showExpandedEvents": true,
+  "startAfter": {},
+  "startAtOperationTime": ""
+}`,
+                },
+                {
+                    key: "$collStats",
+                    template: `{
+  "latencyStats": { "histograms": true } },
+  "storageStats": { "scale": 1 } },
+  "count": {},
+  "queryExecStats": {}
+}`,
+                },
+                {
+                    key: "$count",
+                    template: ``,
+                },
+                {
+                    key: "$densify",
+                    template: `{
+  "field": "",
+  "partitionByFields": [],
+  "range": {
+    "step": 1,
+    "unit": 1,
+    "bounds": "full"
+  }
+}`,
+                },
+                {
+                    key: "$documents",
+                    template: `[]`,
+                },
+                {
+                    key: "$facet",
+                    template: `{\n\t\n}`,
+                },
+                {
+                    key: "$fill",
+                    template: `{
+  "partitionBy": "",
+  "partitionByFields": [],
+  "sortBy": {},
+  "output": {}
+}`,
+                },
+                {
+                    key: "$geoNear",
+                    template: `{
+  "near": { 
+    "type": "Point", 
+    "coordinates": [ 
+      -73.98142, 40.71782
+    ] 
+  },
+  "key": "location",
+  "distanceField": "dist.calculated",
+  "query": { "category": "Parks" }
+}`,
+                },
+                {
+                    key: "$graphLookup",
+                    template: `{
+  "from": "",
+  "startWith": "",
+  "connectFromField": "",
+  "connectToField": "",
+  "as": "",
+  "maxDepth": 1,
+  "depthField": "",
+  "restrictSearchWithMatch": {}
+}`,
+                },
+                {
+                    key: "$group",
+                    template: `{
+  "_id": ""
+}`,
+                },
+                {
+                    key: "$indexStats",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$limit",
+                    template: `1`,
+                },
+                {
+                    key: "$listLocalSessions",
+                    template: `{\n\t\n}`,
+                },
+                {
+                    key: "$listSessions",
+                    template: `{\n\t\n}`,
+                },
+                {
+                    key: "$lookup",
+                    template: `{
+  "from": "",
+  "localField": "",
+  "foreignField": "",
+  "as": ""
+}`,
+                },
+                {
+                    key: "$match",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$merge",
+                    template: `{
+  "into": {},
+  "on": "_id",
+  "whenMatched": "replace",
+  "whenNotMatched": "insert"
+}`,
+                },
+                {
+                    key: "$out",
+                    template: `{
+  "db": "",
+  "coll": ""
+}`,
+                },
+                {
+                    key: "$planCacheStats",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$project",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$redact",
+                    template: "",
+                },
+                {
+                    key: "$replaceRoot",
+                    template: `{ "newRoot": "" }`,
+                },
+                {
+                    key: "$replaceWith",
+                    template: ``,
+                },
+                {
+                    key: "$sample",
+                    template: `{ "size": 3 }`,
+                },
+                {
+                    key: "$set",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$setWindowFields",
+                    template: `{
+  "partitionBy": "",
+  "sortBy": {},
+  "output": {}
+}`,
+                },
+                {
+                    key: "$skip",
+                    template: `1`,
+                },
+                {
+                    key: "$sort",
+                    template: "{\n\t\n}",
+                },
+                {
+                    key: "$sortByCount",
+                    template: "",
+                },
+                {
+                    key: "$unionWith",
+                    template: `{
+  "coll": "",
+  "pipeline": []
+}`,
+                },
+                {
+                    key: "$unset",
+                    template: "",
+                },
+                {
+                    key: "$unwind",
+                    template: `{
+  "path": "",
+  "includeArrayIndex": "",
+  "preserveNullAndEmptyArrays": true
+}`,
+                },
+            ],
+        },
     },
     extra: {
         collection: {
@@ -47,8 +280,8 @@ const SCHEMA = {
             type: types_1.DatasourceFieldType.STRING,
             required: true,
         },
-        actionTypes: {
-            displayName: "Action Types",
+        actionType: {
+            displayName: "Query Type",
             type: types_1.DatasourceFieldType.LIST,
             required: true,
             data: {
@@ -56,6 +289,7 @@ const SCHEMA = {
                 create: ["insertOne", "insertMany"],
                 update: ["updateOne", "updateMany"],
                 delete: ["deleteOne", "deleteMany"],
+                aggregate: ["json", "pipeline"],
             },
         },
     },
@@ -82,7 +316,7 @@ class MongoIntegration {
                     json[field].toLowerCase().startsWith("objectid")) {
                     const id = (_a = json[field].match(/(?<=objectid\(['"]).*(?=['"]\))/gi)) === null || _a === void 0 ? void 0 : _a[0];
                     if (id) {
-                        json[field] = mongodb_1.ObjectID.createFromHexString(id);
+                        json[field] = mongodb_1.ObjectId.createFromHexString(id);
                     }
                 }
             }
@@ -145,7 +379,7 @@ class MongoIntegration {
                 let json = this.createObjectIds(query.json);
                 // For mongodb we add an extra actionType to specify
                 // which method we want to call on the collection
-                switch (query.extra.actionTypes) {
+                switch (query.extra.actionType) {
                     case "insertOne": {
                         return yield collection.insertOne(json);
                     }
@@ -153,7 +387,7 @@ class MongoIntegration {
                         return yield collection.insertMany(json);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for create`);
+                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for create`);
                     }
                 }
             }
@@ -173,7 +407,7 @@ class MongoIntegration {
                 const db = this.client.db(this.config.db);
                 const collection = db.collection(query.extra.collection);
                 let json = this.createObjectIds(query.json);
-                switch (query.extra.actionTypes) {
+                switch (query.extra.actionType) {
                     case "find": {
                         return yield collection.find(json).toArray();
                     }
@@ -194,7 +428,7 @@ class MongoIntegration {
                         return yield collection.distinct(json);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for read`);
+                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for read`);
                     }
                 }
             }
@@ -218,7 +452,7 @@ class MongoIntegration {
                     queryJson = this.parseQueryParams(queryJson, "update");
                 }
                 let json = this.createObjectIds(queryJson);
-                switch (query.extra.actionTypes) {
+                switch (query.extra.actionType) {
                     case "updateOne": {
                         return yield collection.updateOne(json.filter, json.update, json.options);
                     }
@@ -226,7 +460,7 @@ class MongoIntegration {
                         return yield collection.updateMany(json.filter, json.update, json.options);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for update`);
+                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for update`);
                     }
                 }
             }
@@ -256,7 +490,7 @@ class MongoIntegration {
                         options: {},
                     };
                 }
-                switch (query.extra.actionTypes) {
+                switch (query.extra.actionType) {
                     case "deleteOne": {
                         return yield collection.deleteOne(json.filter, json.options);
                     }
@@ -264,9 +498,64 @@ class MongoIntegration {
                         return yield collection.deleteMany(json.filter, json.options);
                     }
                     default: {
-                        throw new Error(`actionType ${query.extra.actionTypes} does not exist on DB for delete`);
+                        throw new Error(`actionType ${query.extra.actionType} does not exist on DB for delete`);
+                    }
+                }
+            }
+            catch (err) {
+                console.error("Error writing to mongodb", err);
+                throw err;
+            }
+            finally {
+                yield this.client.close();
+            }
+        });
+    }
+    aggregate(query) {
+        var e_1, _a, e_2, _b;
+        var _c;
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                yield this.connect();
+                const db = this.client.db(this.config.db);
+                const collection = db.collection(query.extra.collection);
+                let response = [];
+                if (((_c = query.extra) === null || _c === void 0 ? void 0 : _c.actionType) === "pipeline") {
+                    try {
+                        for (var _d = __asyncValues(collection.aggregate(query.steps.map(({ key, value }) => {
+                            let temp = {};
+                            temp[key] = JSON.parse(value.value);
+                            return this.createObjectIds(temp);
+                        }))), _e; _e = yield _d.next(), !_e.done;) {
+                            const doc = _e.value;
+                            response.push(doc);
+                        }
+                    }
+                    catch (e_1_1) { e_1 = { error: e_1_1 }; }
+                    finally {
+                        try {
+                            if (_e && !_e.done && (_a = _d.return)) yield _a.call(_d);
+                        }
+                        finally { if (e_1) throw e_1.error; }
+                    }
+                }
+                else {
+                    const stages = query.json;
+                    try {
+                        for (var _f = __asyncValues(collection.aggregate(stages ? this.createObjectIds(stages) : [])), _g; _g = yield _f.next(), !_g.done;) {
+                            const doc = _g.value;
+                            response.push(doc);
+                        }
+                    }
+                    catch (e_2_1) { e_2 = { error: e_2_1 }; }
+                    finally {
+                        try {
+                            if (_g && !_g.done && (_b = _f.return)) yield _b.call(_f);
+                        }
+                        finally { if (e_2) throw e_2.error; }
                     }
                 }
+                return response;
             }
             catch (err) {
                 console.error("Error writing to mongodb", err);

package/dist/integrations/rest.js

@@ -1,4 +1,27 @@
 "use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
 var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
     function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
     return new (P || (P = Promise))(function (resolve, reject) {
@@ -15,6 +38,7 @@ Object.defineProperty(exports, "__esModule", { value: true });
 const types_1 = require("@budibase/types");
 const datasource_1 = require("../definitions/datasource");
 const lodash_1 = require("lodash");
+const https = __importStar(require("https"));
 const querystring_1 = __importDefault(require("querystring"));
 const fetch = require("node-fetch");
 const { formatBytes } = require("../utilities");
@@ -72,11 +96,11 @@ const SCHEMA = {
             required: false,
             default: {},
         },
-        legacyHttpParser: {
-            display: "Legacy HTTP Support",
+        rejectUnauthorized: {
+            display: "Reject Unauthorized",
             type: types_1.DatasourceFieldType.BOOLEAN,
+            default: true,
             required: false,
-            default: false,
         },
     },
     query: {
@@ -197,8 +221,11 @@ class RestIntegration {
                 queryString = `${paginationString}&${queryString}`;
             }
         }
-        // make sure the query string is fully encoded
-        const main = `${path}?${querystring_1.default.encode(querystring_1.default.decode(queryString))}`;
+        if (queryString) {
+            // make sure the query string is fully encoded
+            queryString = "?" + querystring_1.default.encode(querystring_1.default.decode(queryString));
+        }
+        const main = `${path}${queryString}`;
         let complete = main;
         if (this.config.url && !main.startsWith("http")) {
             complete = !this.config.url ? main : `${this.config.url}/${main}`;
@@ -318,6 +345,12 @@ class RestIntegration {
             }
             let input = { method, headers: this.headers };
             input = this.addBody(bodyType, requestBody, input, pagination, paginationValues);
+            if (this.config.rejectUnauthorized == false) {
+                input.agent = new https.Agent({
+                    rejectUnauthorized: false,
+                });
+            }
+            // Deprecated by rejectUnauthorized
             if (this.config.legacyHttpParser) {
                 // https://github.com/nodejs/node/issues/43798
                 input.extraHttpOptions = { insecureHTTPParser: true };

package/dist/migrations/functions/backfill/app/tables.js

@@ -8,19 +8,16 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
 Object.defineProperty(exports, "__esModule", { value: true });
 exports.backfill = void 0;
 const backend_core_1 = require("@budibase/backend-core");
-const utils_1 = require("../../../../db/utils");
-const getTables = (appDb) => __awaiter(void 0, void 0, void 0, function* () {
-    const response = yield appDb.allDocs((0, utils_1.getTableParams)(null, {
-        include_docs: true,
-    }));
-    return response.rows.map((row) => row.doc);
-});
+const sdk_1 = __importDefault(require("../../../../sdk"));
 const backfill = (appDb, timestamp) => __awaiter(void 0, void 0, void 0, function* () {
     var _a;
-    const tables = yield getTables(appDb);
+    const tables = yield sdk_1.default.tables.getAllInternalTables(appDb);
     for (const table of tables) {
         yield backend_core_1.events.table.created(table, timestamp);
         if (table.views) {

package/dist/migrations/functions/{quotas1.js → syncQuotas.js}

@@ -36,19 +36,15 @@ exports.run = void 0;
 const usageQuotas_1 = require("./usageQuotas");
 const syncApps = __importStar(require("./usageQuotas/syncApps"));
 const syncRows = __importStar(require("./usageQuotas/syncRows"));
+const syncPlugins = __importStar(require("./usageQuotas/syncPlugins"));
 /**
- * Date:
- * January 2022
- *
- * Description:
- * Synchronise the app and row quotas to the state of the db after it was
- * discovered that the quota resets were still in place and the row quotas
- * weren't being decremented correctly.
+ * Synchronise quotas to the state of the db.
  */
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     yield (0, usageQuotas_1.runQuotaMigration)(() => __awaiter(void 0, void 0, void 0, function* () {
         yield syncApps.run();
         yield syncRows.run();
+        yield syncPlugins.run();
     }));
 });
 exports.run = run;

package/dist/migrations/functions/usageQuotas/syncApps.js

@@ -16,7 +16,6 @@ const pro_1 = require("@budibase/pro");
 const types_1 = require("@budibase/types");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     // get app count
-    // @ts-ignore
     const devApps = yield (0, db_1.getAllApps)({ dev: true });
     const appCount = devApps ? devApps.length : 0;
     // sync app count

package/dist/migrations/functions/{pluginCount.js → usageQuotas/syncPlugins.js}

@@ -14,9 +14,7 @@ const backend_core_1 = require("@budibase/backend-core");
 const pro_1 = require("@budibase/pro");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     try {
-        yield backend_core_1.tenancy.doInTenant(backend_core_1.tenancy.DEFAULT_TENANT_ID, () => __awaiter(void 0, void 0, void 0, function* () {
-            yield pro_1.plugins.checkPluginQuotas();
-        }));
+        yield pro_1.plugins.checkPluginQuotas();
     }
     catch (err) {
         backend_core_1.logging.logAlert("Failed to update plugin quotas", err);

package/dist/migrations/functions/usageQuotas/syncRows.js

@@ -17,15 +17,19 @@ const pro_1 = require("@budibase/pro");
 const types_1 = require("@budibase/types");
 const run = () => __awaiter(void 0, void 0, void 0, function* () {
     // get all rows in all apps
-    // @ts-ignore
     const allApps = yield (0, db_1.getAllApps)({ all: true });
-    // @ts-ignore
     const appIds = allApps ? allApps.map((app) => app.appId) : [];
-    const rows = yield (0, rows_1.getUniqueRows)(appIds);
-    const rowCount = rows ? rows.length : 0;
+    const { appRows } = yield (0, rows_1.getUniqueRows)(appIds);
+    // get the counts per app
+    const counts = {};
+    let rowCount = 0;
+    Object.entries(appRows).forEach(([appId, rows]) => {
+        counts[appId] = rows.length;
+        rowCount += rows.length;
+    });
     // sync row count
     const tenantId = (0, tenancy_1.getTenantId)();
     console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`);
-    yield pro_1.quotas.setUsage(rowCount, types_1.StaticQuotaName.ROWS, types_1.QuotaUsageType.STATIC);
+    yield pro_1.quotas.setUsagePerApp(counts, types_1.StaticQuotaName.ROWS, types_1.QuotaUsageType.STATIC);
 });
 exports.run = run;

package/dist/migrations/index.js

@@ -41,10 +41,9 @@ const types_1 = require("@budibase/types");
 const environment_1 = __importDefault(require("../environment"));
 // migration functions
 const userEmailViewCasing = __importStar(require("./functions/userEmailViewCasing"));
-const quota1 = __importStar(require("./functions/quotas1"));
+const syncQuotas = __importStar(require("./functions/syncQuotas"));
 const appUrls = __importStar(require("./functions/appUrls"));
 const backfill = __importStar(require("./functions/backfill"));
-const pluginCount = __importStar(require("./functions/pluginCount"));
 /**
  * Populate the migration function and additional configuration from
  * the static migration definitions.
@@ -58,8 +57,8 @@ const buildMigrations = () => {
                 serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: userEmailViewCasing.run }));
                 break;
             }
-            case types_1.MigrationName.QUOTAS_1: {
-                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: quota1.run }));
+            case types_1.MigrationName.SYNC_QUOTAS: {
+                serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: syncQuotas.run }));
                 break;
             }
             case types_1.MigrationName.APP_URLS: {
@@ -78,11 +77,6 @@ const buildMigrations = () => {
                 serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: backfill.installation.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: !!environment_1.default.SELF_HOSTED }));
                 break;
             }
-            case types_1.MigrationName.PLUGIN_COUNT: {
-                if (environment_1.default.SELF_HOSTED) {
-                    serverMigrations.push(Object.assign(Object.assign({}, definition), { fn: pluginCount.run, silent: !!environment_1.default.SELF_HOSTED, preventRetry: false }));
-                }
-            }
         }
     }
     return serverMigrations;
@@ -101,34 +95,12 @@ const migrate = (options) => __awaiter(void 0, void 0, void 0, function* () {
 });
 exports.migrate = migrate;
 const migrateWithLock = (options) => __awaiter(void 0, void 0, void 0, function* () {
-    // get a new lock client
-    const redlock = yield backend_core_1.redis.clients.getMigrationsRedlock();
-    // lock for 15 minutes
-    const ttl = 1000 * 60 * 15;
-    let migrationLock;
-    // acquire lock
-    try {
-        migrationLock = yield redlock.lock("migrations", ttl);
-    }
-    catch (e) {
-        if (e.name === "LockError") {
-            return;
-        }
-        else {
-            throw e;
-        }
-    }
-    // run migrations
-    try {
+    yield backend_core_1.locks.doWithLock({
+        type: types_1.LockType.TRY_ONCE,
+        name: types_1.LockName.MIGRATIONS,
+        ttl: 1000 * 60 * 15,
+        systemLock: true,
+    }, () => __awaiter(void 0, void 0, void 0, function* () {
         yield backend_core_1.migrations.runMigrations(exports.MIGRATIONS, options);
-    }
-    finally {
-        // release lock
-        try {
-            yield migrationLock.unlock();
-        }
-        catch (e) {
-            console.error("unable to release migration lock");
-        }
-    }
+    }));
 });