@budibase/server 1.0.48 → 1.0.49-alpha.0

package/dist/middleware/usageQuota.js

@@ -10,11 +10,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const CouchDB = require("../db");
 const usageQuota = require("../utilities/usageQuota");
-const env = require("../environment");
-const { getTenantId } = require("@budibase/backend-core/tenancy");
+const { getUniqueRows } = require("../utilities/usageQuota/rows");
 const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils");
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"];
+const migration = require("../migrations/usageQuotas");
 // currently only counting new writes and deletes
 const METHOD_MAP = {
     POST: 1,
@@ -22,13 +20,13 @@ const METHOD_MAP = {
 };
 const DOMAIN_MAP = {
     rows: usageQuota.Properties.ROW,
-    upload: usageQuota.Properties.UPLOAD,
-    views: usageQuota.Properties.VIEW,
-    users: usageQuota.Properties.USER,
+    // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+    // views: usageQuota.Properties.VIEW,               // doesn't work yet
+    // users: usageQuota.Properties.USER,               // doesn't work yet
     applications: usageQuota.Properties.APPS,
     // this will not be updated by endpoint calls
     // instead it will be updated by triggerInfo
-    automationRuns: usageQuota.Properties.AUTOMATION,
+    // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 };
 function getProperty(url) {
     for (let domain of Object.keys(DOMAIN_MAP)) {
@@ -38,9 +36,7 @@ function getProperty(url) {
     }
 }
 module.exports = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
-    const tenantId = getTenantId();
-    // if in development or a self hosted cloud usage quotas should not be executed
-    if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+    if (!usageQuota.useQuotas()) {
         return next();
     }
     let usage = METHOD_MAP[ctx.req.method];
@@ -78,10 +74,83 @@ module.exports = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
         usage = files.map(file => file.size).reduce((total, size) => total + size);
     }
     try {
-        yield usageQuota.update(property, usage);
-        return next();
+        yield migration.run();
+        yield performRequest(ctx, next, property, usage);
     }
     catch (err) {
         ctx.throw(400, err);
     }
 });
+const performRequest = (ctx, next, property, usage) => __awaiter(void 0, void 0, void 0, function* () {
+    const usageContext = {
+        skipNext: false,
+        skipUsage: false,
+        [usageQuota.Properties.APPS]: {},
+    };
+    if (usage === -1) {
+        if (PRE_DELETE[property]) {
+            yield PRE_DELETE[property](ctx, usageContext);
+        }
+    }
+    else {
+        if (PRE_CREATE[property]) {
+            yield PRE_CREATE[property](ctx, usageContext);
+        }
+    }
+    // run the request
+    if (!usageContext.skipNext) {
+        yield usageQuota.update(property, usage, { dryRun: true });
+        yield next();
+    }
+    if (usage === -1) {
+        if (POST_DELETE[property]) {
+            yield POST_DELETE[property](ctx, usageContext);
+        }
+    }
+    else {
+        if (POST_CREATE[property]) {
+            yield POST_CREATE[property](ctx, usageContext);
+        }
+    }
+    // update the usage
+    if (!usageContext.skipUsage) {
+        yield usageQuota.update(property, usage);
+    }
+});
+const appPreDelete = (ctx, usageContext) => __awaiter(void 0, void 0, void 0, function* () {
+    if (ctx.query.unpublish) {
+        // don't run usage decrement for unpublish
+        usageContext.skipUsage = true;
+        return;
+    }
+    // store the row count to delete
+    const rows = yield getUniqueRows([ctx.appId]);
+    if (rows.length) {
+        usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length };
+    }
+});
+const appPostDelete = (ctx, usageContext) => __awaiter(void 0, void 0, void 0, function* () {
+    // delete the app rows from usage
+    const rowCount = usageContext[usageQuota.Properties.APPS].rowCount;
+    if (rowCount) {
+        yield usageQuota.update(usageQuota.Properties.ROW, -rowCount);
+    }
+});
+const appPostCreate = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
+    // app import & template creation
+    if (ctx.request.body.useTemplate === "true") {
+        const rows = yield getUniqueRows([ctx.response.body.appId]);
+        const rowCount = rows ? rows.length : 0;
+        yield usageQuota.update(usageQuota.Properties.ROW, rowCount);
+    }
+});
+const PRE_DELETE = {
+    [usageQuota.Properties.APPS]: appPreDelete,
+};
+const POST_DELETE = {
+    [usageQuota.Properties.APPS]: appPostDelete,
+};
+const PRE_CREATE = {};
+const POST_CREATE = {
+    [usageQuota.Properties.APPS]: appPostCreate,
+};

package/dist/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired, } = require("@budibase/backend-core/migrations");
+const { useQuotas } = require("../../utilities/usageQuota");
+const syncApps = require("./syncApps");
+const syncRows = require("./syncRows");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    if (!useQuotas()) {
+        return;
+    }
+    // Jan 2022
+    yield migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.QUOTAS_1, () => __awaiter(void 0, void 0, void 0, function* () {
+        yield syncApps.run();
+        yield syncRows.run();
+    }));
+});

package/dist/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,26 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
+const { getAllApps } = require("@budibase/backend-core/db");
+const CouchDB = require("../../db");
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    const db = getGlobalDB();
+    // get app count
+    const devApps = yield getAllApps(CouchDB, { dev: true });
+    const appCount = devApps ? devApps.length : 0;
+    // sync app count
+    const tenantId = getTenantId();
+    console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`);
+    const usageDoc = yield getUsageQuotaDoc(db);
+    usageDoc.usageQuota.apps = appCount;
+    yield db.put(usageDoc);
+});

package/dist/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,29 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
+const { getAllApps } = require("@budibase/backend-core/db");
+const CouchDB = require("../../db");
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota");
+const { getUniqueRows } = require("../../utilities/usageQuota/rows");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    const db = getGlobalDB();
+    // get all rows in all apps
+    const allApps = yield getAllApps(CouchDB, { all: true });
+    const appIds = allApps ? allApps.map(app => app.appId) : [];
+    const rows = yield getUniqueRows(appIds);
+    const rowCount = rows ? rows.length : 0;
+    // sync row count
+    const tenantId = getTenantId();
+    console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`);
+    const usageDoc = yield getUsageQuotaDoc(db);
+    usageDoc.usageQuota.rows = rowCount;
+    yield db.put(usageDoc);
+});

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.47",
+    "version": "1.0.48",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -70,9 +70,9 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "^10.0.3",
-        "@budibase/backend-core": "^1.0.47",
-        "@budibase/client": "^1.0.47",
-        "@budibase/string-templates": "^1.0.47",
+        "@budibase/backend-core": "^1.0.48",
+        "@budibase/client": "^1.0.48",
+        "@budibase/string-templates": "^1.0.48",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",
@@ -92,6 +92,8 @@
         "fix-path": "3.0.0",
         "form-data": "^4.0.0",
         "fs-extra": "8.1.0",
+        "google-auth-library": "^7.11.0",
+        "google-spreadsheet": "^3.2.0",
         "jimp": "0.16.1",
         "joi": "17.2.1",
         "js-yaml": "^4.1.0",
@@ -110,7 +112,7 @@
         "mongodb": "3.6.3",
         "mssql": "6.2.3",
         "mysql2": "^2.3.1",
-        "node-fetch": "2.6.0",
+        "node-fetch": "2.6.7",
         "open": "^8.4.0",
         "pg": "8.5.1",
         "pino-pretty": "4.0.0",
@@ -139,6 +141,7 @@
         "@jest/test-sequencer": "^24.8.0",
         "@types/apidoc": "^0.50.0",
         "@types/bull": "^3.15.1",
+        "@types/google-spreadsheet": "^3.1.5",
         "@types/jest": "^26.0.23",
         "@types/koa": "^2.13.3",
         "@types/koa-router": "^7.4.2",

package/dist/tests/utilities/TestConfiguration.js

@@ -21,9 +21,11 @@ const { getGlobalDB } = require("@budibase/backend-core/tenancy");
 const { createASession } = require("@budibase/backend-core/sessions");
 const { user: userCache } = require("@budibase/backend-core/cache");
 const CouchDB = require("../../db");
+const newid = require("../../db/newid");
 core.init(CouchDB);
 const GLOBAL_USER_ID = "us_uuid1";
 const EMAIL = "babs@babs.com";
+const CSRF_TOKEN = "e3727778-7af0-4226-b5eb-f43cbe60a306";
 class TestConfiguration {
     constructor(openServer = true) {
         if (openServer) {
@@ -72,7 +74,11 @@ class TestConfiguration {
                 existing = { email };
             }
             const user = Object.assign(Object.assign({ _id: id }, existing), { roles: roles || {}, tenantId: TENANT_ID });
-            yield createASession(id, { sessionId: "sessionid", tenantId: TENANT_ID });
+            yield createASession(id, {
+                sessionId: "sessionid",
+                tenantId: TENANT_ID,
+                csrfToken: CSRF_TOKEN,
+            });
             if (builder) {
                 user.builder = { global: true };
             }
@@ -83,7 +89,8 @@ class TestConfiguration {
             return Object.assign({ _rev: resp._rev }, user);
         });
     }
-    init(appName = "test_application") {
+    // use a new id as the name to avoid name collisions
+    init(appName = newid()) {
         return __awaiter(this, void 0, void 0, function* () {
             yield this.globalUser();
             return this.createApp(appName);
@@ -116,6 +123,7 @@ class TestConfiguration {
                 `${Cookies.Auth}=${authToken}`,
                 `${Cookies.CurrentApp}=${appToken}`,
             ],
+            [Headers.CSRF_TOKEN]: CSRF_TOKEN,
         };
         if (this.appId) {
             headers[Headers.APP_ID] = this.appId;
@@ -394,10 +402,6 @@ class TestConfiguration {
                     roles: { [this.prodAppId]: roleId },
                 });
             }
-            yield createASession(userId, {
-                sessionId: "sessionid",
-                tenantId: TENANT_ID,
-            });
             // have to fake this
             const auth = {
                 userId,

package/dist/threads/query.js

@@ -75,6 +75,10 @@ class QueryRunner {
                 yield threadUtils.invalidateDynamicVariables(this.cachedVariables);
                 return this.execute();
             }
+            // check for undefined response
+            if (!rows) {
+                rows = [];
+            }
             // needs to an array for next step
             if (!Array.isArray(rows)) {
                 rows = [rows];