@budibase/server 1.0.46-alpha.0 → 1.0.46-alpha.4

package/dist/api/controllers/static/index.js

@@ -22,6 +22,8 @@ const { clientLibraryPath } = require("../../../utilities");
 const { upload } = require("../../../utilities/fileSystem");
 const { attachmentsRelativeURL } = require("../../../utilities");
 const { DocumentTypes } = require("../../../db/utils");
+const AWS = require("aws-sdk");
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1";
 function prepareUpload({ s3Key, bucket, metadata, file }) {
     return __awaiter(this, void 0, void 0, function* () {
         const response = yield upload({
@@ -109,3 +111,51 @@ exports.serveClientLibrary = function (ctx) {
         });
     });
 };
+exports.getSignedUploadURL = function (ctx) {
+    var _a, _b, _c;
+    return __awaiter(this, void 0, void 0, function* () {
+        const database = new CouchDB(ctx.appId);
+        // Ensure datasource is valid
+        let datasource;
+        try {
+            const { datasourceId } = ctx.params;
+            datasource = yield database.get(datasourceId);
+            if (!datasource) {
+                ctx.throw(400, "The specified datasource could not be found");
+            }
+        }
+        catch (error) {
+            ctx.throw(400, "The specified datasource could not be found");
+        }
+        // Ensure we aren't using a custom endpoint
+        if ((_a = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _a === void 0 ? void 0 : _a.endpoint) {
+            ctx.throw(400, "S3 datasources with custom endpoints are not supported");
+        }
+        // Determine type of datasource and generate signed URL
+        let signedUrl;
+        let publicUrl;
+        if (datasource.source === "S3") {
+            const { bucket, key } = ctx.request.body || {};
+            if (!bucket || !key) {
+                ctx.throw(400, "bucket and key values are required");
+                return;
+            }
+            try {
+                const s3 = new AWS.S3({
+                    region: AWS_REGION,
+                    accessKeyId: (_b = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _b === void 0 ? void 0 : _b.accessKeyId,
+                    secretAccessKey: (_c = datasource === null || datasource === void 0 ? void 0 : datasource.config) === null || _c === void 0 ? void 0 : _c.secretAccessKey,
+                    apiVersion: "2006-03-01",
+                    signatureVersion: "v4",
+                });
+                const params = { Bucket: bucket, Key: key };
+                signedUrl = s3.getSignedUrl("putObject", params);
+                publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`;
+            }
+            catch (error) {
+                ctx.throw(400, error);
+            }
+        }
+        ctx.body = { signedUrl, publicUrl };
+    });
+};

package/dist/api/controllers/table/internal.js

@@ -24,6 +24,7 @@ const linkRows = require("../../../db/linkedRows");
 const { getRowParams, generateTableID } = require("../../../db/utils");
 const { FieldTypes } = require("../../../constants");
 const { TableSaveFunctions, hasTypeChanged, getTable, handleDataImport, } = require("./utils");
+const usageQuota = require("../../../utilities/usageQuota");
 exports.save = function (ctx) {
     return __awaiter(this, void 0, void 0, function* () {
         const appId = ctx.appId;
@@ -118,6 +119,7 @@ exports.destroy = function (ctx) {
             include_docs: true,
         }));
         yield db.bulkDocs(rows.rows.map(row => (Object.assign(Object.assign({}, row.doc), { _deleted: true }))));
+        yield usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length);
         // update linked rows
         yield linkRows.updateLinks({
             appId,

package/dist/api/controllers/table/utils.js

@@ -18,6 +18,7 @@ const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants");
 const { isExternalTable, breakExternalTableId, isSQL, } = require("../../../integrations/utils");
 const { getViews, saveView } = require("../view/utils");
 const viewTemplate = require("../view/viewBuilder");
+const usageQuota = require("../../../utilities/usageQuota");
 exports.checkForColumnUpdates = (db, oldTable, updatedTable) => __awaiter(void 0, void 0, void 0, function* () {
     let updatedRows = [];
     const rename = updatedTable._rename;
@@ -95,7 +96,11 @@ exports.handleDataImport = (appId, user, table, dataImport) => __awaiter(void 0,
         }
         finalData.push(row);
     }
+    yield usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+        dryRun: true,
+    });
     yield db.bulkDocs(finalData);
+    yield usageQuota.update(usageQuota.Properties.ROW, finalData.length);
     let response = yield db.put(table);
     table._rev = response._rev;
     return table;

package/dist/api/routes/static.js

@@ -43,5 +43,6 @@ router
     .post("/api/attachments/process", authorized(BUILDER), controller.uploadFile)
     .post("/api/attachments/:tableId/upload", paramResource("tableId"), authorized(PermissionTypes.TABLE, PermissionLevels.WRITE), controller.uploadFile)
     // TODO: this likely needs to be secured in some way
-    .get("/:appId/:path*", controller.serveApp);
+    .get("/:appId/:path*", controller.serveApp)
+    .post("/api/attachments/:datasourceId/url", authorized(PermissionTypes.TABLE, PermissionLevels.READ), controller.getSignedUploadURL);
 module.exports = router;

package/dist/automations/steps/createRow.js

@@ -10,7 +10,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const rowController = require("../../api/controllers/row");
 const automationUtils = require("../automationUtils");
-const env = require("../../environment");
 const usage = require("../../utilities/usageQuota");
 const { buildCtx } = require("./utils");
 exports.definition = {
@@ -87,10 +86,9 @@ exports.run = function ({ inputs, appId, emitter }) {
         });
         try {
             inputs.row = yield automationUtils.cleanUpRow(appId, inputs.row.tableId, inputs.row);
-            if (env.USE_QUOTAS) {
-                yield usage.update(usage.Properties.ROW, 1);
-            }
+            yield usage.update(usage.Properties.ROW, 1, { dryRun: true });
             yield rowController.save(ctx);
+            yield usage.update(usage.Properties.ROW, 1);
             return {
                 row: inputs.row,
                 response: ctx.body,

package/dist/automations/steps/deleteRow.js

@@ -9,7 +9,6 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
     });
 };
 const rowController = require("../../api/controllers/row");
-const env = require("../../environment");
 const usage = require("../../utilities/usageQuota");
 const { buildCtx } = require("./utils");
 const automationUtils = require("../automationUtils");
@@ -81,9 +80,7 @@ exports.run = function ({ inputs, appId, emitter }) {
             },
         });
         try {
-            if (env.isProd()) {
-                yield usage.update(usage.Properties.ROW, -1);
-            }
+            yield usage.update(usage.Properties.ROW, -1);
             yield rowController.destroy(ctx);
             return {
                 response: ctx.body,

package/dist/environment.js

@@ -30,6 +30,7 @@ module.exports = {
     MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
     MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
     USE_QUOTAS: process.env.USE_QUOTAS,
+    EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
     REDIS_URL: process.env.REDIS_URL,
     REDIS_PASSWORD: process.env.REDIS_PASSWORD,
     INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

package/dist/middleware/usageQuota.js

@@ -10,11 +10,9 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
 };
 const CouchDB = require("../db");
 const usageQuota = require("../utilities/usageQuota");
-const env = require("../environment");
-const { getTenantId } = require("@budibase/backend-core/tenancy");
+const { getUniqueRows } = require("../utilities/usageQuota/rows");
 const { isExternalTable, isRowId: isExternalRowId, } = require("../integrations/utils");
-// tenants without limits
-const EXCLUDED_TENANTS = ["bb", "default", "bbtest", "bbstaging"];
+const migration = require("../migrations/usageQuotas");
 // currently only counting new writes and deletes
 const METHOD_MAP = {
     POST: 1,
@@ -22,13 +20,13 @@ const METHOD_MAP = {
 };
 const DOMAIN_MAP = {
     rows: usageQuota.Properties.ROW,
-    upload: usageQuota.Properties.UPLOAD,
-    views: usageQuota.Properties.VIEW,
-    users: usageQuota.Properties.USER,
+    // upload: usageQuota.Properties.UPLOAD,            // doesn't work yet
+    // views: usageQuota.Properties.VIEW,               // doesn't work yet
+    // users: usageQuota.Properties.USER,               // doesn't work yet
     applications: usageQuota.Properties.APPS,
     // this will not be updated by endpoint calls
     // instead it will be updated by triggerInfo
-    automationRuns: usageQuota.Properties.AUTOMATION,
+    // automationRuns: usageQuota.Properties.AUTOMATION, // doesn't work yet
 };
 function getProperty(url) {
     for (let domain of Object.keys(DOMAIN_MAP)) {
@@ -38,9 +36,7 @@ function getProperty(url) {
     }
 }
 module.exports = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
-    const tenantId = getTenantId();
-    // if in development or a self hosted cloud usage quotas should not be executed
-    if (env.isDev() || env.SELF_HOSTED || EXCLUDED_TENANTS.includes(tenantId)) {
+    if (!usageQuota.useQuotas()) {
         return next();
     }
     let usage = METHOD_MAP[ctx.req.method];
@@ -78,10 +74,83 @@ module.exports = (ctx, next) => __awaiter(void 0, void 0, void 0, function* () {
         usage = files.map(file => file.size).reduce((total, size) => total + size);
     }
     try {
-        yield usageQuota.update(property, usage);
-        return next();
+        yield migration.run();
+        yield performRequest(ctx, next, property, usage);
     }
     catch (err) {
         ctx.throw(400, err);
     }
 });
+const performRequest = (ctx, next, property, usage) => __awaiter(void 0, void 0, void 0, function* () {
+    const usageContext = {
+        skipNext: false,
+        skipUsage: false,
+        [usageQuota.Properties.APPS]: {},
+    };
+    if (usage === -1) {
+        if (PRE_DELETE[property]) {
+            yield PRE_DELETE[property](ctx, usageContext);
+        }
+    }
+    else {
+        if (PRE_CREATE[property]) {
+            yield PRE_CREATE[property](ctx, usageContext);
+        }
+    }
+    // run the request
+    if (!usageContext.skipNext) {
+        yield usageQuota.update(property, usage, { dryRun: true });
+        yield next();
+    }
+    if (usage === -1) {
+        if (POST_DELETE[property]) {
+            yield POST_DELETE[property](ctx, usageContext);
+        }
+    }
+    else {
+        if (POST_CREATE[property]) {
+            yield POST_CREATE[property](ctx, usageContext);
+        }
+    }
+    // update the usage
+    if (!usageContext.skipUsage) {
+        yield usageQuota.update(property, usage);
+    }
+});
+const appPreDelete = (ctx, usageContext) => __awaiter(void 0, void 0, void 0, function* () {
+    if (ctx.query.unpublish) {
+        // don't run usage decrement for unpublish
+        usageContext.skipUsage = true;
+        return;
+    }
+    // store the row count to delete
+    const rows = yield getUniqueRows([ctx.appId]);
+    if (rows.length) {
+        usageContext[usageQuota.Properties.APPS] = { rowCount: rows.length };
+    }
+});
+const appPostDelete = (ctx, usageContext) => __awaiter(void 0, void 0, void 0, function* () {
+    // delete the app rows from usage
+    const rowCount = usageContext[usageQuota.Properties.APPS].rowCount;
+    if (rowCount) {
+        yield usageQuota.update(usageQuota.Properties.ROW, -rowCount);
+    }
+});
+const appPostCreate = (ctx) => __awaiter(void 0, void 0, void 0, function* () {
+    // app import & template creation
+    if (ctx.request.body.useTemplate === "true") {
+        const rows = yield getUniqueRows([ctx.response.body.appId]);
+        const rowCount = rows ? rows.length : 0;
+        yield usageQuota.update(usageQuota.Properties.ROW, rowCount);
+    }
+});
+const PRE_DELETE = {
+    [usageQuota.Properties.APPS]: appPreDelete,
+};
+const POST_DELETE = {
+    [usageQuota.Properties.APPS]: appPostDelete,
+};
+const PRE_CREATE = {};
+const POST_CREATE = {
+    [usageQuota.Properties.APPS]: appPostCreate,
+};

package/dist/migrations/usageQuotas/index.js

@@ -0,0 +1,24 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { MIGRATIONS, MIGRATION_DBS, migrateIfRequired, } = require("@budibase/backend-core/migrations");
+const { useQuotas } = require("../../utilities/usageQuota");
+const syncApps = require("./syncApps");
+const syncRows = require("./syncRows");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    if (!useQuotas()) {
+        return;
+    }
+    // Jan 2022
+    yield migrateIfRequired(MIGRATION_DBS.GLOBAL_DB, MIGRATIONS.QUOTAS_1, () => __awaiter(void 0, void 0, void 0, function* () {
+        yield syncApps.run();
+        yield syncRows.run();
+    }));
+});

package/dist/migrations/usageQuotas/syncApps.js

@@ -0,0 +1,26 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
+const { getAllApps } = require("@budibase/backend-core/db");
+const CouchDB = require("../../db");
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    const db = getGlobalDB();
+    // get app count
+    const devApps = yield getAllApps(CouchDB, { dev: true });
+    const appCount = devApps ? devApps.length : 0;
+    // sync app count
+    const tenantId = getTenantId();
+    console.log(`[Tenant: ${tenantId}] Syncing app count: ${appCount}`);
+    const usageDoc = yield getUsageQuotaDoc(db);
+    usageDoc.usageQuota.apps = appCount;
+    yield db.put(usageDoc);
+});

package/dist/migrations/usageQuotas/syncRows.js

@@ -0,0 +1,29 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
+const { getAllApps } = require("@budibase/backend-core/db");
+const CouchDB = require("../../db");
+const { getUsageQuotaDoc } = require("../../utilities/usageQuota");
+const { getUniqueRows } = require("../../utilities/usageQuota/rows");
+exports.run = () => __awaiter(void 0, void 0, void 0, function* () {
+    const db = getGlobalDB();
+    // get all rows in all apps
+    const allApps = yield getAllApps(CouchDB, { all: true });
+    const appIds = allApps ? allApps.map(app => app.appId) : [];
+    const rows = yield getUniqueRows(appIds);
+    const rowCount = rows ? rows.length : 0;
+    // sync row count
+    const tenantId = getTenantId();
+    console.log(`[Tenant: ${tenantId}] Syncing row count: ${rowCount}`);
+    const usageDoc = yield getUsageQuotaDoc(db);
+    usageDoc.usageQuota.rows = rowCount;
+    yield db.put(usageDoc);
+});

package/dist/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@budibase/server",
     "email": "hi@budibase.com",
-    "version": "1.0.45",
+    "version": "1.0.46-alpha.3",
     "description": "Budibase Web Server",
     "main": "src/index.ts",
     "repository": {
@@ -70,9 +70,9 @@
     "license": "GPL-3.0",
     "dependencies": {
         "@apidevtools/swagger-parser": "^10.0.3",
-        "@budibase/backend-core": "^1.0.45",
-        "@budibase/client": "^1.0.45",
-        "@budibase/string-templates": "^1.0.45",
+        "@budibase/backend-core": "^1.0.46-alpha.3",
+        "@budibase/client": "^1.0.46-alpha.3",
+        "@budibase/string-templates": "^1.0.46-alpha.3",
         "@bull-board/api": "^3.7.0",
         "@bull-board/koa": "^3.7.0",
         "@elastic/elasticsearch": "7.10.0",