@budibase/server 1.0.46-alpha.0 → 1.0.46-alpha.4

package/dist/utilities/{usageQuota.js → usageQuota/index.js}

@@ -8,12 +8,24 @@ var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, ge
         step((generator = generator.apply(thisArg, _arguments || [])).next());
     });
 };
-const env = require("../environment");
-const { getGlobalDB } = require("@budibase/backend-core/tenancy");
+const env = require("../../environment");
+const { getGlobalDB, getTenantId } = require("@budibase/backend-core/tenancy");
 const { StaticDatabases, generateNewUsageQuotaDoc, } = require("@budibase/backend-core/db");
-function getNewQuotaReset() {
-    return Date.now() + 2592000000;
-}
+exports.useQuotas = () => {
+    // check if quotas are enabled
+    if (env.USE_QUOTAS) {
+        // check if there are any tenants without limits
+        if (env.EXCLUDE_QUOTAS_TENANTS) {
+            const excludedTenants = env.EXCLUDE_QUOTAS_TENANTS.replace(/\s/g, "").split(",");
+            const tenantId = getTenantId();
+            if (excludedTenants.includes(tenantId)) {
+                return false;
+            }
+        }
+        return true;
+    }
+    return false;
+};
 exports.Properties = {
     ROW: "rows",
     UPLOAD: "storage",
@@ -21,21 +33,21 @@ exports.Properties = {
     USER: "users",
     AUTOMATION: "automationRuns",
     APPS: "apps",
-    EMAILS: "emails",
+    EMAILS: "emails", // doesn't work yet
 };
-function getUsageQuotaDoc(db) {
-    return __awaiter(this, void 0, void 0, function* () {
-        let quota;
-        try {
-            quota = yield db.get(StaticDatabases.PLATFORM_INFO.docs.usageQuota);
-        }
-        catch (err) {
-            // doc doesn't exist. Create it
-            quota = yield db.post(generateNewUsageQuotaDoc());
-        }
-        return quota;
-    });
-}
+exports.getUsageQuotaDoc = (db) => __awaiter(void 0, void 0, void 0, function* () {
+    let quota;
+    try {
+        quota = yield db.get(StaticDatabases.GLOBAL.docs.usageQuota);
+    }
+    catch (err) {
+        // doc doesn't exist. Create it
+        quota = generateNewUsageQuotaDoc();
+        const response = yield db.put(quota);
+        quota._rev = response.rev;
+    }
+    return quota;
+});
 /**
  * Given a specified tenantId this will add to the usage object for the specified property.
  * @param {string} property The property which is to be added to (within the nested usageQuota object).
@@ -43,27 +55,28 @@ function getUsageQuotaDoc(db) {
  * @returns {Promise<void>} When this completes the API key will now be up to date - the quota period may have
  * also been reset after this call.
  */
-exports.update = (property, usage) => __awaiter(void 0, void 0, void 0, function* () {
-    if (!env.USE_QUOTAS) {
+exports.update = (property, usage, opts = { dryRun: false }) => __awaiter(void 0, void 0, void 0, function* () {
+    if (!exports.useQuotas()) {
         return;
     }
     try {
         const db = getGlobalDB();
-        const quota = yield getUsageQuotaDoc(db);
-        // Check if the quota needs reset
-        if (Date.now() >= quota.quotaReset) {
-            quota.quotaReset = getNewQuotaReset();
-            for (let prop of Object.keys(quota.usageQuota)) {
-                quota.usageQuota[prop] = 0;
-            }
-        }
+        const quota = yield exports.getUsageQuotaDoc(db);
         // increment the quota
         quota.usageQuota[property] += usage;
-        if (quota.usageQuota[property] > quota.usageLimits[property]) {
+        if (quota.usageQuota[property] > quota.usageLimits[property] &&
+            usage > 0 // allow for decrementing usage when the quota is already exceeded
+        ) {
             throw new Error(`You have exceeded your usage quota of ${quota.usageLimits[property]} ${property}.`);
         }
+        if (quota.usageQuota[property] < 0) {
+            // never go negative if the quota has previously been exceeded
+            quota.usageQuota[property] = 0;
+        }
         // update the usage quotas
-        yield db.put(quota);
+        if (!opts.dryRun) {
+            yield db.put(quota);
+        }
     }
     catch (err) {
         console.error(`Error updating usage quotas for ${property}`, err);

package/dist/utilities/usageQuota/rows.js

@@ -0,0 +1,77 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+const { getRowParams, USER_METDATA_PREFIX } = require("../../db/utils");
+const CouchDB = require("../../db");
+const { isDevAppID, getDevelopmentAppID } = require("@budibase/backend-core/db");
+const ROW_EXCLUSIONS = [USER_METDATA_PREFIX];
+const getAppPairs = appIds => {
+    // collect the app ids into dev / prod pairs
+    // keyed by the dev app id
+    const pairs = {};
+    for (let appId of appIds) {
+        const devId = getDevelopmentAppID(appId);
+        if (!pairs[devId]) {
+            pairs[devId] = {};
+        }
+        if (isDevAppID(appId)) {
+            pairs[devId].devId = appId;
+        }
+        else {
+            pairs[devId].prodId = appId;
+        }
+    }
+    return pairs;
+};
+const getAppRows = (appId) => __awaiter(void 0, void 0, void 0, function* () {
+    const appDb = new CouchDB(appId);
+    const response = yield appDb.allDocs(getRowParams(null, null, {
+        include_docs: false,
+    }));
+    return response.rows
+        .map(r => r.id)
+        .filter(id => {
+        for (let exclusion of ROW_EXCLUSIONS) {
+            if (id.startsWith(exclusion)) {
+                return false;
+            }
+        }
+        return true;
+    });
+});
+/**
+ * Return a set of all rows in the given app ids.
+ * The returned rows will be unique on a per dev/prod app basis.
+ * Rows duplicates may exist across apps due to data import so they are not filtered out.
+ */
+exports.getUniqueRows = (appIds) => __awaiter(void 0, void 0, void 0, function* () {
+    let uniqueRows = [];
+    const pairs = getAppPairs(appIds);
+    for (let pair of Object.values(pairs)) {
+        let appRows = [];
+        for (let appId of [pair.devId, pair.prodId]) {
+            if (!appId) {
+                continue;
+            }
+            try {
+                appRows.push(yield getAppRows(appId));
+            }
+            catch (e) {
+                console.error(e);
+                // don't error out if we can't count the app rows, just continue
+            }
+        }
+        // ensure uniqueness on a per app pair basis
+        // this can't be done on all rows because app import results in
+        // duplicate row ids across apps
+        uniqueRows = uniqueRows.concat(...new Set(appRows));
+    }
+    return uniqueRows;
+});

package/dist/utilities/usageQuota/usageQuoteReset.js

@@ -0,0 +1,16 @@
+"use strict";
+// UNUSED CODE
+// Preserved for future use
+/* eslint-disable no-unused-vars */
+function getNewQuotaReset() {
+    return Date.now() + 2592000000;
+}
+function resetQuotasIfRequired(quota) {
+    // Check if the quota needs reset
+    if (Date.now() >= quota.quotaReset) {
+        quota.quotaReset = getNewQuotaReset();
+        for (let prop of Object.keys(quota.usageQuota)) {
+            quota.usageQuota[prop] = 0;
+        }
+    }
+}

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "1.0.46-alpha.0",
+  "version": "1.0.46-alpha.4",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -70,9 +70,9 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "^10.0.3",
-    "@budibase/backend-core": "^1.0.46-alpha.0",
-    "@budibase/client": "^1.0.46-alpha.0",
-    "@budibase/string-templates": "^1.0.46-alpha.0",
+    "@budibase/backend-core": "^1.0.46-alpha.4",
+    "@budibase/client": "^1.0.46-alpha.4",
+    "@budibase/string-templates": "^1.0.46-alpha.4",
     "@bull-board/api": "^3.7.0",
     "@bull-board/koa": "^3.7.0",
     "@elastic/elasticsearch": "7.10.0",
@@ -165,5 +165,5 @@
   "optionalDependencies": {
     "oracledb": "^5.3.0"
   },
-  "gitHead": "364d02d883291374e72a4a3d8ebb62c1f4ed185b"
+  "gitHead": "965525d1b1437e7c94eaff0b21f2143e3f2f0ad7"
 }

package/src/api/controllers/static/index.js

@@ -17,6 +17,8 @@ const { clientLibraryPath } = require("../../../utilities")
 const { upload } = require("../../../utilities/fileSystem")
 const { attachmentsRelativeURL } = require("../../../utilities")
 const { DocumentTypes } = require("../../../db/utils")
+const AWS = require("aws-sdk")
+const AWS_REGION = env.AWS_REGION ? env.AWS_REGION : "eu-west-1"
 
 async function prepareUpload({ s3Key, bucket, metadata, file }) {
   const response = await upload({
@@ -104,3 +106,51 @@ exports.serveClientLibrary = async function (ctx) {
     root: join(NODE_MODULES_PATH, "@budibase", "client", "dist"),
   })
 }
+
+exports.getSignedUploadURL = async function (ctx) {
+  const database = new CouchDB(ctx.appId)
+
+  // Ensure datasource is valid
+  let datasource
+  try {
+    const { datasourceId } = ctx.params
+    datasource = await database.get(datasourceId)
+    if (!datasource) {
+      ctx.throw(400, "The specified datasource could not be found")
+    }
+  } catch (error) {
+    ctx.throw(400, "The specified datasource could not be found")
+  }
+
+  // Ensure we aren't using a custom endpoint
+  if (datasource?.config?.endpoint) {
+    ctx.throw(400, "S3 datasources with custom endpoints are not supported")
+  }
+
+  // Determine type of datasource and generate signed URL
+  let signedUrl
+  let publicUrl
+  if (datasource.source === "S3") {
+    const { bucket, key } = ctx.request.body || {}
+    if (!bucket || !key) {
+      ctx.throw(400, "bucket and key values are required")
+      return
+    }
+    try {
+      const s3 = new AWS.S3({
+        region: AWS_REGION,
+        accessKeyId: datasource?.config?.accessKeyId,
+        secretAccessKey: datasource?.config?.secretAccessKey,
+        apiVersion: "2006-03-01",
+        signatureVersion: "v4",
+      })
+      const params = { Bucket: bucket, Key: key }
+      signedUrl = s3.getSignedUrl("putObject", params)
+      publicUrl = `https://${bucket}.s3.${AWS_REGION}.amazonaws.com/${key}`
+    } catch (error) {
+      ctx.throw(400, error)
+    }
+  }
+
+  ctx.body = { signedUrl, publicUrl }
+}

package/src/api/controllers/table/internal.js

@@ -8,6 +8,7 @@ const {
   getTable,
   handleDataImport,
 } = require("./utils")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.save = async function (ctx) {
   const appId = ctx.appId
@@ -119,6 +120,7 @@ exports.destroy = async function (ctx) {
     })
   )
   await db.bulkDocs(rows.rows.map(row => ({ ...row.doc, _deleted: true })))
+  await usageQuota.update(usageQuota.Properties.ROW, -rows.rows.length)
 
   // update linked rows
   await linkRows.updateLinks({

package/src/api/controllers/table/utils.js

@@ -16,6 +16,7 @@ const {
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
+const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -112,7 +113,11 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
+    dryRun: true,
+  })
   await db.bulkDocs(finalData)
+  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table

package/src/api/routes/static.js

@@ -46,5 +46,10 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
+  .post(
+    "/api/attachments/:datasourceId/url",
+    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
+    controller.getSignedUploadURL
+  )
 
 module.exports = router

package/src/api/routes/tests/static.spec.js

@@ -0,0 +1,98 @@
+jest.mock("node-fetch")
+jest.mock("aws-sdk", () => ({
+  config: {
+    update: jest.fn(),
+  },
+  DynamoDB: {
+    DocumentClient: jest.fn(),
+  },
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(() => {
+      return "my-url"
+    }),
+  })),
+}))
+
+const setup = require("./utilities")
+
+describe("/attachments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("generateSignedUrls", () => {
+    let datasource
+
+    beforeEach(async () => {
+      datasource = await config.createDatasource({
+        datasource: {
+          type: "datasource",
+          name: "Test",
+          source: "S3",
+          config: {},
+        },
+      })
+    })
+
+    it("should be able to generate a signed upload URL", async () => {
+      const bucket = "foo"
+      const key = "bar"
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({ bucket, key })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.signedUrl).toEqual("my-url")
+      expect(res.body.publicUrl).toEqual(
+        `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
+      )
+    })
+
+    it("should handle an invalid datasource ID", async () => {
+      const res = await request
+        .post(`/api/attachments/foo/url`)
+        .send({
+          bucket: "foo",
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "The specified datasource could not be found"
+      )
+    })
+
+    it("should require a bucket parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: undefined,
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+
+    it("should require a key parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: "foo",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+  })
+})

package/src/automations/steps/createRow.js

@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -83,10 +82,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
-      await usage.update(usage.Properties.ROW, 1)
-    }
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,

package/src/automations/steps/deleteRow.js

@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
-      await usage.update(usage.Properties.ROW, -1)
-    }
+    await usage.update(usage.Properties.ROW, -1)
     await rowController.destroy(ctx)
     return {
       response: ctx.body,

package/src/environment.js

@@ -38,6 +38,7 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
+  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

package/src/middleware/tests/usageQuota.spec.js

@@ -1,11 +1,5 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
-jest.mock("../../environment", () => ({
-  isTest: () => true,
-  isProd: () => false,
-  isDev: () => true,
-  _set: () => {},
-}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -29,9 +23,10 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/rows"
+        url: "/applications"
       }
     }
+    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -113,7 +108,6 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
-    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -121,20 +115,20 @@ describe("usageQuota middleware", () => {
     expect(config.next).toHaveBeenCalled()
   })
 
-  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-    config.setUrl("/upload")
-    config.setProd(true)
-    config.setFiles([
-      {
-        size: 100
-      },
-      {
-        size: 10000
-      },
-    ])
-    await config.executeMiddleware()
-
-    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-    expect(config.next).toHaveBeenCalled()
-  })
+  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+  //   config.setUrl("/upload")
+  //   config.setProd(true)
+  //   config.setFiles([
+  //     {
+  //       size: 100
+  //     },
+  //     {
+  //       size: 10000
+  //     },
+  //   ])
+  //   await config.executeMiddleware()
+
+  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+  //   expect(config.next).toHaveBeenCalled()
+  // })
 })