@budibase/server 1.0.49 → 1.0.50-alpha.3

package/src/api/routes/tests/routing.spec.js

@@ -1,10 +1,15 @@
 const setup = require("./utilities")
 const { basicScreen } = setup.structures
-const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const { checkBuilderEndpoint, runInProd } = require("./utilities/TestFunctions")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { doInAppContext } = require("@budibase/backend-core/context")
 
 const route = "/test"
 
+// there are checks which are disabled in test env,
+// these checks need to be enabled for this test
+
+
 describe("/routing", () => {
   let request = setup.getRequest()
   let config = setup.getConfig()
@@ -26,20 +31,24 @@ describe("/routing", () => {
 
   describe("fetch", () => {
     it("prevents a public user from accessing development app", async () => {
-      await request
-        .get(`/api/routing/client`)
-        .set(config.publicHeaders({ prodApp: false }))
-        .expect(302)
+      await runInProd(() => {
+        return request
+          .get(`/api/routing/client`)
+          .set(config.publicHeaders({ prodApp: false }))
+          .expect(302)
+      })
     })
 
     it("prevents a non builder from accessing development app", async () => {
-      await request
-        .get(`/api/routing/client`)
-        .set(await config.roleHeaders({
-          roleId: BUILTIN_ROLE_IDS.BASIC,
-          prodApp: false
-        }))
-        .expect(302)
+      await runInProd(async () => {
+        return request
+          .get(`/api/routing/client`)
+          .set(await config.roleHeaders({
+            roleId: BUILTIN_ROLE_IDS.BASIC,
+            prodApp: false
+          }))
+          .expect(302)
+      })
     })
     it("returns the correct routing for basic user", async () => {
       const res = await request

package/src/api/routes/tests/row.spec.js

@@ -1,6 +1,7 @@
 const { outputProcessing } = require("../../../utilities/rowProcessor")
 const setup = require("./utilities")
 const { basicRow } = setup.structures
+const { doInAppContext } = require("@budibase/backend-core/context")
 
 // mock the fetch for the search system
 jest.mock("node-fetch")
@@ -387,10 +388,12 @@ describe("/rows", () => {
       })
       // the environment needs configured for this
       await setup.switchToSelfHosted(async () => {
-        const enriched = await outputProcessing({ appId: config.getAppId() }, table, [row])
-        expect(enriched[0].attachment[0].url).toBe(
-          `/prod-budi-app-assets/${config.getAppId()}/attachments/test/thing.csv`
-        )
+        doInAppContext(config.getAppId(), async () => {
+          const enriched = await outputProcessing(table, [row])
+          expect(enriched[0].attachment[0].url).toBe(
+            `/prod-budi-app-assets/${config.getAppId()}/attachments/test/thing.csv`
+          )
+        })
       })
     })
   })

package/src/api/routes/tests/static.spec.js

@@ -0,0 +1,98 @@
+jest.mock("node-fetch")
+jest.mock("aws-sdk", () => ({
+  config: {
+    update: jest.fn(),
+  },
+  DynamoDB: {
+    DocumentClient: jest.fn(),
+  },
+  S3: jest.fn(() => ({
+    getSignedUrl: jest.fn(() => {
+      return "my-url"
+    }),
+  })),
+}))
+
+const setup = require("./utilities")
+
+describe("/attachments", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("generateSignedUrls", () => {
+    let datasource
+
+    beforeEach(async () => {
+      datasource = await config.createDatasource({
+        datasource: {
+          type: "datasource",
+          name: "Test",
+          source: "S3",
+          config: {},
+        },
+      })
+    })
+
+    it("should be able to generate a signed upload URL", async () => {
+      const bucket = "foo"
+      const key = "bar"
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({ bucket, key })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.signedUrl).toEqual("my-url")
+      expect(res.body.publicUrl).toEqual(
+        `https://${bucket}.s3.eu-west-1.amazonaws.com/${key}`
+      )
+    })
+
+    it("should handle an invalid datasource ID", async () => {
+      const res = await request
+        .post(`/api/attachments/foo/url`)
+        .send({
+          bucket: "foo",
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual(
+        "The specified datasource could not be found"
+      )
+    })
+
+    it("should require a bucket parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: undefined,
+          key: "bar",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+
+    it("should require a key parameter", async () => {
+      const res = await request
+        .post(`/api/attachments/${datasource._id}/url`)
+        .send({
+          bucket: "foo",
+        })
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(400)
+      expect(res.body.message).toEqual("bucket and key values are required")
+    })
+  })
+})

package/src/api/routes/tests/utilities/TestFunctions.js

@@ -1,9 +1,10 @@
 const rowController = require("../../../controllers/row")
 const appController = require("../../../controllers/application")
-const CouchDB = require("../../../../db")
 const { AppStatus } = require("../../../../db/utils")
 const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
 const { TENANT_ID } = require("../../../../tests/utilities/structures")
+const { getAppDB, doInAppContext } = require("@budibase/backend-core/context")
+const env = require("../../../../environment")
 
 function Request(appId, params) {
   this.appId = appId
@@ -11,9 +12,15 @@ function Request(appId, params) {
   this.request = {}
 }
 
+function runRequest(appId, controlFunc, request) {
+  return doInAppContext(appId, async () => {
+    return controlFunc(request)
+  })
+}
+
 exports.getAllTableRows = async config => {
   const req = new Request(config.appId, { tableId: config.table._id })
-  await rowController.fetch(req)
+  await runRequest(config.appId, rowController.fetch, req)
   return req.body
 }
 
@@ -26,14 +33,17 @@ exports.clearAllApps = async (tenantId = TENANT_ID) => {
   }
   for (let app of apps) {
     const { appId } = app
-    await appController.delete(new Request(null, { appId }))
+    const req = new Request(null, { appId })
+    await runRequest(appId, appController.delete, req)
   }
 }
 
 exports.clearAllAutomations = async config => {
   const automations = await config.getAllAutomations()
   for (let auto of automations) {
-    await config.deleteAutomation(auto)
+    await doInAppContext(config.appId, async () => {
+      await config.deleteAutomation(auto)
+    })
   }
 }
 
@@ -96,20 +106,32 @@ exports.checkPermissionsEndpoint = async ({
     .expect(403)
 }
 
-exports.getDB = config => {
-  return new CouchDB(config.getAppId())
+exports.getDB = () => {
+  return getAppDB()
 }
 
 exports.testAutomation = async (config, automation) => {
-  return await config.request
-    .post(`/api/automations/${automation._id}/test`)
-    .send({
-      row: {
-        name: "Test",
-        description: "TEST",
-      },
-    })
-    .set(config.defaultHeaders())
-    .expect("Content-Type", /json/)
-    .expect(200)
+  return runRequest(automation.appId, async () => {
+    return await config.request
+      .post(`/api/automations/${automation._id}/test`)
+      .send({
+        row: {
+          name: "Test",
+          description: "TEST",
+        },
+      })
+      .set(config.defaultHeaders())
+      .expect("Content-Type", /json/)
+      .expect(200)
+  })
+}
+
+exports.runInProd = async func => {
+  const nodeEnv = env.NODE_ENV
+  const workerId = env.JEST_WORKER_ID
+  env._set("NODE_ENV", "PRODUCTION")
+  env._set("JEST_WORKER_ID", null)
+  await func()
+  env._set("NODE_ENV", nodeEnv)
+  env._set("JEST_WORKER_ID", workerId)
 }

package/src/app.ts

@@ -1,7 +1,7 @@
 // need to load environment first
 import { ExtendableContext } from "koa"
 
-const env = require("./environment")
+import * as env from "./environment"
 const CouchDB = require("./db")
 require("@budibase/backend-core").init(CouchDB)
 const Koa = require("koa")
@@ -16,6 +16,7 @@ const Sentry = require("@sentry/node")
 const fileSystem = require("./utilities/fileSystem")
 const bullboard = require("./automations/bullboard")
 const redis = require("./utilities/redis")
+import * as migrations from "./migrations"
 
 const app = new Koa()
 
@@ -84,13 +85,25 @@ module.exports = server.listen(env.PORT || 0, async () => {
   await automations.init()
 })
 
-process.on("uncaughtException", err => {
-  console.error(err)
+const shutdown = () => {
   server.close()
   server.destroy()
+}
+
+process.on("uncaughtException", err => {
+  console.error(err)
+  shutdown()
 })
 
 process.on("SIGTERM", () => {
-  server.close()
-  server.destroy()
+  shutdown()
 })
+
+// run migrations on startup if not done via http
+// not recommended in a clustered environment
+if (!env.HTTP_MIGRATIONS) {
+  migrations.migrate().catch(err => {
+    console.error("Error performing migrations. Exiting.\n", err)
+    shutdown()
+  })
+}

package/src/automations/automationUtils.js

@@ -53,13 +53,12 @@ exports.cleanInputValues = (inputs, schema) => {
  * the automation but is instead part of the Table/Table. This function will get the table schema and use it to instead
  * perform the cleanInputValues function on the input row.
  *
- * @param {string} appId The instance which the Table/Table is contained under.
  * @param {string} tableId The ID of the Table/Table which the schema is to be retrieved for.
  * @param {object} row The input row structure which requires clean-up after having been through template statements.
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
-exports.cleanUpRow = async (appId, tableId, row) => {
-  let table = await getTable(appId, tableId)
+exports.cleanUpRow = async (tableId, row) => {
+  let table = await getTable(tableId)
   return exports.cleanInputValues(row, { properties: table.schema })
 }
 

package/src/automations/steps/createRow.js

@@ -1,6 +1,5 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -79,14 +78,12 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     inputs.row = await automationUtils.cleanUpRow(
-      appId,
       inputs.row.tableId,
       inputs.row
     )
-    if (env.USE_QUOTAS) {
-      await usage.update(usage.Properties.ROW, 1)
-    }
+    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
     await rowController.save(ctx)
+    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,

package/src/automations/steps/deleteRow.js

@@ -1,5 +1,4 @@
 const rowController = require("../../api/controllers/row")
-const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -74,9 +73,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    if (env.isProd()) {
-      await usage.update(usage.Properties.ROW, -1)
-    }
+    await usage.update(usage.Properties.ROW, -1)
     await rowController.destroy(ctx)
     return {
       response: ctx.body,

package/src/automations/steps/updateRow.js

@@ -87,7 +87,7 @@ exports.run = async function ({ inputs, appId, emitter }) {
 
   try {
     if (tableId) {
-      inputs.row = await automationUtils.cleanUpRow(appId, tableId, inputs.row)
+      inputs.row = await automationUtils.cleanUpRow(tableId, inputs.row)
     }
     await rowController.patch(ctx)
     return {

package/src/automations/triggers.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../db")
 const emitter = require("../events/index")
 const { getAutomationParams } = require("../db/utils")
 const { coerce } = require("../utilities/rowProcessor")
@@ -9,6 +8,7 @@ const { queue } = require("./bullboard")
 const { checkTestFlag } = require("../utilities/redis")
 const utils = require("./utils")
 const env = require("../environment")
+const { doInAppContext, getAppDB } = require("@budibase/backend-core/context")
 
 const TRIGGER_DEFINITIONS = definitions
 const JOB_OPTS = {
@@ -21,39 +21,41 @@ async function queueRelevantRowAutomations(event, eventType) {
     throw `No appId specified for ${eventType} - check event emitters.`
   }
 
-  const db = new CouchDB(event.appId)
-  let automations = await db.allDocs(
-    getAutomationParams(null, { include_docs: true })
-  )
+  doInAppContext(event.appId, async () => {
+    const db = getAppDB()
+    let automations = await db.allDocs(
+      getAutomationParams(null, { include_docs: true })
+    )
 
-  // filter down to the correct event type
-  automations = automations.rows
-    .map(automation => automation.doc)
-    .filter(automation => {
-      const trigger = automation.definition.trigger
-      return trigger && trigger.event === eventType
-    })
+    // filter down to the correct event type
+    automations = automations.rows
+      .map(automation => automation.doc)
+      .filter(automation => {
+        const trigger = automation.definition.trigger
+        return trigger && trigger.event === eventType
+      })
 
-  for (let automation of automations) {
-    let automationDef = automation.definition
-    let automationTrigger = automationDef ? automationDef.trigger : {}
-    // don't queue events which are for dev apps, only way to test automations is
-    // running tests on them, in production the test flag will never
-    // be checked due to lazy evaluation (first always false)
-    if (
-      !env.ALLOW_DEV_AUTOMATIONS &&
-      isDevAppID(event.appId) &&
-      !(await checkTestFlag(automation._id))
-    ) {
-      continue
+    for (let automation of automations) {
+      let automationDef = automation.definition
+      let automationTrigger = automationDef ? automationDef.trigger : {}
+      // don't queue events which are for dev apps, only way to test automations is
+      // running tests on them, in production the test flag will never
+      // be checked due to lazy evaluation (first always false)
+      if (
+        !env.ALLOW_DEV_AUTOMATIONS &&
+        isDevAppID(event.appId) &&
+        !(await checkTestFlag(automation._id))
+      ) {
+        continue
+      }
+      if (
+        automationTrigger.inputs &&
+        automationTrigger.inputs.tableId === event.row.tableId
+      ) {
+        await queue.add({ automation, event }, JOB_OPTS)
+      }
     }
-    if (
-      automationTrigger.inputs &&
-      automationTrigger.inputs.tableId === event.row.tableId
-    ) {
-      await queue.add({ automation, event }, JOB_OPTS)
-    }
-  }
+  })
 }
 
 emitter.on("row:save", async function (event) {

package/src/automations/utils.js

@@ -6,8 +6,9 @@ const { queue } = require("./bullboard")
 const newid = require("../db/newid")
 const { updateEntityMetadata } = require("../utilities")
 const { MetadataTypes } = require("../constants")
-const { getDeployedAppID } = require("@budibase/backend-core/db")
+const { getProdAppID } = require("@budibase/backend-core/db")
 const { cloneDeep } = require("lodash/fp")
+const { getAppDB, getAppId } = require("@budibase/backend-core/context")
 
 const WH_STEP_ID = definitions.WEBHOOK.stepId
 const CRON_STEP_ID = definitions.CRON.stepId
@@ -27,7 +28,6 @@ exports.processEvent = async job => {
 
 exports.updateTestHistory = async (appId, automation, history) => {
   return updateEntityMetadata(
-    appId,
     MetadataTypes.AUTOMATION_TEST_HISTORY,
     automation._id,
     metadata => {
@@ -93,6 +93,9 @@ exports.enableCronTrigger = async (appId, automation) => {
     )
     // Assign cron job ID from bull so we can remove it later if the cron trigger is removed
     trigger.cronJobId = job.id
+    // can't use getAppDB here as this is likely to be called from dev app,
+    // but this call could be for dev app or prod app, need to just use what
+    // was passed in
     const db = new CouchDB(appId)
     const response = await db.put(automation)
     automation._id = response.id
@@ -109,7 +112,8 @@ exports.enableCronTrigger = async (appId, automation) => {
  * @returns {Promise<object|undefined>} After this is complete the new automation object may have been updated and should be
  * written to DB (this does not write to DB as it would be wasteful to repeat).
  */
-exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
+exports.checkForWebhooks = async ({ oldAuto, newAuto }) => {
+  const appId = getAppId()
   const oldTrigger = oldAuto ? oldAuto.definition.trigger : null
   const newTrigger = newAuto ? newAuto.definition.trigger : null
   const triggerChanged =
@@ -128,7 +132,7 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
     oldTrigger.webhookId
   ) {
     try {
-      let db = new CouchDB(appId)
+      let db = getAppDB()
       // need to get the webhook to get the rev
       const webhook = await db.get(oldTrigger.webhookId)
       const ctx = {
@@ -166,7 +170,7 @@ exports.checkForWebhooks = async ({ appId, oldAuto, newAuto }) => {
     // the app ID has to be development for this endpoint
     // it can only be used when building the app
     // but the trigger endpoint will always be used in production
-    const prodAppId = getDeployedAppID(appId)
+    const prodAppId = getProdAppID(appId)
     newTrigger.inputs = {
       schemaUrl: `api/webhooks/schema/${appId}/${id}`,
       triggerUrl: `api/webhooks/trigger/${prodAppId}/${id}`,

package/src/constants/index.js

@@ -61,6 +61,11 @@ exports.RelationshipTypes = {
   MANY_TO_MANY: "many-to-many",
 }
 
+exports.FormulaTypes = {
+  STATIC: "static",
+  DYNAMIC: "dynamic",
+}
+
 exports.AuthTypes = {
   APP: "app",
   BUILDER: "builder",
@@ -78,6 +83,10 @@ exports.DataSourceOperation = {
   DELETE_TABLE: "DELETE_TABLE",
 }
 
+exports.DatasourceAuthTypes = {
+  GOOGLE: "google",
+}
+
 exports.SortDirection = {
   ASCENDING: "ASCENDING",
   DESCENDING: "DESCENDING",

package/src/db/linkedRows/LinkController.js

@@ -1,4 +1,3 @@
-const CouchDB = require("../index")
 const { IncludeDocs, getLinkDocuments } = require("./linkUtils")
 const {
   generateLinkID,
@@ -7,6 +6,7 @@ const {
 } = require("../utils")
 const Sentry = require("@sentry/node")
 const { FieldTypes, RelationshipTypes } = require("../../constants")
+const { getAppDB } = require("@budibase/backend-core/context")
 
 /**
  * Creates a new link document structure which can be put to the database. It is important to
@@ -52,9 +52,8 @@ function LinkDocument(
 }
 
 class LinkController {
-  constructor({ appId, tableId, row, table, oldTable }) {
-    this._appId = appId
-    this._db = new CouchDB(appId)
+  constructor({ tableId, row, table, oldTable }) {
+    this._db = getAppDB()
     this._tableId = tableId
     this._row = row
     this._table = table
@@ -99,7 +98,6 @@ class LinkController {
    */
   getRowLinkDocs(rowId) {
     return getLinkDocuments({
-      appId: this._appId,
       tableId: this._tableId,
       rowId,
       includeDocs: IncludeDocs.INCLUDE,
@@ -111,7 +109,6 @@ class LinkController {
    */
   getTableLinkDocs() {
     return getLinkDocuments({
-      appId: this._appId,
       tableId: this._tableId,
       includeDocs: IncludeDocs.INCLUDE,
     })
@@ -230,7 +227,6 @@ class LinkController {
           if (linkedSchema.relationshipType === RelationshipTypes.ONE_TO_MANY) {
             let links = (
               await getLinkDocuments({
-                appId: this._appId,
                 tableId: field.tableId,
                 rowId: linkId,
                 includeDocs: IncludeDocs.EXCLUDE,