@budibase/server 2.1.40-alpha.6 → 2.1.42

package/src/integrations/s3.ts

@@ -1,12 +1,5 @@
-import {
-  Integration,
-  QueryType,
-  IntegrationBase,
-  DatasourceFieldType,
-} from "@budibase/types"
-
+import { Integration, QueryType, IntegrationBase } from "@budibase/types"
 const AWS = require("aws-sdk")
-const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -47,106 +40,16 @@ const SCHEMA: Integration = {
     },
   },
   query: {
-    create: {
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          display: "New Bucket",
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        location: {
-          required: true,
-          default: "us-east-1",
-          type: DatasourceFieldType.STRING,
-        },
-        grantFullControl: {
-          display: "Grant full control",
-          type: DatasourceFieldType.STRING,
-        },
-        grantRead: {
-          display: "Grant read",
-          type: DatasourceFieldType.STRING,
-        },
-        grantReadAcp: {
-          display: "Grant read ACP",
-          type: DatasourceFieldType.STRING,
-        },
-        grantWrite: {
-          display: "Grant write",
-          type: DatasourceFieldType.STRING,
-        },
-        grantWriteAcp: {
-          display: "Grant write ACP",
-          type: DatasourceFieldType.STRING,
-        },
-      },
-    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        delimiter: {
-          type: DatasourceFieldType.STRING,
-        },
-        marker: {
-          type: DatasourceFieldType.STRING,
-        },
-        maxKeys: {
-          type: DatasourceFieldType.NUMBER,
-          display: "Max Keys",
-        },
-        prefix: {
-          type: DatasourceFieldType.STRING,
-        },
-      },
-    },
-    readCsv: {
-      displayName: "Read CSV",
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        key: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-      },
-    },
-    delete: {
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        delete: {
-          type: DatasourceFieldType.JSON,
+          type: "string",
           required: true,
         },
       },
     },
   },
-  extra: {
-    acl: {
-      required: false,
-      displayName: "ACL",
-      type: DatasourceFieldType.LIST,
-      data: {
-        create: [
-          "private",
-          "public-read",
-          "public-read-write",
-          "authenticated-read",
-        ],
-      },
-    },
-  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -164,93 +67,14 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async create(query: {
-    bucket: string
-    location: string
-    grantFullControl: string
-    grantRead: string
-    grantReadAcp: string
-    grantWrite: string
-    grantWriteAcp: string
-    extra: {
-      acl: string
-    }
-  }) {
-    let params: any = {
-      Bucket: query.bucket,
-      ACL: query.extra?.acl,
-      GrantFullControl: query.grantFullControl,
-      GrantRead: query.grantRead,
-      GrantReadACP: query.grantReadAcp,
-      GrantWrite: query.grantWrite,
-      GrantWriteACP: query.grantWriteAcp,
-    }
-    if (query.location) {
-      params["CreateBucketConfiguration"] = {
-        LocationConstraint: query.location,
-      }
-    }
-    return await this.client.createBucket(params).promise()
-  }
-
-  async read(query: {
-    bucket: string
-    delimiter: string
-    expectedBucketOwner: string
-    marker: string
-    maxKeys: number
-    prefix: string
-  }) {
+  async read(query: { bucket: string }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
-        Delimiter: query.delimiter,
-        Marker: query.marker,
-        MaxKeys: query.maxKeys,
-        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
-
-  async readCsv(query: { bucket: string; key: string }) {
-    const stream = this.client
-      .getObject({
-        Bucket: query.bucket,
-        Key: query.key,
-      })
-      .createReadStream()
-
-    let csvError = false
-    return new Promise((resolve, reject) => {
-      stream.on("error", (err: Error) => {
-        reject(err)
-      })
-      const response = csv()
-        .fromStream(stream)
-        .on("error", () => {
-          csvError = true
-        })
-      stream.on("finish", () => {
-        resolve(response)
-      })
-    }).catch(err => {
-      if (csvError) {
-        throw new Error("Could not read CSV")
-      } else {
-        throw err
-      }
-    })
-  }
-
-  async delete(query: { bucket: string; delete: string }) {
-    return await this.client
-      .deleteObjects({
-        Bucket: query.bucket,
-        Delete: JSON.parse(query.delete),
-      })
-      .promise()
-  }
 }
 
 export default {

package/src/integrations/tests/s3.spec.ts

@@ -18,95 +18,11 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    await config.integration.read({
+    const response = await config.integration.read({
       bucket: "test",
-      delimiter: "/",
-      marker: "file.txt",
-      maxKeys: 999,
-      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
-      Delimiter: "/",
-      Marker: "file.txt",
-      MaxKeys: 999,
-      Prefix: "directory/",
-    })
-  })
-
-  it("calls the create method with the correct params", async () => {
-    await config.integration.create({
-      bucket: "test",
-      location: "af-south-1",
-      grantFullControl: "me",
-      grantRead: "him",
-      grantReadAcp: "her",
-      grantWrite: "she",
-      grantWriteAcp: "he",
-      objectLockEnabledForBucket: true,
-      extra: {
-        acl: "private",
-      },
-    })
-    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
-      Bucket: "test",
-      CreateBucketConfiguration: {
-        LocationConstraint: "af-south-1",
-      },
-      GrantFullControl: "me",
-      GrantRead: "him",
-      GrantReadACP: "her",
-      GrantWrite: "she",
-      GrantWriteACP: "he",
-      ACL: "private",
-    })
-  })
-
-  it("does not add undefined location constraint when calling the create method", async () => {
-    await config.integration.create({
-      bucket: "test",
-    })
-    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
-      Bucket: "test",
-      GrantFullControl: undefined,
-      GrantRead: undefined,
-      GrantReadACP: undefined,
-      GrantWrite: undefined,
-      GrantWriteACP: undefined,
-      ACL: undefined,
-    })
-  })
-
-  it("calls the delete method with the correct params ", async () => {
-    await config.integration.delete({
-      bucket: "test",
-      delete: `{
-        "Objects": [
-          {
-         "Key": "HappyFace.jpg", 
-         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
-        }, 
-          {
-         "Key": "HappyFace.jpg", 
-         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
-        }
-       ]
-      }`,
-    })
-    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
-      Bucket: "test",
-      Delete: {
-        Objects: [
-          {
-            Key: "HappyFace.jpg",
-            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
-          },
-          {
-            Key: "HappyFace.jpg",
-            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
-          },
-        ],
-      },
     })
   })
 })

package/src/integrations/utils.ts

@@ -230,7 +230,7 @@ function shouldCopySpecialColumn(
   const fetchedIsNumber =
     !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
   return (
-    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
+    specialTypes.indexOf(column.type) !== -1 ||
     (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
   )
 }
@@ -292,11 +292,7 @@ export function finaliseExternalTables(
     if (table.primary == null || table.primary.length === 0) {
       errors[name] = BuildSchemaErrors.NO_KEY
       continue
-    } else if (
-      schemaFields.find(field =>
-        invalidColumns.includes(field as InvalidColumns)
-      )
-    ) {
+    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
       errors[name] = BuildSchemaErrors.INVALID_COLUMN
       continue
     }

package/src/middleware/appInfo.js

@@ -0,0 +1,19 @@
+const { isDevAppID, isProdAppID } = require("../db/utils")
+
+exports.AppType = {
+  DEV: "dev",
+  PROD: "prod",
+}
+
+exports.middleware =
+  ({ appType } = {}) =>
+  (ctx, next) => {
+    const appId = ctx.appId
+    if (appType === exports.AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === exports.AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }

package/src/middleware/authorized.ts

@@ -1,13 +1,23 @@
-import { roles, permissions, auth, context } from "@budibase/backend-core"
-import { Role } from "@budibase/types"
-import builderMiddleware from "./builder"
-import { isWebhookEndpoint } from "./utils"
+import {
+  getUserRoleHierarchy,
+  getRequiredResourceRole,
+  BUILTIN_ROLE_IDS,
+} from "@budibase/backend-core/roles"
+const {
+  PermissionTypes,
+  PermissionLevels,
+  doesHaveBasePermission,
+} = require("@budibase/backend-core/permissions")
+const builderMiddleware = require("./builder")
+const { isWebhookEndpoint } = require("./utils")
+const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
+const { getAppId } = require("@budibase/backend-core/context")
 
 function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
-const csrf = auth.buildCsrfMiddleware()
+const csrf = buildCsrfMiddleware()
 
 /**
  * Apply authorization to the requested resource:
@@ -23,7 +33,7 @@ const checkAuthorized = async (
 ) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+  const isBuilderApi = permType === PermissionTypes.BUILDER
   if (isBuilderApi && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
@@ -41,10 +51,10 @@ const checkAuthorizedResource = async (
   permLevel: any
 ) => {
   // get the user's roles
-  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
+  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await getUserRoleHierarchy(roleId, {
     idOnly: false,
-  })) as Role[]
+  })) as { _id: string }[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {
@@ -56,9 +66,7 @@ const checkAuthorizedResource = async (
       ctx.throw(403, permError)
     }
     // fallback to the base permissions when no resource roles are found
-  } else if (
-    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
-  ) {
+  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
     ctx.throw(403, permError)
   }
 }
@@ -83,22 +91,21 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     let resourceRoles: any = []
     let otherLevelRoles: any = []
     const otherLevel =
-      permLevel === permissions.PermissionLevel.READ
-        ? permissions.PermissionLevel.WRITE
-        : permissions.PermissionLevel.READ
-    const appId = context.getAppId()
+      permLevel === PermissionLevels.READ
+        ? PermissionLevels.WRITE
+        : PermissionLevels.READ
+    const appId = getAppId()
     if (appId && hasResource(ctx)) {
-      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
+      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
       if (opts && opts.schema) {
-        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
+        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
       }
     }
 
     // if the resource is public, proceed
     if (
-      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
-      (otherLevelRoles &&
-        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
+      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
     ) {
       return next()
     }

package/src/middleware/{builder.ts → builder.js}

@@ -1,16 +1,14 @@
-import {
-  APP_DEV_PREFIX,
-  DocumentType,
-  getGlobalIDFromUserMetadataID,
-} from "../db/utils"
-import {
+const { APP_DEV_PREFIX } = require("../db/utils")
+const {
   doesUserHaveLock,
   updateLock,
   checkDebounce,
   setDebounce,
-} from "../utilities/redis"
-import { db as dbCore, cache, permissions } from "@budibase/backend-core"
-import { BBContext, Database } from "@budibase/types"
+} = require("../utilities/redis")
+const { doWithDB } = require("@budibase/backend-core/db")
+const { DocumentType, getGlobalIDFromUserMetadataID } = require("../db/utils")
+const { PermissionTypes } = require("@budibase/backend-core/permissions")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -23,11 +21,11 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx: BBContext) {
+async function checkDevAppLocks(ctx) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
-  if (!ctx.user?._id && !ctx.user?.userId) {
+  if (!ctx.user._id && !ctx.user.userId) {
     return
   }
 
@@ -43,34 +41,34 @@ async function checkDevAppLocks(ctx: BBContext) {
   await updateLock(appId, ctx.user)
 }
 
-async function updateAppUpdatedAt(ctx: BBContext) {
+async function updateAppUpdatedAt(ctx) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  await dbCore.doWithDB(appId, async (db: Database) => {
+  await doWithDB(appId, async db => {
     const metadata = await db.get(DocumentType.APP_METADATA)
     metadata.updatedAt = new Date().toISOString()
 
-    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId)
 
     const response = await db.put(metadata)
     metadata._rev = response.rev
-    await cache.app.invalidateAppMetadata(appId, metadata)
+    await appCache.invalidateAppMetadata(appId, metadata)
     // set a new debounce record with a short TTL
     await setDebounce(appId, DEBOUNCE_TIME_SEC)
   })
 }
 
-export = async function builder(ctx: BBContext, permType: string) {
+module.exports = async (ctx, permType) => {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+  const isBuilderApi = permType === PermissionTypes.BUILDER
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"

package/src/middleware/{currentapp.ts → currentapp.js}

@@ -1,26 +1,29 @@
-import {
-  utils,
-  constants,
-  roles,
-  db as dbCore,
-  tenancy,
-  context,
-} from "@budibase/backend-core"
-import { generateUserMetadataID, isDevAppID } from "../db/utils"
-import { getCachedSelf } from "../utilities/global"
-import env from "../environment"
-import { isWebhookEndpoint } from "./utils"
-import { BBContext } from "@budibase/types"
+const {
+  getAppIdFromCtx,
+  setCookie,
+  getCookie,
+  clearCookie,
+} = require("@budibase/backend-core/utils")
+const { Cookies, Headers } = require("@budibase/backend-core/constants")
+const { getRole } = require("@budibase/backend-core/roles")
+const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
+const { generateUserMetadataID, isDevAppID } = require("../db/utils")
+const { dbExists } = require("@budibase/backend-core/db")
+const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
+const { getCachedSelf } = require("../utilities/global")
+const env = require("../environment")
+const { isWebhookEndpoint } = require("./utils")
+const { doInAppContext } = require("@budibase/backend-core/context")
 
-export = async (ctx: BBContext, next: any) => {
+module.exports = async (ctx, next) => {
   // try to get the appID from the request
-  let requestAppId = await utils.getAppIdFromCtx(ctx)
+  let requestAppId = await getAppIdFromCtx(ctx)
   // get app cookie if it exists
-  let appCookie: { appId?: string } | undefined
+  let appCookie = null
   try {
-    appCookie = utils.getCookie(ctx, constants.Cookie.CurrentApp)
+    appCookie = getCookie(ctx, Cookies.CurrentApp)
   } catch (err) {
-    utils.clearCookie(ctx, constants.Cookie.CurrentApp)
+    clearCookie(ctx, Cookies.CurrentApp)
   }
   if (!appCookie && !requestAppId) {
     return next()
@@ -28,9 +31,9 @@ export = async (ctx: BBContext, next: any) => {
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
-    const exists = await dbCore.dbExists(appId)
+    const exists = await dbExists(appId)
     if (!exists) {
-      utils.clearCookie(ctx, constants.Cookie.CurrentApp)
+      clearCookie(ctx, Cookies.CurrentApp)
       return next()
     }
     // if the request app ID wasn't set, update it with the cookie
@@ -44,13 +47,13 @@ export = async (ctx: BBContext, next: any) => {
       !isWebhookEndpoint(ctx) &&
       (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
     ) {
-      utils.clearCookie(ctx, constants.Cookie.CurrentApp)
+      clearCookie(ctx, Cookies.CurrentApp)
       return ctx.redirect("/")
     }
   }
 
-  let appId: string | undefined,
-    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+  let appId,
+    roleId = BUILTIN_ROLE_IDS.PUBLIC
   if (!ctx.user) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
@@ -65,20 +68,16 @@ export = async (ctx: BBContext, next: any) => {
     const isBuilder =
       globalUser && globalUser.builder && globalUser.builder.global
     const isDevApp = appId && isDevAppID(appId)
-    const roleHeader =
-      ctx.request &&
-      (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string)
+    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
     if (isBuilder && isDevApp && roleHeader) {
       // Ensure the role is valid by ensuring a definition exists
       try {
-        if (roleHeader) {
-          await roles.getRole(roleHeader)
-          roleId = roleHeader
+        await getRole(roleHeader)
+        roleId = roleHeader
 
-          // Delete admin and builder flags so that the specified role is honoured
-          delete ctx.user.builder
-          delete ctx.user.admin
-        }
+        // Delete admin and builder flags so that the specified role is honoured
+        delete ctx.user.builder
+        delete ctx.user.admin
       } catch (error) {
         // Swallow error and do nothing
       }
@@ -90,7 +89,7 @@ export = async (ctx: BBContext, next: any) => {
     return next()
   }
 
-  return context.doInAppContext(appId, async () => {
+  return doInAppContext(appId, async () => {
     let skipCookie = false
     // if the user not in the right tenant then make sure they have no permissions
     // need to judge this only based on the request app ID,
@@ -98,14 +97,14 @@ export = async (ctx: BBContext, next: any) => {
       env.MULTI_TENANCY &&
       ctx.user &&
       requestAppId &&
-      !tenancy.isUserInAppTenant(requestAppId, ctx.user)
+      !isUserInAppTenant(requestAppId, ctx.user)
     ) {
       // don't error, simply remove the users rights (they are a public user)
       delete ctx.user.builder
       delete ctx.user.admin
       delete ctx.user.roles
       ctx.isAuthenticated = false
-      roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
+      roleId = BUILTIN_ROLE_IDS.PUBLIC
       skipCookie = true
     }
 
@@ -113,17 +112,15 @@ export = async (ctx: BBContext, next: any) => {
     if (roleId) {
       ctx.roleId = roleId
       const globalId = ctx.user ? ctx.user._id : undefined
-      const userId = ctx.user
-        ? generateUserMetadataID(ctx.user._id!)
-        : undefined
+      const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
       ctx.user = {
-        ...ctx.user!,
+        ...ctx.user,
         // override userID with metadata one
         _id: userId,
         userId,
         globalId,
         roleId,
-        role: await roles.getRole(roleId),
+        role: await getRole(roleId),
       }
     }
     if (
@@ -132,7 +129,7 @@ export = async (ctx: BBContext, next: any) => {
         appCookie.appId !== requestAppId) &&
       !skipCookie
     ) {
-      utils.setCookie(ctx, { appId }, constants.Cookie.CurrentApp)
+      setCookie(ctx, { appId }, Cookies.CurrentApp)
     }
 
     return next()