@budibase/server 2.1.42 → 2.1.43-alpha.1

package/src/integrations/s3.ts

@@ -1,5 +1,12 @@
-import { Integration, QueryType, IntegrationBase } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
+
 const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -40,16 +47,106 @@ const SCHEMA: Integration = {
     },
   },
   query: {
+    create: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          display: "New Bucket",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        location: {
+          required: true,
+          default: "us-east-1",
+          type: DatasourceFieldType.STRING,
+        },
+        grantFullControl: {
+          display: "Grant full control",
+          type: DatasourceFieldType.STRING,
+        },
+        grantRead: {
+          display: "Grant read",
+          type: DatasourceFieldType.STRING,
+        },
+        grantReadAcp: {
+          display: "Grant read ACP",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWrite: {
+          display: "Grant write",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWriteAcp: {
+          display: "Grant write ACP",
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: "string",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delimiter: {
+          type: DatasourceFieldType.STRING,
+        },
+        marker: {
+          type: DatasourceFieldType.STRING,
+        },
+        maxKeys: {
+          type: DatasourceFieldType.NUMBER,
+          display: "Max Keys",
+        },
+        prefix: {
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
+    readCsv: {
+      displayName: "Read CSV",
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        key: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
+    delete: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delete: {
+          type: DatasourceFieldType.JSON,
           required: true,
         },
       },
     },
   },
+  extra: {
+    acl: {
+      required: false,
+      displayName: "ACL",
+      type: DatasourceFieldType.LIST,
+      data: {
+        create: [
+          "private",
+          "public-read",
+          "public-read-write",
+          "authenticated-read",
+        ],
+      },
+    },
+  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async read(query: { bucket: string }) {
+  async create(query: {
+    bucket: string
+    location: string
+    grantFullControl: string
+    grantRead: string
+    grantReadAcp: string
+    grantWrite: string
+    grantWriteAcp: string
+    extra: {
+      acl: string
+    }
+  }) {
+    let params: any = {
+      Bucket: query.bucket,
+      ACL: query.extra?.acl,
+      GrantFullControl: query.grantFullControl,
+      GrantRead: query.grantRead,
+      GrantReadACP: query.grantReadAcp,
+      GrantWrite: query.grantWrite,
+      GrantWriteACP: query.grantWriteAcp,
+    }
+    if (query.location) {
+      params["CreateBucketConfiguration"] = {
+        LocationConstraint: query.location,
+      }
+    }
+    return await this.client.createBucket(params).promise()
+  }
+
+  async read(query: {
+    bucket: string
+    delimiter: string
+    expectedBucketOwner: string
+    marker: string
+    maxKeys: number
+    prefix: string
+  }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
+        Delimiter: query.delimiter,
+        Marker: query.marker,
+        MaxKeys: query.maxKeys,
+        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
+
+  async readCsv(query: { bucket: string; key: string }) {
+    const stream = this.client
+      .getObject({
+        Bucket: query.bucket,
+        Key: query.key,
+      })
+      .createReadStream()
+
+    let csvError = false
+    return new Promise((resolve, reject) => {
+      stream.on("error", (err: Error) => {
+        reject(err)
+      })
+      const response = csv()
+        .fromStream(stream)
+        .on("error", () => {
+          csvError = true
+        })
+      stream.on("finish", () => {
+        resolve(response)
+      })
+    }).catch(err => {
+      if (csvError) {
+        throw new Error("Could not read CSV")
+      } else {
+        throw err
+      }
+    })
+  }
+
+  async delete(query: { bucket: string; delete: string }) {
+    return await this.client
+      .deleteObjects({
+        Bucket: query.bucket,
+        Delete: JSON.parse(query.delete),
+      })
+      .promise()
+  }
 }
 
 export default {

package/src/integrations/tests/s3.spec.ts

@@ -18,11 +18,95 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    const response = await config.integration.read({
+    await config.integration.read({
       bucket: "test",
+      delimiter: "/",
+      marker: "file.txt",
+      maxKeys: 999,
+      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
+      Delimiter: "/",
+      Marker: "file.txt",
+      MaxKeys: 999,
+      Prefix: "directory/",
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({
+      bucket: "test",
+      location: "af-south-1",
+      grantFullControl: "me",
+      grantRead: "him",
+      grantReadAcp: "her",
+      grantWrite: "she",
+      grantWriteAcp: "he",
+      objectLockEnabledForBucket: true,
+      extra: {
+        acl: "private",
+      },
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      CreateBucketConfiguration: {
+        LocationConstraint: "af-south-1",
+      },
+      GrantFullControl: "me",
+      GrantRead: "him",
+      GrantReadACP: "her",
+      GrantWrite: "she",
+      GrantWriteACP: "he",
+      ACL: "private",
+    })
+  })
+
+  it("does not add undefined location constraint when calling the create method", async () => {
+    await config.integration.create({
+      bucket: "test",
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      GrantFullControl: undefined,
+      GrantRead: undefined,
+      GrantReadACP: undefined,
+      GrantWrite: undefined,
+      GrantWriteACP: undefined,
+      ACL: undefined,
+    })
+  })
+
+  it("calls the delete method with the correct params ", async () => {
+    await config.integration.delete({
+      bucket: "test",
+      delete: `{
+        "Objects": [
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
+        }, 
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
+        }
+       ]
+      }`,
+    })
+    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
+      Bucket: "test",
+      Delete: {
+        Objects: [
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
+          },
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
+          },
+        ],
+      },
     })
   })
 })

package/src/integrations/utils.ts

@@ -230,7 +230,7 @@ function shouldCopySpecialColumn(
   const fetchedIsNumber =
     !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
   return (
-    specialTypes.indexOf(column.type) !== -1 ||
+    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
     (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
   )
 }
@@ -292,7 +292,11 @@ export function finaliseExternalTables(
     if (table.primary == null || table.primary.length === 0) {
       errors[name] = BuildSchemaErrors.NO_KEY
       continue
-    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
+    } else if (
+      schemaFields.find(field =>
+        invalidColumns.includes(field as InvalidColumns)
+      )
+    ) {
       errors[name] = BuildSchemaErrors.INVALID_COLUMN
       continue
     }

package/src/middleware/appInfo.ts

@@ -0,0 +1,20 @@
+import { isDevAppID, isProdAppID } from "../db/utils"
+import { BBContext } from "@budibase/types"
+
+export enum AppType {
+  DEV = "dev",
+  PROD = "prod",
+}
+
+export function middleware({ appType }: { appType?: AppType } = {}) {
+  return (ctx: BBContext, next: any) => {
+    const appId = ctx.appId
+    if (appType === AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }
+}

package/src/middleware/authorized.ts

@@ -1,23 +1,13 @@
-import {
-  getUserRoleHierarchy,
-  getRequiredResourceRole,
-  BUILTIN_ROLE_IDS,
-} from "@budibase/backend-core/roles"
-const {
-  PermissionTypes,
-  PermissionLevels,
-  doesHaveBasePermission,
-} = require("@budibase/backend-core/permissions")
-const builderMiddleware = require("./builder")
-const { isWebhookEndpoint } = require("./utils")
-const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
-const { getAppId } = require("@budibase/backend-core/context")
+import { roles, permissions, auth, context } from "@budibase/backend-core"
+import { Role } from "@budibase/types"
+import builderMiddleware from "./builder"
+import { isWebhookEndpoint } from "./utils"
 
 function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
-const csrf = buildCsrfMiddleware()
+const csrf = auth.buildCsrfMiddleware()
 
 /**
  * Apply authorization to the requested resource:
@@ -33,7 +23,7 @@ const checkAuthorized = async (
 ) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === PermissionTypes.BUILDER
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
   if (isBuilderApi && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
@@ -51,10 +41,10 @@ const checkAuthorizedResource = async (
   permLevel: any
 ) => {
   // get the user's roles
-  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await getUserRoleHierarchy(roleId, {
+  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
     idOnly: false,
-  })) as { _id: string }[]
+  })) as Role[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {
@@ -66,7 +56,9 @@ const checkAuthorizedResource = async (
       ctx.throw(403, permError)
     }
     // fallback to the base permissions when no resource roles are found
-  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
+  } else if (
+    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
+  ) {
     ctx.throw(403, permError)
   }
 }
@@ -91,21 +83,22 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     let resourceRoles: any = []
     let otherLevelRoles: any = []
     const otherLevel =
-      permLevel === PermissionLevels.READ
-        ? PermissionLevels.WRITE
-        : PermissionLevels.READ
-    const appId = getAppId()
+      permLevel === permissions.PermissionLevel.READ
+        ? permissions.PermissionLevel.WRITE
+        : permissions.PermissionLevel.READ
+    const appId = context.getAppId()
     if (appId && hasResource(ctx)) {
-      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
+      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
       if (opts && opts.schema) {
-        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
+        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
       }
     }
 
     // if the resource is public, proceed
     if (
-      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
-      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
+      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles &&
+        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
     ) {
       return next()
     }

package/src/middleware/{builder.js → builder.ts}

@@ -1,14 +1,16 @@
-const { APP_DEV_PREFIX } = require("../db/utils")
-const {
+import {
+  APP_DEV_PREFIX,
+  DocumentType,
+  getGlobalIDFromUserMetadataID,
+} from "../db/utils"
+import {
   doesUserHaveLock,
   updateLock,
   checkDebounce,
   setDebounce,
-} = require("../utilities/redis")
-const { doWithDB } = require("@budibase/backend-core/db")
-const { DocumentType, getGlobalIDFromUserMetadataID } = require("../db/utils")
-const { PermissionTypes } = require("@budibase/backend-core/permissions")
-const { app: appCache } = require("@budibase/backend-core/cache")
+} from "../utilities/redis"
+import { db as dbCore, cache, permissions } from "@budibase/backend-core"
+import { BBContext, Database } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -21,11 +23,11 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx) {
+async function checkDevAppLocks(ctx: BBContext) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
-  if (!ctx.user._id && !ctx.user.userId) {
+  if (!ctx.user?._id && !ctx.user?.userId) {
     return
   }
 
@@ -41,34 +43,34 @@ async function checkDevAppLocks(ctx) {
   await updateLock(appId, ctx.user)
 }
 
-async function updateAppUpdatedAt(ctx) {
+async function updateAppUpdatedAt(ctx: BBContext) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  await doWithDB(appId, async db => {
+  await dbCore.doWithDB(appId, async (db: Database) => {
     const metadata = await db.get(DocumentType.APP_METADATA)
     metadata.updatedAt = new Date().toISOString()
 
-    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId)
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
 
     const response = await db.put(metadata)
     metadata._rev = response.rev
-    await appCache.invalidateAppMetadata(appId, metadata)
+    await cache.app.invalidateAppMetadata(appId, metadata)
     // set a new debounce record with a short TTL
     await setDebounce(appId, DEBOUNCE_TIME_SEC)
   })
 }
 
-module.exports = async (ctx, permType) => {
+export = async function builder(ctx: BBContext, permType: string) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === PermissionTypes.BUILDER
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"

package/src/middleware/{currentapp.js → currentapp.ts}

@@ -1,29 +1,26 @@
-const {
-  getAppIdFromCtx,
-  setCookie,
-  getCookie,
-  clearCookie,
-} = require("@budibase/backend-core/utils")
-const { Cookies, Headers } = require("@budibase/backend-core/constants")
-const { getRole } = require("@budibase/backend-core/roles")
-const { BUILTIN_ROLE_IDS } = require("@budibase/backend-core/roles")
-const { generateUserMetadataID, isDevAppID } = require("../db/utils")
-const { dbExists } = require("@budibase/backend-core/db")
-const { isUserInAppTenant } = require("@budibase/backend-core/tenancy")
-const { getCachedSelf } = require("../utilities/global")
-const env = require("../environment")
-const { isWebhookEndpoint } = require("./utils")
-const { doInAppContext } = require("@budibase/backend-core/context")
+import {
+  utils,
+  constants,
+  roles,
+  db as dbCore,
+  tenancy,
+  context,
+} from "@budibase/backend-core"
+import { generateUserMetadataID, isDevAppID } from "../db/utils"
+import { getCachedSelf } from "../utilities/global"
+import env from "../environment"
+import { isWebhookEndpoint } from "./utils"
+import { BBContext } from "@budibase/types"
 
-module.exports = async (ctx, next) => {
+export = async (ctx: BBContext, next: any) => {
   // try to get the appID from the request
-  let requestAppId = await getAppIdFromCtx(ctx)
+  let requestAppId = await utils.getAppIdFromCtx(ctx)
   // get app cookie if it exists
-  let appCookie = null
+  let appCookie: { appId?: string } | undefined
   try {
-    appCookie = getCookie(ctx, Cookies.CurrentApp)
+    appCookie = utils.getCookie(ctx, constants.Cookie.CurrentApp)
   } catch (err) {
-    clearCookie(ctx, Cookies.CurrentApp)
+    utils.clearCookie(ctx, constants.Cookie.CurrentApp)
   }
   if (!appCookie && !requestAppId) {
     return next()
@@ -31,9 +28,9 @@ module.exports = async (ctx, next) => {
   // check the app exists referenced in cookie
   if (appCookie) {
     const appId = appCookie.appId
-    const exists = await dbExists(appId)
+    const exists = await dbCore.dbExists(appId)
     if (!exists) {
-      clearCookie(ctx, Cookies.CurrentApp)
+      utils.clearCookie(ctx, constants.Cookie.CurrentApp)
       return next()
     }
     // if the request app ID wasn't set, update it with the cookie
@@ -47,13 +44,13 @@ module.exports = async (ctx, next) => {
       !isWebhookEndpoint(ctx) &&
       (!ctx.user || !ctx.user.builder || !ctx.user.builder.global)
     ) {
-      clearCookie(ctx, Cookies.CurrentApp)
+      utils.clearCookie(ctx, constants.Cookie.CurrentApp)
       return ctx.redirect("/")
     }
   }
 
-  let appId,
-    roleId = BUILTIN_ROLE_IDS.PUBLIC
+  let appId: string | undefined,
+    roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
   if (!ctx.user) {
     // not logged in, try to set a cookie for public apps
     appId = requestAppId
@@ -68,16 +65,20 @@ module.exports = async (ctx, next) => {
     const isBuilder =
       globalUser && globalUser.builder && globalUser.builder.global
     const isDevApp = appId && isDevAppID(appId)
-    const roleHeader = ctx.request && ctx.request.headers[Headers.PREVIEW_ROLE]
+    const roleHeader =
+      ctx.request &&
+      (ctx.request.headers[constants.Header.PREVIEW_ROLE] as string)
     if (isBuilder && isDevApp && roleHeader) {
       // Ensure the role is valid by ensuring a definition exists
       try {
-        await getRole(roleHeader)
-        roleId = roleHeader
+        if (roleHeader) {
+          await roles.getRole(roleHeader)
+          roleId = roleHeader
 
-        // Delete admin and builder flags so that the specified role is honoured
-        delete ctx.user.builder
-        delete ctx.user.admin
+          // Delete admin and builder flags so that the specified role is honoured
+          delete ctx.user.builder
+          delete ctx.user.admin
+        }
       } catch (error) {
         // Swallow error and do nothing
       }
@@ -89,7 +90,7 @@ module.exports = async (ctx, next) => {
     return next()
   }
 
-  return doInAppContext(appId, async () => {
+  return context.doInAppContext(appId, async () => {
     let skipCookie = false
     // if the user not in the right tenant then make sure they have no permissions
     // need to judge this only based on the request app ID,
@@ -97,14 +98,14 @@ module.exports = async (ctx, next) => {
       env.MULTI_TENANCY &&
       ctx.user &&
       requestAppId &&
-      !isUserInAppTenant(requestAppId, ctx.user)
+      !tenancy.isUserInAppTenant(requestAppId, ctx.user)
     ) {
       // don't error, simply remove the users rights (they are a public user)
       delete ctx.user.builder
       delete ctx.user.admin
       delete ctx.user.roles
       ctx.isAuthenticated = false
-      roleId = BUILTIN_ROLE_IDS.PUBLIC
+      roleId = roles.BUILTIN_ROLE_IDS.PUBLIC
       skipCookie = true
     }
 
@@ -112,15 +113,17 @@ module.exports = async (ctx, next) => {
     if (roleId) {
       ctx.roleId = roleId
       const globalId = ctx.user ? ctx.user._id : undefined
-      const userId = ctx.user ? generateUserMetadataID(ctx.user._id) : null
+      const userId = ctx.user
+        ? generateUserMetadataID(ctx.user._id!)
+        : undefined
       ctx.user = {
-        ...ctx.user,
+        ...ctx.user!,
         // override userID with metadata one
         _id: userId,
         userId,
         globalId,
         roleId,
-        role: await getRole(roleId),
+        role: await roles.getRole(roleId),
       }
     }
     if (
@@ -129,7 +132,7 @@ module.exports = async (ctx, next) => {
         appCookie.appId !== requestAppId) &&
       !skipCookie
     ) {
-      setCookie(ctx, { appId }, Cookies.CurrentApp)
+      utils.setCookie(ctx, { appId }, constants.Cookie.CurrentApp)
     }
 
     return next()