@budibase/server 2.1.38 → 2.1.40-alpha.0

package/src/integrations/s3.ts

@@ -1,5 +1,12 @@
-import { Integration, QueryType, IntegrationBase } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
+
 const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -40,16 +47,106 @@ const SCHEMA: Integration = {
     },
   },
   query: {
+    create: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          display: "New Bucket",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        location: {
+          required: true,
+          default: "us-east-1",
+          type: DatasourceFieldType.STRING,
+        },
+        grantFullControl: {
+          display: "Grant full control",
+          type: DatasourceFieldType.STRING,
+        },
+        grantRead: {
+          display: "Grant read",
+          type: DatasourceFieldType.STRING,
+        },
+        grantReadAcp: {
+          display: "Grant read ACP",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWrite: {
+          display: "Grant write",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWriteAcp: {
+          display: "Grant write ACP",
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: "string",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delimiter: {
+          type: DatasourceFieldType.STRING,
+        },
+        marker: {
+          type: DatasourceFieldType.STRING,
+        },
+        maxKeys: {
+          type: DatasourceFieldType.NUMBER,
+          display: "Max Keys",
+        },
+        prefix: {
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
+    readCsv: {
+      displayName: "Read CSV",
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        key: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
+    delete: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delete: {
+          type: DatasourceFieldType.JSON,
           required: true,
         },
       },
     },
   },
+  extra: {
+    acl: {
+      required: false,
+      displayName: "ACL",
+      type: DatasourceFieldType.LIST,
+      data: {
+        create: [
+          "private",
+          "public-read",
+          "public-read-write",
+          "authenticated-read",
+        ],
+      },
+    },
+  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async read(query: { bucket: string }) {
+  async create(query: {
+    bucket: string
+    location: string
+    grantFullControl: string
+    grantRead: string
+    grantReadAcp: string
+    grantWrite: string
+    grantWriteAcp: string
+    extra: {
+      acl: string
+    }
+  }) {
+    let params: any = {
+      Bucket: query.bucket,
+      ACL: query.extra?.acl,
+      GrantFullControl: query.grantFullControl,
+      GrantRead: query.grantRead,
+      GrantReadACP: query.grantReadAcp,
+      GrantWrite: query.grantWrite,
+      GrantWriteACP: query.grantWriteAcp,
+    }
+    if (query.location) {
+      params["CreateBucketConfiguration"] = {
+        LocationConstraint: query.location,
+      }
+    }
+    return await this.client.createBucket(params).promise()
+  }
+
+  async read(query: {
+    bucket: string
+    delimiter: string
+    expectedBucketOwner: string
+    marker: string
+    maxKeys: number
+    prefix: string
+  }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
+        Delimiter: query.delimiter,
+        Marker: query.marker,
+        MaxKeys: query.maxKeys,
+        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
+
+  async readCsv(query: { bucket: string; key: string }) {
+    const stream = this.client
+      .getObject({
+        Bucket: query.bucket,
+        Key: query.key,
+      })
+      .createReadStream()
+
+    let csvError = false
+    return new Promise((resolve, reject) => {
+      stream.on("error", (err: Error) => {
+        reject(err)
+      })
+      const response = csv()
+        .fromStream(stream)
+        .on("error", () => {
+          csvError = true
+        })
+      stream.on("finish", () => {
+        resolve(response)
+      })
+    }).catch(err => {
+      if (csvError) {
+        throw new Error("Could not read CSV")
+      } else {
+        throw err
+      }
+    })
+  }
+
+  async delete(query: { bucket: string; delete: string }) {
+    return await this.client
+      .deleteObjects({
+        Bucket: query.bucket,
+        Delete: JSON.parse(query.delete),
+      })
+      .promise()
+  }
 }
 
 export default {

package/src/integrations/tests/mysql.spec.ts

@@ -73,4 +73,61 @@ describe("MySQL Integration", () => {
       expect(response).toEqual([{ deleted: true }])
     })
   })
+
+  describe("binding type coerce", () => {
+    it("ignores non-string types ", async () => {
+      const sql = "select * from users;"
+      const date = new Date()
+      await config.integration.read({
+        sql,
+        bindings: [11, date, ["a", "b", "c"], { id: 1 }],
+      })
+      expect(config.integration.client.query).toHaveBeenCalledWith(sql, [
+        11,
+        date,
+        ["a", "b", "c"],
+        { id: 1 },
+      ])
+    })
+
+    it("parses strings matching a number regex", async () => {
+      const sql = "select * from users;"
+      await config.integration.read({
+        sql,
+        bindings: ["101", "3.14"],
+      })
+      expect(config.integration.client.query).toHaveBeenCalledWith(
+        sql,
+        [101, 3.14]
+      )
+    })
+
+    it("parses strings matching a valid date format", async () => {
+      const sql = "select * from users;"
+      await config.integration.read({
+        sql,
+        bindings: [
+          "2001-10-30",
+          "2010-09-01T13:30:59.123Z",
+          "2021-02-05 12:01 PM",
+        ],
+      })
+      expect(config.integration.client.query).toHaveBeenCalledWith(sql, [
+        new Date("2001-10-30T00:00:00.000Z"),
+        new Date("2010-09-01T13:30:59.123Z"),
+        new Date("2021-02-05T12:01:00.000Z"),
+      ])
+    })
+
+    it("does not parse string matching a valid array of numbers as date", async () => {
+      const sql = "select * from users;"
+      await config.integration.read({
+        sql,
+        bindings: ["1,2,2017"],
+      })
+      expect(config.integration.client.query).toHaveBeenCalledWith(sql, [
+        "1,2,2017",
+      ])
+    })
+  })
 })

package/src/integrations/tests/s3.spec.ts

@@ -18,11 +18,95 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    const response = await config.integration.read({
+    await config.integration.read({
       bucket: "test",
+      delimiter: "/",
+      marker: "file.txt",
+      maxKeys: 999,
+      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
+      Delimiter: "/",
+      Marker: "file.txt",
+      MaxKeys: 999,
+      Prefix: "directory/",
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({
+      bucket: "test",
+      location: "af-south-1",
+      grantFullControl: "me",
+      grantRead: "him",
+      grantReadAcp: "her",
+      grantWrite: "she",
+      grantWriteAcp: "he",
+      objectLockEnabledForBucket: true,
+      extra: {
+        acl: "private",
+      },
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      CreateBucketConfiguration: {
+        LocationConstraint: "af-south-1",
+      },
+      GrantFullControl: "me",
+      GrantRead: "him",
+      GrantReadACP: "her",
+      GrantWrite: "she",
+      GrantWriteACP: "he",
+      ACL: "private",
+    })
+  })
+
+  it("does not add undefined location constraint when calling the create method", async () => {
+    await config.integration.create({
+      bucket: "test",
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      GrantFullControl: undefined,
+      GrantRead: undefined,
+      GrantReadACP: undefined,
+      GrantWrite: undefined,
+      GrantWriteACP: undefined,
+      ACL: undefined,
+    })
+  })
+
+  it("calls the delete method with the correct params ", async () => {
+    await config.integration.delete({
+      bucket: "test",
+      delete: `{
+        "Objects": [
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
+        }, 
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
+        }
+       ]
+      }`,
+    })
+    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
+      Bucket: "test",
+      Delete: {
+        Objects: [
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
+          },
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
+          },
+        ],
+      },
     })
   })
 })

package/src/integrations/utils.ts

@@ -230,7 +230,7 @@ function shouldCopySpecialColumn(
   const fetchedIsNumber =
     !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
   return (
-    specialTypes.indexOf(column.type) !== -1 ||
+    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
     (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
   )
 }
@@ -292,7 +292,11 @@ export function finaliseExternalTables(
     if (table.primary == null || table.primary.length === 0) {
       errors[name] = BuildSchemaErrors.NO_KEY
       continue
-    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
+    } else if (
+      schemaFields.find(field =>
+        invalidColumns.includes(field as InvalidColumns)
+      )
+    ) {
       errors[name] = BuildSchemaErrors.INVALID_COLUMN
       continue
     }

package/src/middleware/appInfo.ts

@@ -0,0 +1,20 @@
+import { isDevAppID, isProdAppID } from "../db/utils"
+import { BBContext } from "@budibase/types"
+
+export enum AppType {
+  DEV = "dev",
+  PROD = "prod",
+}
+
+export function middleware({ appType }: { appType?: AppType } = {}) {
+  return (ctx: BBContext, next: any) => {
+    const appId = ctx.appId
+    if (appType === AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }
+}

package/src/middleware/authorized.ts

@@ -1,23 +1,13 @@
-import {
-  getUserRoleHierarchy,
-  getRequiredResourceRole,
-  BUILTIN_ROLE_IDS,
-} from "@budibase/backend-core/roles"
-const {
-  PermissionTypes,
-  PermissionLevels,
-  doesHaveBasePermission,
-} = require("@budibase/backend-core/permissions")
-const builderMiddleware = require("./builder")
-const { isWebhookEndpoint } = require("./utils")
-const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
-const { getAppId } = require("@budibase/backend-core/context")
+import { roles, permissions, auth, context } from "@budibase/backend-core"
+import { Role } from "@budibase/types"
+import builderMiddleware from "./builder"
+import { isWebhookEndpoint } from "./utils"
 
 function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
-const csrf = buildCsrfMiddleware()
+const csrf = auth.buildCsrfMiddleware()
 
 /**
  * Apply authorization to the requested resource:
@@ -33,7 +23,7 @@ const checkAuthorized = async (
 ) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === PermissionTypes.BUILDER
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
   if (isBuilderApi && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
@@ -51,10 +41,10 @@ const checkAuthorizedResource = async (
   permLevel: any
 ) => {
   // get the user's roles
-  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await getUserRoleHierarchy(roleId, {
+  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
     idOnly: false,
-  })) as { _id: string }[]
+  })) as Role[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {
@@ -66,7 +56,9 @@ const checkAuthorizedResource = async (
       ctx.throw(403, permError)
     }
     // fallback to the base permissions when no resource roles are found
-  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
+  } else if (
+    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
+  ) {
     ctx.throw(403, permError)
   }
 }
@@ -91,21 +83,22 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     let resourceRoles: any = []
     let otherLevelRoles: any = []
     const otherLevel =
-      permLevel === PermissionLevels.READ
-        ? PermissionLevels.WRITE
-        : PermissionLevels.READ
-    const appId = getAppId()
+      permLevel === permissions.PermissionLevel.READ
+        ? permissions.PermissionLevel.WRITE
+        : permissions.PermissionLevel.READ
+    const appId = context.getAppId()
     if (appId && hasResource(ctx)) {
-      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
+      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
       if (opts && opts.schema) {
-        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
+        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
       }
     }
 
     // if the resource is public, proceed
     if (
-      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
-      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
+      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles &&
+        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
     ) {
       return next()
     }

package/src/middleware/{builder.js → builder.ts}

@@ -1,14 +1,16 @@
-const { APP_DEV_PREFIX } = require("../db/utils")
-const {
+import {
+  APP_DEV_PREFIX,
+  DocumentType,
+  getGlobalIDFromUserMetadataID,
+} from "../db/utils"
+import {
   doesUserHaveLock,
   updateLock,
   checkDebounce,
   setDebounce,
-} = require("../utilities/redis")
-const { doWithDB } = require("@budibase/backend-core/db")
-const { DocumentType, getGlobalIDFromUserMetadataID } = require("../db/utils")
-const { PermissionTypes } = require("@budibase/backend-core/permissions")
-const { app: appCache } = require("@budibase/backend-core/cache")
+} from "../utilities/redis"
+import { db as dbCore, cache, permissions } from "@budibase/backend-core"
+import { BBContext, Database } from "@budibase/types"
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -21,11 +23,11 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx) {
+async function checkDevAppLocks(ctx: BBContext) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
-  if (!ctx.user._id && !ctx.user.userId) {
+  if (!ctx.user?._id && !ctx.user?.userId) {
     return
   }
 
@@ -41,34 +43,34 @@ async function checkDevAppLocks(ctx) {
   await updateLock(appId, ctx.user)
 }
 
-async function updateAppUpdatedAt(ctx) {
+async function updateAppUpdatedAt(ctx: BBContext) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  await doWithDB(appId, async db => {
+  await dbCore.doWithDB(appId, async (db: Database) => {
     const metadata = await db.get(DocumentType.APP_METADATA)
     metadata.updatedAt = new Date().toISOString()
 
-    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId)
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
 
     const response = await db.put(metadata)
     metadata._rev = response.rev
-    await appCache.invalidateAppMetadata(appId, metadata)
+    await cache.app.invalidateAppMetadata(appId, metadata)
     // set a new debounce record with a short TTL
     await setDebounce(appId, DEBOUNCE_TIME_SEC)
   })
 }
 
-module.exports = async (ctx, permType) => {
+export = async function builder(ctx: BBContext, permType: string) {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === PermissionTypes.BUILDER
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"