@budibase/server 2.1.44 → 2.1.46-alpha.0

package/src/integrations/microsoftSqlServer.ts

@@ -17,6 +17,7 @@ import {
   SqlClient,
 } from "./utils"
 import Sql from "./base/sql"
+import { MSSQLTablesResponse, MSSQLColumn } from "./base/types"
 
 const sqlServer = require("mssql")
 const DEFAULT_SCHEMA = "dbo"
@@ -31,13 +32,6 @@ interface MSSQLConfig {
   encrypt?: boolean
 }
 
-interface TablesResponse {
-  TABLE_CATALOG: string
-  TABLE_SCHEMA: string
-  TABLE_NAME: string
-  TABLE_TYPE: string
-}
-
 const SCHEMA: Integration = {
   docs: "https://github.com/tediousjs/node-mssql",
   plus: true,
@@ -210,7 +204,7 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
    */
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
     await this.connect()
-    let tableInfo: TablesResponse[] = await this.runSQL(this.TABLES_SQL)
+    let tableInfo: MSSQLTablesResponse[] = await this.runSQL(this.TABLES_SQL)
     if (tableInfo == null || !Array.isArray(tableInfo)) {
       throw "Unable to get list of tables in database"
     }
@@ -228,15 +222,20 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
       // find primary key constraints
       const constraints = await this.runSQL(this.getConstraintsSQL(tableName))
       // find the computed and identity columns (auto columns)
-      const columns = await this.runSQL(this.getAutoColumnsSQL(tableName))
+      const columns: MSSQLColumn[] = await this.runSQL(
+        this.getAutoColumnsSQL(tableName)
+      )
       const primaryKeys = constraints
         .filter(
           (constraint: any) => constraint.CONSTRAINT_TYPE === "PRIMARY KEY"
         )
         .map((constraint: any) => constraint.COLUMN_NAME)
       const autoColumns = columns
-        .filter((col: any) => col.IS_COMPUTED || col.IS_IDENTITY)
-        .map((col: any) => col.COLUMN_NAME)
+        .filter(col => col.IS_COMPUTED || col.IS_IDENTITY)
+        .map(col => col.COLUMN_NAME)
+      const requiredColumns = columns
+        .filter(col => col.IS_NULLABLE === "NO")
+        .map(col => col.COLUMN_NAME)
 
       let schema: TableSchema = {}
       for (let def of definition) {
@@ -245,8 +244,11 @@ class SqlServerIntegration extends Sql implements DatasourcePlus {
           continue
         }
         schema[name] = {
-          autocolumn: !!autoColumns.find((col: string) => col === name),
+          autocolumn: !!autoColumns.find(col => col === name),
           name: name,
+          constraints: {
+            presence: requiredColumns.find(col => col === name),
+          },
           ...convertSqlType(def.DATA_TYPE),
           externalType: def.DATA_TYPE,
         }

package/src/integrations/mysql.ts

@@ -18,6 +18,7 @@ import {
 import dayjs from "dayjs"
 const { NUMBER_REGEX } = require("../utilities")
 import Sql from "./base/sql"
+import { MySQLColumn } from "./base/types"
 
 const mysql = require("mysql2/promise")
 
@@ -203,11 +204,11 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
 
     try {
       // get the tables first
-      const tablesResp = await this.internalQuery(
+      const tablesResp: Record<string, string>[] = await this.internalQuery(
         { sql: "SHOW TABLES;" },
         { connect: false }
       )
-      const tableNames = tablesResp.map(
+      const tableNames: string[] = tablesResp.map(
         (obj: any) =>
           obj[`Tables_in_${database}`] ||
           obj[`Tables_in_${database.toLowerCase()}`]
@@ -215,7 +216,7 @@ class MySQLIntegration extends Sql implements DatasourcePlus {
       for (let tableName of tableNames) {
         const primaryKeys = []
         const schema: TableSchema = {}
-        const descResp = await this.internalQuery(
+        const descResp: MySQLColumn[] = await this.internalQuery(
           { sql: `DESCRIBE \`${tableName}\`;` },
           { connect: false }
         )

package/src/integrations/oracle.ts

@@ -24,6 +24,12 @@ import {
   ExecuteOptions,
   Result,
 } from "oracledb"
+import {
+  OracleTable,
+  OracleColumn,
+  OracleColumnsResponse,
+  OracleConstraint,
+} from "./base/types"
 let oracledb: any
 try {
   oracledb = require("oracledb")
@@ -89,50 +95,6 @@ const SCHEMA: Integration = {
 
 const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
 
-/**
- * Raw query response
- */
-interface ColumnsResponse {
-  TABLE_NAME: string
-  COLUMN_NAME: string
-  DATA_TYPE: string
-  DATA_DEFAULT: string | null
-  COLUMN_ID: number
-  CONSTRAINT_NAME: string | null
-  CONSTRAINT_TYPE: string | null
-  R_CONSTRAINT_NAME: string | null
-  SEARCH_CONDITION: string | null
-}
-
-/**
- * An oracle constraint
- */
-interface OracleConstraint {
-  name: string
-  type: string
-  relatedConstraintName: string | null
-  searchCondition: string | null
-}
-
-/**
- * An oracle column and it's related constraints
- */
-interface OracleColumn {
-  name: string
-  type: string
-  default: string | null
-  id: number
-  constraints: { [key: string]: OracleConstraint }
-}
-
-/**
- * An oracle table and it's related columns
- */
-interface OracleTable {
-  name: string
-  columns: { [key: string]: OracleColumn }
-}
-
 const OracleContraintTypes = {
   PRIMARY: "P",
   NOT_NULL_OR_CHECK: "C",
@@ -195,7 +157,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   /**
    * Map the flat tabular columns and constraints data into a nested object
    */
-  private mapColumns(result: Result<ColumnsResponse>): {
+  private mapColumns(result: Result<OracleColumnsResponse>): {
     [key: string]: OracleTable
   } {
     const oracleTables: { [key: string]: OracleTable } = {}
@@ -299,7 +261,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    * @param entities - the tables that are to be built
    */
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    const columnsResponse = await this.internalQuery<ColumnsResponse>({
+    const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
@@ -334,6 +296,9 @@ class OracleIntegration extends Sql implements DatasourcePlus {
             fieldSchema = {
               autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
               name: columnName,
+              constraints: {
+                presence: false,
+              },
               ...this.internalConvertType(oracleColumn),
             }
             table.schema[columnName] = fieldSchema
@@ -343,6 +308,12 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           Object.values(oracleColumn.constraints).forEach(oracleConstraint => {
             if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
               table.primary!.push(columnName)
+            } else if (
+              oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK
+            ) {
+              table.schema[columnName].constraints = {
+                presence: true,
+              }
             }
           })
         })

package/src/integrations/postgres.ts

@@ -15,9 +15,10 @@ import {
   SqlClient,
 } from "./utils"
 import Sql from "./base/sql"
+import { PostgresColumn } from "./base/types"
+import { escapeDangerousCharacters } from "../utilities"
 
 const { Client, types } = require("pg")
-const { escapeDangerousCharacters } = require("../utilities")
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -237,7 +238,8 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     }
 
     try {
-      const columnsResponse = await this.client.query(this.COLUMNS_SQL)
+      const columnsResponse: { rows: PostgresColumn[] } =
+        await this.client.query(this.COLUMNS_SQL)
 
       const tables: { [key: string]: Table } = {}
 
@@ -260,6 +262,9 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.identity_start ||
           column.identity_increment
         )
+        const constraints = {
+          presence: column.is_nullable === "NO",
+        }
         const hasDefault =
           typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
@@ -269,6 +274,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
+          constraints,
           ...convertSqlType(column.data_type),
           externalType: column.data_type,
         }

package/src/integrations/s3.ts

@@ -1,5 +1,12 @@
-import { Integration, QueryType, IntegrationBase } from "@budibase/types"
+import {
+  Integration,
+  QueryType,
+  IntegrationBase,
+  DatasourceFieldType,
+} from "@budibase/types"
+
 const AWS = require("aws-sdk")
+const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -40,16 +47,106 @@ const SCHEMA: Integration = {
     },
   },
   query: {
+    create: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          display: "New Bucket",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        location: {
+          required: true,
+          default: "us-east-1",
+          type: DatasourceFieldType.STRING,
+        },
+        grantFullControl: {
+          display: "Grant full control",
+          type: DatasourceFieldType.STRING,
+        },
+        grantRead: {
+          display: "Grant read",
+          type: DatasourceFieldType.STRING,
+        },
+        grantReadAcp: {
+          display: "Grant read ACP",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWrite: {
+          display: "Grant write",
+          type: DatasourceFieldType.STRING,
+        },
+        grantWriteAcp: {
+          display: "Grant write ACP",
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: "string",
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delimiter: {
+          type: DatasourceFieldType.STRING,
+        },
+        marker: {
+          type: DatasourceFieldType.STRING,
+        },
+        maxKeys: {
+          type: DatasourceFieldType.NUMBER,
+          display: "Max Keys",
+        },
+        prefix: {
+          type: DatasourceFieldType.STRING,
+        },
+      },
+    },
+    readCsv: {
+      displayName: "Read CSV",
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        key: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
+    delete: {
+      type: QueryType.FIELDS,
+      fields: {
+        bucket: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+        delete: {
+          type: DatasourceFieldType.JSON,
           required: true,
         },
       },
     },
   },
+  extra: {
+    acl: {
+      required: false,
+      displayName: "ACL",
+      type: DatasourceFieldType.LIST,
+      data: {
+        create: [
+          "private",
+          "public-read",
+          "public-read-write",
+          "authenticated-read",
+        ],
+      },
+    },
+  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -67,14 +164,93 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async read(query: { bucket: string }) {
+  async create(query: {
+    bucket: string
+    location: string
+    grantFullControl: string
+    grantRead: string
+    grantReadAcp: string
+    grantWrite: string
+    grantWriteAcp: string
+    extra: {
+      acl: string
+    }
+  }) {
+    let params: any = {
+      Bucket: query.bucket,
+      ACL: query.extra?.acl,
+      GrantFullControl: query.grantFullControl,
+      GrantRead: query.grantRead,
+      GrantReadACP: query.grantReadAcp,
+      GrantWrite: query.grantWrite,
+      GrantWriteACP: query.grantWriteAcp,
+    }
+    if (query.location) {
+      params["CreateBucketConfiguration"] = {
+        LocationConstraint: query.location,
+      }
+    }
+    return await this.client.createBucket(params).promise()
+  }
+
+  async read(query: {
+    bucket: string
+    delimiter: string
+    expectedBucketOwner: string
+    marker: string
+    maxKeys: number
+    prefix: string
+  }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
+        Delimiter: query.delimiter,
+        Marker: query.marker,
+        MaxKeys: query.maxKeys,
+        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
+
+  async readCsv(query: { bucket: string; key: string }) {
+    const stream = this.client
+      .getObject({
+        Bucket: query.bucket,
+        Key: query.key,
+      })
+      .createReadStream()
+
+    let csvError = false
+    return new Promise((resolve, reject) => {
+      stream.on("error", (err: Error) => {
+        reject(err)
+      })
+      const response = csv()
+        .fromStream(stream)
+        .on("error", () => {
+          csvError = true
+        })
+      stream.on("finish", () => {
+        resolve(response)
+      })
+    }).catch(err => {
+      if (csvError) {
+        throw new Error("Could not read CSV")
+      } else {
+        throw err
+      }
+    })
+  }
+
+  async delete(query: { bucket: string; delete: string }) {
+    return await this.client
+      .deleteObjects({
+        Bucket: query.bucket,
+        Delete: JSON.parse(query.delete),
+      })
+      .promise()
+  }
 }
 
 export default {

package/src/integrations/tests/s3.spec.ts

@@ -18,11 +18,95 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    const response = await config.integration.read({
+    await config.integration.read({
       bucket: "test",
+      delimiter: "/",
+      marker: "file.txt",
+      maxKeys: 999,
+      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
+      Delimiter: "/",
+      Marker: "file.txt",
+      MaxKeys: 999,
+      Prefix: "directory/",
+    })
+  })
+
+  it("calls the create method with the correct params", async () => {
+    await config.integration.create({
+      bucket: "test",
+      location: "af-south-1",
+      grantFullControl: "me",
+      grantRead: "him",
+      grantReadAcp: "her",
+      grantWrite: "she",
+      grantWriteAcp: "he",
+      objectLockEnabledForBucket: true,
+      extra: {
+        acl: "private",
+      },
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      CreateBucketConfiguration: {
+        LocationConstraint: "af-south-1",
+      },
+      GrantFullControl: "me",
+      GrantRead: "him",
+      GrantReadACP: "her",
+      GrantWrite: "she",
+      GrantWriteACP: "he",
+      ACL: "private",
+    })
+  })
+
+  it("does not add undefined location constraint when calling the create method", async () => {
+    await config.integration.create({
+      bucket: "test",
+    })
+    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
+      Bucket: "test",
+      GrantFullControl: undefined,
+      GrantRead: undefined,
+      GrantReadACP: undefined,
+      GrantWrite: undefined,
+      GrantWriteACP: undefined,
+      ACL: undefined,
+    })
+  })
+
+  it("calls the delete method with the correct params ", async () => {
+    await config.integration.delete({
+      bucket: "test",
+      delete: `{
+        "Objects": [
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
+        }, 
+          {
+         "Key": "HappyFace.jpg", 
+         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
+        }
+       ]
+      }`,
+    })
+    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
+      Bucket: "test",
+      Delete: {
+        Objects: [
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
+          },
+          {
+            Key: "HappyFace.jpg",
+            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
+          },
+        ],
+      },
     })
   })
 })

package/src/integrations/utils.ts

@@ -230,7 +230,7 @@ function shouldCopySpecialColumn(
   const fetchedIsNumber =
     !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
   return (
-    specialTypes.indexOf(column.type) !== -1 ||
+    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
     (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
   )
 }
@@ -292,7 +292,11 @@ export function finaliseExternalTables(
     if (table.primary == null || table.primary.length === 0) {
       errors[name] = BuildSchemaErrors.NO_KEY
       continue
-    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
+    } else if (
+      schemaFields.find(field =>
+        invalidColumns.includes(field as InvalidColumns)
+      )
+    ) {
       errors[name] = BuildSchemaErrors.INVALID_COLUMN
       continue
     }

package/src/middleware/appInfo.ts

@@ -0,0 +1,20 @@
+import { isDevAppID, isProdAppID } from "../db/utils"
+import { BBContext } from "@budibase/types"
+
+export enum AppType {
+  DEV = "dev",
+  PROD = "prod",
+}
+
+export function middleware({ appType }: { appType?: AppType } = {}) {
+  return (ctx: BBContext, next: any) => {
+    const appId = ctx.appId
+    if (appType === AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }
+}

package/src/middleware/authorized.ts

@@ -1,23 +1,13 @@
-import {
-  getUserRoleHierarchy,
-  getRequiredResourceRole,
-  BUILTIN_ROLE_IDS,
-} from "@budibase/backend-core/roles"
-const {
-  PermissionTypes,
-  PermissionLevels,
-  doesHaveBasePermission,
-} = require("@budibase/backend-core/permissions")
-const builderMiddleware = require("./builder")
-const { isWebhookEndpoint } = require("./utils")
-const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
-const { getAppId } = require("@budibase/backend-core/context")
+import { roles, permissions, auth, context } from "@budibase/backend-core"
+import { Role } from "@budibase/types"
+import builderMiddleware from "./builder"
+import { isWebhookEndpoint } from "./utils"
 
 function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
-const csrf = buildCsrfMiddleware()
+const csrf = auth.buildCsrfMiddleware()
 
 /**
  * Apply authorization to the requested resource:
@@ -33,7 +23,7 @@ const checkAuthorized = async (
 ) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === PermissionTypes.BUILDER
+  const isBuilderApi = permType === permissions.PermissionType.BUILDER
   if (isBuilderApi && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
@@ -51,10 +41,10 @@ const checkAuthorizedResource = async (
   permLevel: any
 ) => {
   // get the user's roles
-  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await getUserRoleHierarchy(roleId, {
+  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
     idOnly: false,
-  })) as { _id: string }[]
+  })) as Role[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {
@@ -66,7 +56,9 @@ const checkAuthorizedResource = async (
       ctx.throw(403, permError)
     }
     // fallback to the base permissions when no resource roles are found
-  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
+  } else if (
+    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
+  ) {
     ctx.throw(403, permError)
   }
 }
@@ -91,21 +83,22 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     let resourceRoles: any = []
     let otherLevelRoles: any = []
     const otherLevel =
-      permLevel === PermissionLevels.READ
-        ? PermissionLevels.WRITE
-        : PermissionLevels.READ
-    const appId = getAppId()
+      permLevel === permissions.PermissionLevel.READ
+        ? permissions.PermissionLevel.WRITE
+        : permissions.PermissionLevel.READ
+    const appId = context.getAppId()
     if (appId && hasResource(ctx)) {
-      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
+      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
       if (opts && opts.schema) {
-        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
+        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
       }
     }
 
     // if the resource is public, proceed
     if (
-      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
-      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
+      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles &&
+        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
     ) {
       return next()
     }