@budibase/server 2.1.43-alpha.6 → 2.1.44

package/src/integrations/oracle.ts

@@ -24,12 +24,6 @@ import {
   ExecuteOptions,
   Result,
 } from "oracledb"
-import {
-  OracleTable,
-  OracleColumn,
-  OracleColumnsResponse,
-  OracleConstraint,
-} from "./base/types"
 let oracledb: any
 try {
   oracledb = require("oracledb")
@@ -95,6 +89,50 @@ const SCHEMA: Integration = {
 
 const UNSUPPORTED_TYPES = ["BLOB", "CLOB", "NCLOB"]
 
+/**
+ * Raw query response
+ */
+interface ColumnsResponse {
+  TABLE_NAME: string
+  COLUMN_NAME: string
+  DATA_TYPE: string
+  DATA_DEFAULT: string | null
+  COLUMN_ID: number
+  CONSTRAINT_NAME: string | null
+  CONSTRAINT_TYPE: string | null
+  R_CONSTRAINT_NAME: string | null
+  SEARCH_CONDITION: string | null
+}
+
+/**
+ * An oracle constraint
+ */
+interface OracleConstraint {
+  name: string
+  type: string
+  relatedConstraintName: string | null
+  searchCondition: string | null
+}
+
+/**
+ * An oracle column and it's related constraints
+ */
+interface OracleColumn {
+  name: string
+  type: string
+  default: string | null
+  id: number
+  constraints: { [key: string]: OracleConstraint }
+}
+
+/**
+ * An oracle table and it's related columns
+ */
+interface OracleTable {
+  name: string
+  columns: { [key: string]: OracleColumn }
+}
+
 const OracleContraintTypes = {
   PRIMARY: "P",
   NOT_NULL_OR_CHECK: "C",
@@ -157,7 +195,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
   /**
    * Map the flat tabular columns and constraints data into a nested object
    */
-  private mapColumns(result: Result<OracleColumnsResponse>): {
+  private mapColumns(result: Result<ColumnsResponse>): {
     [key: string]: OracleTable
   } {
     const oracleTables: { [key: string]: OracleTable } = {}
@@ -261,7 +299,7 @@ class OracleIntegration extends Sql implements DatasourcePlus {
    * @param entities - the tables that are to be built
    */
   async buildSchema(datasourceId: string, entities: Record<string, Table>) {
-    const columnsResponse = await this.internalQuery<OracleColumnsResponse>({
+    const columnsResponse = await this.internalQuery<ColumnsResponse>({
       sql: this.COLUMNS_SQL,
     })
     const oracleTables = this.mapColumns(columnsResponse)
@@ -296,9 +334,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
             fieldSchema = {
               autocolumn: OracleIntegration.isAutoColumn(oracleColumn),
               name: columnName,
-              constraints: {
-                presence: false,
-              },
               ...this.internalConvertType(oracleColumn),
             }
             table.schema[columnName] = fieldSchema
@@ -308,12 +343,6 @@ class OracleIntegration extends Sql implements DatasourcePlus {
           Object.values(oracleColumn.constraints).forEach(oracleConstraint => {
             if (oracleConstraint.type === OracleContraintTypes.PRIMARY) {
               table.primary!.push(columnName)
-            } else if (
-              oracleConstraint.type === OracleContraintTypes.NOT_NULL_OR_CHECK
-            ) {
-              table.schema[columnName].constraints = {
-                presence: true,
-              }
             }
           })
         })

package/src/integrations/postgres.ts

@@ -15,10 +15,9 @@ import {
   SqlClient,
 } from "./utils"
 import Sql from "./base/sql"
-import { PostgresColumn } from "./base/types"
-import { escapeDangerousCharacters } from "../utilities"
 
 const { Client, types } = require("pg")
+const { escapeDangerousCharacters } = require("../utilities")
 
 // Return "date" and "timestamp" types as plain strings.
 // This lets us reference the original stored timezone.
@@ -238,8 +237,7 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
     }
 
     try {
-      const columnsResponse: { rows: PostgresColumn[] } =
-        await this.client.query(this.COLUMNS_SQL)
+      const columnsResponse = await this.client.query(this.COLUMNS_SQL)
 
       const tables: { [key: string]: Table } = {}
 
@@ -262,9 +260,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
           column.identity_start ||
           column.identity_increment
         )
-        const constraints = {
-          presence: column.is_nullable === "NO",
-        }
         const hasDefault =
           typeof column.column_default === "string" &&
           column.column_default.startsWith("nextval")
@@ -274,7 +269,6 @@ class PostgresIntegration extends Sql implements DatasourcePlus {
         tables[tableName].schema[columnName] = {
           autocolumn: isAuto,
           name: columnName,
-          constraints,
           ...convertSqlType(column.data_type),
           externalType: column.data_type,
         }

package/src/integrations/s3.ts

@@ -1,12 +1,5 @@
-import {
-  Integration,
-  QueryType,
-  IntegrationBase,
-  DatasourceFieldType,
-} from "@budibase/types"
-
+import { Integration, QueryType, IntegrationBase } from "@budibase/types"
 const AWS = require("aws-sdk")
-const csv = require("csvtojson")
 
 interface S3Config {
   region: string
@@ -47,106 +40,16 @@ const SCHEMA: Integration = {
     },
   },
   query: {
-    create: {
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          display: "New Bucket",
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        location: {
-          required: true,
-          default: "us-east-1",
-          type: DatasourceFieldType.STRING,
-        },
-        grantFullControl: {
-          display: "Grant full control",
-          type: DatasourceFieldType.STRING,
-        },
-        grantRead: {
-          display: "Grant read",
-          type: DatasourceFieldType.STRING,
-        },
-        grantReadAcp: {
-          display: "Grant read ACP",
-          type: DatasourceFieldType.STRING,
-        },
-        grantWrite: {
-          display: "Grant write",
-          type: DatasourceFieldType.STRING,
-        },
-        grantWriteAcp: {
-          display: "Grant write ACP",
-          type: DatasourceFieldType.STRING,
-        },
-      },
-    },
     read: {
       type: QueryType.FIELDS,
       fields: {
         bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        delimiter: {
-          type: DatasourceFieldType.STRING,
-        },
-        marker: {
-          type: DatasourceFieldType.STRING,
-        },
-        maxKeys: {
-          type: DatasourceFieldType.NUMBER,
-          display: "Max Keys",
-        },
-        prefix: {
-          type: DatasourceFieldType.STRING,
-        },
-      },
-    },
-    readCsv: {
-      displayName: "Read CSV",
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        key: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-      },
-    },
-    delete: {
-      type: QueryType.FIELDS,
-      fields: {
-        bucket: {
-          type: DatasourceFieldType.STRING,
-          required: true,
-        },
-        delete: {
-          type: DatasourceFieldType.JSON,
+          type: "string",
           required: true,
         },
       },
     },
   },
-  extra: {
-    acl: {
-      required: false,
-      displayName: "ACL",
-      type: DatasourceFieldType.LIST,
-      data: {
-        create: [
-          "private",
-          "public-read",
-          "public-read-write",
-          "authenticated-read",
-        ],
-      },
-    },
-  },
 }
 
 class S3Integration implements IntegrationBase {
@@ -164,93 +67,14 @@ class S3Integration implements IntegrationBase {
     this.client = new AWS.S3(this.config)
   }
 
-  async create(query: {
-    bucket: string
-    location: string
-    grantFullControl: string
-    grantRead: string
-    grantReadAcp: string
-    grantWrite: string
-    grantWriteAcp: string
-    extra: {
-      acl: string
-    }
-  }) {
-    let params: any = {
-      Bucket: query.bucket,
-      ACL: query.extra?.acl,
-      GrantFullControl: query.grantFullControl,
-      GrantRead: query.grantRead,
-      GrantReadACP: query.grantReadAcp,
-      GrantWrite: query.grantWrite,
-      GrantWriteACP: query.grantWriteAcp,
-    }
-    if (query.location) {
-      params["CreateBucketConfiguration"] = {
-        LocationConstraint: query.location,
-      }
-    }
-    return await this.client.createBucket(params).promise()
-  }
-
-  async read(query: {
-    bucket: string
-    delimiter: string
-    expectedBucketOwner: string
-    marker: string
-    maxKeys: number
-    prefix: string
-  }) {
+  async read(query: { bucket: string }) {
     const response = await this.client
       .listObjects({
         Bucket: query.bucket,
-        Delimiter: query.delimiter,
-        Marker: query.marker,
-        MaxKeys: query.maxKeys,
-        Prefix: query.prefix,
       })
       .promise()
     return response.Contents
   }
-
-  async readCsv(query: { bucket: string; key: string }) {
-    const stream = this.client
-      .getObject({
-        Bucket: query.bucket,
-        Key: query.key,
-      })
-      .createReadStream()
-
-    let csvError = false
-    return new Promise((resolve, reject) => {
-      stream.on("error", (err: Error) => {
-        reject(err)
-      })
-      const response = csv()
-        .fromStream(stream)
-        .on("error", () => {
-          csvError = true
-        })
-      stream.on("finish", () => {
-        resolve(response)
-      })
-    }).catch(err => {
-      if (csvError) {
-        throw new Error("Could not read CSV")
-      } else {
-        throw err
-      }
-    })
-  }
-
-  async delete(query: { bucket: string; delete: string }) {
-    return await this.client
-      .deleteObjects({
-        Bucket: query.bucket,
-        Delete: JSON.parse(query.delete),
-      })
-      .promise()
-  }
 }
 
 export default {

package/src/integrations/tests/s3.spec.ts

@@ -18,95 +18,11 @@ describe("S3 Integration", () => {
   })
 
   it("calls the read method with the correct params", async () => {
-    await config.integration.read({
+    const response = await config.integration.read({
       bucket: "test",
-      delimiter: "/",
-      marker: "file.txt",
-      maxKeys: 999,
-      prefix: "directory/",
     })
     expect(config.integration.client.listObjects).toHaveBeenCalledWith({
       Bucket: "test",
-      Delimiter: "/",
-      Marker: "file.txt",
-      MaxKeys: 999,
-      Prefix: "directory/",
-    })
-  })
-
-  it("calls the create method with the correct params", async () => {
-    await config.integration.create({
-      bucket: "test",
-      location: "af-south-1",
-      grantFullControl: "me",
-      grantRead: "him",
-      grantReadAcp: "her",
-      grantWrite: "she",
-      grantWriteAcp: "he",
-      objectLockEnabledForBucket: true,
-      extra: {
-        acl: "private",
-      },
-    })
-    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
-      Bucket: "test",
-      CreateBucketConfiguration: {
-        LocationConstraint: "af-south-1",
-      },
-      GrantFullControl: "me",
-      GrantRead: "him",
-      GrantReadACP: "her",
-      GrantWrite: "she",
-      GrantWriteACP: "he",
-      ACL: "private",
-    })
-  })
-
-  it("does not add undefined location constraint when calling the create method", async () => {
-    await config.integration.create({
-      bucket: "test",
-    })
-    expect(config.integration.client.createBucket).toHaveBeenCalledWith({
-      Bucket: "test",
-      GrantFullControl: undefined,
-      GrantRead: undefined,
-      GrantReadACP: undefined,
-      GrantWrite: undefined,
-      GrantWriteACP: undefined,
-      ACL: undefined,
-    })
-  })
-
-  it("calls the delete method with the correct params ", async () => {
-    await config.integration.delete({
-      bucket: "test",
-      delete: `{
-        "Objects": [
-          {
-         "Key": "HappyFace.jpg", 
-         "VersionId": "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b"
-        }, 
-          {
-         "Key": "HappyFace.jpg", 
-         "VersionId": "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd"
-        }
-       ]
-      }`,
-    })
-    expect(config.integration.client.deleteObjects).toHaveBeenCalledWith({
-      Bucket: "test",
-      Delete: {
-        Objects: [
-          {
-            Key: "HappyFace.jpg",
-            VersionId: "2LWg7lQLnY41.maGB5Z6SWW.dcq0vx7b",
-          },
-          {
-            Key: "HappyFace.jpg",
-            VersionId: "yoz3HB.ZhCS_tKVEmIOr7qYyyAaZSKVd",
-          },
-        ],
-      },
     })
   })
 })

package/src/integrations/utils.ts

@@ -230,7 +230,7 @@ function shouldCopySpecialColumn(
   const fetchedIsNumber =
     !fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER
   return (
-    specialTypes.indexOf(column.type as FieldTypes) !== -1 ||
+    specialTypes.indexOf(column.type) !== -1 ||
     (fetchedIsNumber && column.type === FieldTypes.BOOLEAN)
   )
 }
@@ -292,11 +292,7 @@ export function finaliseExternalTables(
     if (table.primary == null || table.primary.length === 0) {
       errors[name] = BuildSchemaErrors.NO_KEY
       continue
-    } else if (
-      schemaFields.find(field =>
-        invalidColumns.includes(field as InvalidColumns)
-      )
-    ) {
+    } else if (schemaFields.find(field => invalidColumns.includes(field))) {
       errors[name] = BuildSchemaErrors.INVALID_COLUMN
       continue
     }

package/src/middleware/appInfo.js

@@ -0,0 +1,19 @@
+const { isDevAppID, isProdAppID } = require("../db/utils")
+
+exports.AppType = {
+  DEV: "dev",
+  PROD: "prod",
+}
+
+exports.middleware =
+  ({ appType } = {}) =>
+  (ctx, next) => {
+    const appId = ctx.appId
+    if (appType === exports.AppType.DEV && appId && !isDevAppID(appId)) {
+      ctx.throw(400, "Only apps in development support this endpoint")
+    }
+    if (appType === exports.AppType.PROD && appId && !isProdAppID(appId)) {
+      ctx.throw(400, "Only apps in production support this endpoint")
+    }
+    return next()
+  }

package/src/middleware/authorized.ts

@@ -1,13 +1,23 @@
-import { roles, permissions, auth, context } from "@budibase/backend-core"
-import { Role } from "@budibase/types"
-import builderMiddleware from "./builder"
-import { isWebhookEndpoint } from "./utils"
+import {
+  getUserRoleHierarchy,
+  getRequiredResourceRole,
+  BUILTIN_ROLE_IDS,
+} from "@budibase/backend-core/roles"
+const {
+  PermissionTypes,
+  PermissionLevels,
+  doesHaveBasePermission,
+} = require("@budibase/backend-core/permissions")
+const builderMiddleware = require("./builder")
+const { isWebhookEndpoint } = require("./utils")
+const { buildCsrfMiddleware } = require("@budibase/backend-core/auth")
+const { getAppId } = require("@budibase/backend-core/context")
 
 function hasResource(ctx: any) {
   return ctx.resourceId != null
 }
 
-const csrf = auth.buildCsrfMiddleware()
+const csrf = buildCsrfMiddleware()
 
 /**
  * Apply authorization to the requested resource:
@@ -23,7 +33,7 @@ const checkAuthorized = async (
 ) => {
   // check if this is a builder api and the user is not a builder
   const isBuilder = ctx.user && ctx.user.builder && ctx.user.builder.global
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+  const isBuilderApi = permType === PermissionTypes.BUILDER
   if (isBuilderApi && !isBuilder) {
     return ctx.throw(403, "Not Authorized")
   }
@@ -41,10 +51,10 @@ const checkAuthorizedResource = async (
   permLevel: any
 ) => {
   // get the user's roles
-  const roleId = ctx.roleId || roles.BUILTIN_ROLE_IDS.PUBLIC
-  const userRoles = (await roles.getUserRoleHierarchy(roleId, {
+  const roleId = ctx.roleId || BUILTIN_ROLE_IDS.PUBLIC
+  const userRoles = (await getUserRoleHierarchy(roleId, {
     idOnly: false,
-  })) as Role[]
+  })) as { _id: string }[]
   const permError = "User does not have permission"
   // check if the user has the required role
   if (resourceRoles.length > 0) {
@@ -56,9 +66,7 @@ const checkAuthorizedResource = async (
       ctx.throw(403, permError)
     }
     // fallback to the base permissions when no resource roles are found
-  } else if (
-    !permissions.doesHaveBasePermission(permType, permLevel, userRoles)
-  ) {
+  } else if (!doesHaveBasePermission(permType, permLevel, userRoles)) {
     ctx.throw(403, permError)
   }
 }
@@ -83,22 +91,21 @@ export = (permType: any, permLevel: any = null, opts = { schema: false }) =>
     let resourceRoles: any = []
     let otherLevelRoles: any = []
     const otherLevel =
-      permLevel === permissions.PermissionLevel.READ
-        ? permissions.PermissionLevel.WRITE
-        : permissions.PermissionLevel.READ
-    const appId = context.getAppId()
+      permLevel === PermissionLevels.READ
+        ? PermissionLevels.WRITE
+        : PermissionLevels.READ
+    const appId = getAppId()
     if (appId && hasResource(ctx)) {
-      resourceRoles = await roles.getRequiredResourceRole(permLevel, ctx)
+      resourceRoles = await getRequiredResourceRole(permLevel, ctx)
       if (opts && opts.schema) {
-        otherLevelRoles = await roles.getRequiredResourceRole(otherLevel, ctx)
+        otherLevelRoles = await getRequiredResourceRole(otherLevel, ctx)
       }
     }
 
     // if the resource is public, proceed
     if (
-      resourceRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC) ||
-      (otherLevelRoles &&
-        otherLevelRoles.includes(roles.BUILTIN_ROLE_IDS.PUBLIC))
+      resourceRoles.includes(BUILTIN_ROLE_IDS.PUBLIC) ||
+      (otherLevelRoles && otherLevelRoles.includes(BUILTIN_ROLE_IDS.PUBLIC))
     ) {
       return next()
     }

package/src/middleware/{builder.ts → builder.js}

@@ -1,16 +1,14 @@
-import {
-  APP_DEV_PREFIX,
-  DocumentType,
-  getGlobalIDFromUserMetadataID,
-} from "../db/utils"
-import {
+const { APP_DEV_PREFIX } = require("../db/utils")
+const {
   doesUserHaveLock,
   updateLock,
   checkDebounce,
   setDebounce,
-} from "../utilities/redis"
-import { db as dbCore, cache, permissions } from "@budibase/backend-core"
-import { BBContext, Database } from "@budibase/types"
+} = require("../utilities/redis")
+const { doWithDB } = require("@budibase/backend-core/db")
+const { DocumentType, getGlobalIDFromUserMetadataID } = require("../db/utils")
+const { PermissionTypes } = require("@budibase/backend-core/permissions")
+const { app: appCache } = require("@budibase/backend-core/cache")
 
 const DEBOUNCE_TIME_SEC = 30
 
@@ -23,11 +21,11 @@ const DEBOUNCE_TIME_SEC = 30
  * through the authorized middleware                *
  ****************************************************/
 
-async function checkDevAppLocks(ctx: BBContext) {
+async function checkDevAppLocks(ctx) {
   const appId = ctx.appId
 
   // if any public usage, don't proceed
-  if (!ctx.user?._id && !ctx.user?.userId) {
+  if (!ctx.user._id && !ctx.user.userId) {
     return
   }
 
@@ -43,34 +41,34 @@ async function checkDevAppLocks(ctx: BBContext) {
   await updateLock(appId, ctx.user)
 }
 
-async function updateAppUpdatedAt(ctx: BBContext) {
+async function updateAppUpdatedAt(ctx) {
   const appId = ctx.appId
   // if debouncing skip this update
   // get methods also aren't updating
   if (ctx.method === "GET" || (await checkDebounce(appId))) {
     return
   }
-  await dbCore.doWithDB(appId, async (db: Database) => {
+  await doWithDB(appId, async db => {
     const metadata = await db.get(DocumentType.APP_METADATA)
     metadata.updatedAt = new Date().toISOString()
 
-    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user?.userId!)
+    metadata.updatedBy = getGlobalIDFromUserMetadataID(ctx.user.userId)
 
     const response = await db.put(metadata)
     metadata._rev = response.rev
-    await cache.app.invalidateAppMetadata(appId, metadata)
+    await appCache.invalidateAppMetadata(appId, metadata)
     // set a new debounce record with a short TTL
     await setDebounce(appId, DEBOUNCE_TIME_SEC)
   })
 }
 
-export = async function builder(ctx: BBContext, permType: string) {
+module.exports = async (ctx, permType) => {
   const appId = ctx.appId
   // this only functions within an app context
   if (!appId) {
     return
   }
-  const isBuilderApi = permType === permissions.PermissionType.BUILDER
+  const isBuilderApi = permType === PermissionTypes.BUILDER
   const referer = ctx.headers["referer"]
 
   const overviewPath = "/builder/portal/overview/"