@budibase/server 1.0.46-alpha.7 → 1.0.48

package/src/api/controllers/table/utils.js

@@ -8,15 +8,17 @@ const {
 const { isEqual } = require("lodash/fp")
 const { AutoFieldSubTypes, FieldTypes } = require("../../../constants")
 const { inputProcessing } = require("../../../utilities/rowProcessor")
-const { USERS_TABLE_SCHEMA, SwitchableTypes } = require("../../../constants")
+const {
+  USERS_TABLE_SCHEMA,
+  SwitchableTypes,
+  CanSwitchTypes,
+} = require("../../../constants")
 const {
   isExternalTable,
   breakExternalTableId,
-  isSQL,
 } = require("../../../integrations/utils")
 const { getViews, saveView } = require("../view/utils")
 const viewTemplate = require("../view/viewBuilder")
-const usageQuota = require("../../../utilities/usageQuota")
 
 exports.checkForColumnUpdates = async (db, oldTable, updatedTable) => {
   let updatedRows = []
@@ -113,11 +115,7 @@ exports.handleDataImport = async (appId, user, table, dataImport) => {
     finalData.push(row)
   }
 
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length, {
-    dryRun: true,
-  })
   await db.bulkDocs(finalData)
-  await usageQuota.update(usageQuota.Properties.ROW, finalData.length)
   let response = await db.put(table)
   table._rev = response._rev
   return table
@@ -248,9 +246,7 @@ exports.getTable = async (appId, tableId) => {
   const db = new CouchDB(appId)
   if (isExternalTable(tableId)) {
     let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await exports.getExternalTable(appId, datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
+    return exports.getExternalTable(appId, datasourceId, tableName)
   } else {
     return db.get(tableId)
   }
@@ -348,6 +344,23 @@ exports.foreignKeyStructure = (keyName, meta = null) => {
   return structure
 }
 
+exports.areSwitchableTypes = (type1, type2) => {
+  if (
+    SwitchableTypes.indexOf(type1) === -1 &&
+    SwitchableTypes.indexOf(type2) === -1
+  ) {
+    return false
+  }
+  for (let option of CanSwitchTypes) {
+    const index1 = option.indexOf(type1),
+      index2 = option.indexOf(type2)
+    if (index1 !== -1 && index2 !== -1 && index1 !== index2) {
+      return true
+    }
+  }
+  return false
+}
+
 exports.hasTypeChanged = (table, oldTable) => {
   if (!oldTable) {
     return false
@@ -358,7 +371,7 @@ exports.hasTypeChanged = (table, oldTable) => {
       continue
     }
     const newType = table.schema[key].type
-    if (oldType !== newType && SwitchableTypes.indexOf(oldType) === -1) {
+    if (oldType !== newType && !exports.areSwitchableTypes(oldType, newType)) {
       return true
     }
   }

package/src/api/routes/hosting.js

@@ -0,0 +1,13 @@
+const Router = require("@koa/router")
+const controller = require("../controllers/hosting")
+const authorized = require("../../middleware/authorized")
+const { BUILDER } = require("@budibase/backend-core/permissions")
+
+const router = Router()
+
+router
+  .get("/api/hosting/urls", authorized(BUILDER), controller.fetchUrls)
+  // this isn't risky, doesn't return anything about apps other than names and URLs
+  .get("/api/hosting/apps", controller.getDeployedApps)
+
+module.exports = router

package/src/api/routes/index.js

@@ -20,6 +20,7 @@ const integrationRoutes = require("./integration")
 const permissionRoutes = require("./permission")
 const datasourceRoutes = require("./datasource")
 const queryRoutes = require("./query")
+const hostingRoutes = require("./hosting")
 const backupRoutes = require("./backup")
 const metadataRoutes = require("./metadata")
 const devRoutes = require("./dev")
@@ -45,6 +46,7 @@ exports.mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
+  hostingRoutes,
   backupRoutes,
   metadataRoutes,
   devRoutes,

package/src/api/routes/static.js

@@ -46,10 +46,5 @@ router
   )
   // TODO: this likely needs to be secured in some way
   .get("/:appId/:path*", controller.serveApp)
-  .post(
-    "/api/attachments/:datasourceId/url",
-    authorized(PermissionTypes.TABLE, PermissionLevels.READ),
-    controller.getSignedUploadURL
-  )
 
 module.exports = router

package/src/api/routes/tests/application.spec.js

@@ -53,8 +53,8 @@ describe("/applications", () => {
 
   describe("fetch", () => {
     it("lists all applications", async () => {
-      await config.createApp("app1")
-      await config.createApp("app2")
+      await config.createApp(request, "app1")
+      await config.createApp(request, "app2")
 
       const res = await request
         .get(`/api/applications?status=${AppStatus.DEV}`)

package/src/api/routes/tests/hosting.spec.js

@@ -0,0 +1,36 @@
+// mock out node fetch for this
+jest.mock("node-fetch")
+
+const { checkBuilderEndpoint } = require("./utilities/TestFunctions")
+const setup = require("./utilities")
+
+describe("/hosting", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+  let app
+
+  afterAll(setup.afterAll)
+
+  beforeEach(async () => {
+    app = await config.init()
+  })
+
+  describe("fetchUrls", () => {
+    it("should be able to fetch current app URLs", async () => {
+      const res = await request
+        .get(`/api/hosting/urls`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      expect(res.body.app).toEqual(`http://localhost:10000/app`)
+    })
+
+    it("should apply authorization to endpoint", async () => {
+      await checkBuilderEndpoint({
+        config,
+        method: "GET",
+        url: `/api/hosting/urls`,
+      })
+    })
+  })
+})

package/src/automations/steps/createRow.js

@@ -1,5 +1,6 @@
 const rowController = require("../../api/controllers/row")
 const automationUtils = require("../automationUtils")
+const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 
@@ -82,9 +83,10 @@ exports.run = async function ({ inputs, appId, emitter }) {
       inputs.row.tableId,
       inputs.row
     )
-    await usage.update(usage.Properties.ROW, 1, { dryRun: true })
+    if (env.USE_QUOTAS) {
+      await usage.update(usage.Properties.ROW, 1)
+    }
     await rowController.save(ctx)
-    await usage.update(usage.Properties.ROW, 1)
     return {
       row: inputs.row,
       response: ctx.body,

package/src/automations/steps/deleteRow.js

@@ -1,4 +1,5 @@
 const rowController = require("../../api/controllers/row")
+const env = require("../../environment")
 const usage = require("../../utilities/usageQuota")
 const { buildCtx } = require("./utils")
 const automationUtils = require("../automationUtils")
@@ -73,7 +74,9 @@ exports.run = async function ({ inputs, appId, emitter }) {
   })
 
   try {
-    await usage.update(usage.Properties.ROW, -1)
+    if (env.isProd()) {
+      await usage.update(usage.Properties.ROW, -1)
+    }
     await rowController.destroy(ctx)
     return {
       response: ctx.body,

package/src/constants/index.js

@@ -45,13 +45,16 @@ exports.FieldTypes = {
   INTERNAL: "internal",
 }
 
-exports.SwitchableTypes = [
-  exports.FieldTypes.STRING,
-  exports.FieldTypes.OPTIONS,
-  exports.FieldTypes.NUMBER,
-  exports.FieldTypes.BOOLEAN,
+exports.CanSwitchTypes = [
+  [exports.FieldTypes.JSON, exports.FieldTypes.ARRAY],
+  [exports.FieldTypes.STRING, exports.FieldTypes.OPTIONS],
+  [exports.FieldTypes.BOOLEAN, exports.FieldTypes.NUMBER],
 ]
 
+exports.SwitchableTypes = exports.CanSwitchTypes.reduce((prev, current) =>
+  prev ? prev.concat(current) : current
+)
+
 exports.RelationshipTypes = {
   ONE_TO_MANY: "one-to-many",
   MANY_TO_ONE: "many-to-one",
@@ -75,10 +78,6 @@ exports.DataSourceOperation = {
   DELETE_TABLE: "DELETE_TABLE",
 }
 
-exports.DatasourceAuthTypes = {
-  GOOGLE: "google",
-}
-
 exports.SortDirection = {
   ASCENDING: "ASCENDING",
   DESCENDING: "DESCENDING",

package/src/definitions/datasource.ts

@@ -47,7 +47,6 @@ export enum SourceNames {
   ARANGODB = "ARANGODB",
   REST = "REST",
   ORACLE = "ORACLE",
-  GOOGLE_SHEETS = "GOOGLE_SHEETS",
 }
 
 export enum IncludeRelationships {
@@ -87,8 +86,6 @@ export interface ExtraQueryConfig {
 export interface Integration {
   docs: string
   plus?: boolean
-  auth?: { type: string }
-  relationships?: boolean
   description: string
   friendlyName: string
   datasource: {}

package/src/environment.js

@@ -38,7 +38,6 @@ module.exports = {
   MINIO_ACCESS_KEY: process.env.MINIO_ACCESS_KEY,
   MINIO_SECRET_KEY: process.env.MINIO_SECRET_KEY,
   USE_QUOTAS: process.env.USE_QUOTAS,
-  EXCLUDE_QUOTAS_TENANTS: process.env.EXCLUDE_QUOTAS_TENANTS,
   REDIS_URL: process.env.REDIS_URL,
   REDIS_PASSWORD: process.env.REDIS_PASSWORD,
   INTERNAL_API_KEY: process.env.INTERNAL_API_KEY,

package/src/integrations/base/sql.ts

@@ -72,22 +72,16 @@ class InternalBuilder {
 
   // right now we only do filters on the specific table being queried
   addFilters(
+    tableName: string,
     query: KnexQuery,
-    filters: SearchFilters | undefined,
-    opts: { relationship?: boolean; tableName?: string }
+    filters: SearchFilters | undefined
   ): KnexQuery {
     function iterate(
       structure: { [key: string]: any },
       fn: (key: string, value: any) => void
     ) {
       for (let [key, value] of Object.entries(structure)) {
-        const isRelationshipField = key.includes(".")
-        if (!opts.relationship && !isRelationshipField) {
-          fn(`${opts.tableName}.${key}`, value)
-        }
-        if (opts.relationship && isRelationshipField) {
-          fn(key, value)
-        }
+        fn(`${tableName}.${key}`, value)
       }
     }
     if (!filters) {
@@ -278,7 +272,7 @@ class InternalBuilder {
     if (foundOffset) {
       query = query.offset(foundOffset)
     }
-    query = this.addFilters(query, filters, { tableName })
+    query = this.addFilters(tableName, query, filters)
     // add sorting to pre-query
     query = this.addSorting(query, json)
     // @ts-ignore
@@ -291,21 +285,20 @@ class InternalBuilder {
       preQuery = this.addSorting(preQuery, json)
     }
     // handle joins
-    query = this.addRelationships(
+    return this.addRelationships(
       knex,
       preQuery,
       selectStatement,
       tableName,
       relationships
     )
-    return this.addFilters(query, filters, { relationship: true })
   }
 
   update(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, body, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
     const parsedBody = parseBody(body)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.update(parsedBody)
@@ -317,7 +310,7 @@ class InternalBuilder {
   delete(knex: Knex, json: QueryJson, opts: QueryOptions): KnexQuery {
     const { endpoint, filters } = json
     let query: KnexQuery = knex(endpoint.entityId)
-    query = this.addFilters(query, filters, { tableName: endpoint.entityId })
+    query = this.addFilters(endpoint.entityId, query, filters)
     // mysql can't use returning
     if (opts.disableReturning) {
       return query.delete()

package/src/integrations/index.ts

@@ -9,7 +9,6 @@ const airtable = require("./airtable")
 const mysql = require("./mysql")
 const arangodb = require("./arangodb")
 const rest = require("./rest")
-const googlesheets = require("./googlesheets")
 const { SourceNames } = require("../definitions/datasource")
 
 const DEFINITIONS = {
@@ -24,7 +23,6 @@ const DEFINITIONS = {
   [SourceNames.MYSQL]: mysql.schema,
   [SourceNames.ARANGODB]: arangodb.schema,
   [SourceNames.REST]: rest.schema,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.schema,
 }
 
 const INTEGRATIONS = {
@@ -39,7 +37,6 @@ const INTEGRATIONS = {
   [SourceNames.MYSQL]: mysql.integration,
   [SourceNames.ARANGODB]: arangodb.integration,
   [SourceNames.REST]: rest.integration,
-  [SourceNames.GOOGLE_SHEETS]: googlesheets.integration,
 }
 
 // optionally add oracle integration if the oracle binary can be installed

package/src/integrations/tests/sql.spec.js

@@ -119,22 +119,6 @@ describe("SQL query builder", () => {
     })
   })
 
-  it("should allow filtering on a related field", () => {
-    const query = sql._query(generateReadJson({
-      filters: {
-        equal: {
-          age: 10,
-          "task.name": "task 1",
-        },
-      },
-    }))
-    // order of bindings changes because relationship filters occur outside inner query
-    expect(query).toEqual({
-      bindings: [10, limit, "task 1"],
-      sql: `select * from (select * from "${TABLE_NAME}" where "${TABLE_NAME}"."age" = $1 limit $2) as "${TABLE_NAME}" where "task"."name" = $3`
-    })
-  })
-
   it("should test an create statement", () => {
     const query = sql._query(generateCreateJson(TABLE_NAME, {
       name: "Michael",

package/src/integrations/utils.ts

@@ -1,5 +1,6 @@
-import { SourceNames, SqlQuery } from "../definitions/datasource"
+import { SqlQuery } from "../definitions/datasource"
 import { Datasource, Table } from "../definitions/common"
+import { SourceNames } from "../definitions/datasource"
 import { DocumentTypes, SEPARATOR } from "../db/utils"
 import { FieldTypes, BuildSchemaErrors, InvalidColumns } from "../constants"
 
@@ -126,12 +127,7 @@ export function isSQL(datasource: Datasource): boolean {
   if (!datasource || !datasource.source) {
     return false
   }
-  const SQL = [
-    SourceNames.POSTGRES,
-    SourceNames.SQL_SERVER,
-    SourceNames.MYSQL,
-    SourceNames.ORACLE,
-  ]
+  const SQL = [SourceNames.POSTGRES, SourceNames.SQL_SERVER, SourceNames.MYSQL]
   return SQL.indexOf(datasource.source) !== -1
 }
 
@@ -143,59 +139,11 @@ export function isIsoDateString(str: string) {
   return d.toISOString() === str
 }
 
-/**
- * This function will determine whether a column is a relationship and whether it
- * is currently valid. The reason for the validity check is that tables can be deleted
- * outside of Budibase control and if this is the case it will break Budibase relationships.
- * The tableIds is a list passed down from the main finalise tables function, which is
- * based on the tables that have just been fetched. This will only really be used on subsequent
- * fetches to the first one - if the user is periodically refreshing Budibase knowledge of tables.
- * @param column The column to check, to see if it is a valid relationship.
- * @param tableIds The IDs of the tables which currently exist.
- */
-function shouldCopyRelationship(
-  column: { type: string; tableId?: string },
-  tableIds: [string]
-) {
-  return (
-    column.type === FieldTypes.LINK &&
-    column.tableId &&
-    tableIds.includes(column.tableId)
-  )
-}
-
-/**
- * Similar function to the shouldCopyRelationship function, but instead this looks for options and boolean
- * types. It is possible to switch a string -> options and a number -> boolean (and vice versus) need to make
- * sure that these get copied over when tables are fetched. Also checks whether they are still valid, if a
- * column has changed type in the external database then copying it over may not be possible.
- * @param column The column to check for options or boolean type.
- * @param fetchedColumn The fetched column to check for the type in the external database.
- */
-function shouldCopySpecialColumn(
-  column: { type: string },
-  fetchedColumn: { type: string } | undefined
-) {
-  return (
-    column.type === FieldTypes.OPTIONS ||
-    ((!fetchedColumn || fetchedColumn.type === FieldTypes.NUMBER) &&
-      column.type === FieldTypes.BOOLEAN)
-  )
-}
-
-/**
- * Looks for columns which need to be copied over into the new table definitions, like relationships
- * and options types.
- * @param tableName The name of the table which is being checked.
- * @param table The specific table which is being checked.
- * @param entities All the tables that existed before - the old table definitions.
- * @param tableIds The IDs of the tables which exist now, to check if anything has been removed.
- */
+// add the existing relationships from the entities if they exist, to prevent them from being overridden
 function copyExistingPropsOver(
   tableName: string,
   table: Table,
-  entities: { [key: string]: any },
-  tableIds: [string]
+  entities: { [key: string]: any }
 ) {
   if (entities && entities[tableName]) {
     if (entities[tableName].primaryDisplay) {
@@ -206,10 +154,12 @@ function copyExistingPropsOver(
       if (!existingTableSchema.hasOwnProperty(key)) {
         continue
       }
-      const column = existingTableSchema[key]
       if (
-        shouldCopyRelationship(column, tableIds) ||
-        shouldCopySpecialColumn(column, table.schema[key])
+        existingTableSchema[key].type === FieldTypes.LINK ||
+        existingTableSchema[key].type === FieldTypes.OPTIONS ||
+        existingTableSchema[key].type === FieldTypes.ARRAY ||
+        ((!table.schema[key] || table.schema[key].type === FieldTypes.NUMBER) &&
+          existingTableSchema[key].type === FieldTypes.BOOLEAN)
       ) {
         table.schema[key] = existingTableSchema[key]
       }
@@ -218,13 +168,6 @@ function copyExistingPropsOver(
   return table
 }
 
-/**
- * Look through the final table definitions to see if anything needs to be
- * copied over from the old and if any errors have occurred mark them so
- * that the user can be made aware.
- * @param tables The list of tables that have been retrieved from the external database.
- * @param entities The old list of tables, if there was any to look for definitions in.
- */
 export function finaliseExternalTables(
   tables: { [key: string]: any },
   entities: { [key: string]: any }
@@ -232,8 +175,6 @@ export function finaliseExternalTables(
   const invalidColumns = Object.values(InvalidColumns)
   let finalTables: { [key: string]: any } = {}
   const errors: { [key: string]: string } = {}
-  // @ts-ignore
-  const tableIds: [string] = Object.values(tables).map(table => table._id)
   for (let [name, table] of Object.entries(tables)) {
     const schemaFields = Object.keys(table.schema)
     // make sure every table has a key
@@ -245,7 +186,7 @@ export function finaliseExternalTables(
       continue
     }
     // make sure all previous props have been added back
-    finalTables[name] = copyExistingPropsOver(name, table, entities, tableIds)
+    finalTables[name] = copyExistingPropsOver(name, table, entities)
   }
   // sort the tables by name
   finalTables = Object.entries(finalTables)

package/src/middleware/tests/usageQuota.spec.js

@@ -1,5 +1,11 @@
 jest.mock("../../db")
 jest.mock("../../utilities/usageQuota")
+jest.mock("../../environment", () => ({
+  isTest: () => true,
+  isProd: () => false,
+  isDev: () => true,
+  _set: () => {},
+}))
 jest.mock("@budibase/backend-core/tenancy", () => ({
   getTenantId: () => "testing123"
 }))
@@ -23,10 +29,9 @@ class TestConfiguration {
       },
       req: {
         method: "POST",
-        url: "/applications"
+        url: "/rows"
       }
     }
-    usageQuota.useQuotas = () => true
   }
 
   executeMiddleware() {
@@ -108,6 +113,7 @@ describe("usageQuota middleware", () => {
 
   it("calculates and persists the correct usage quota for the relevant action", async () => {
     config.setUrl("/rows")
+    config.setProd(true)
 
     await config.executeMiddleware()
 
@@ -115,20 +121,20 @@ describe("usageQuota middleware", () => {
     expect(config.next).toHaveBeenCalled()
   })
 
-  // it("calculates the correct file size from a file upload call and adds it to quota", async () => {
-  //   config.setUrl("/upload")
-  //   config.setProd(true)
-  //   config.setFiles([
-  //     {
-  //       size: 100
-  //     },
-  //     {
-  //       size: 10000
-  //     },
-  //   ])
-  //   await config.executeMiddleware()
-
-  //   expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
-  //   expect(config.next).toHaveBeenCalled()
-  // })
+  it("calculates the correct file size from a file upload call and adds it to quota", async () => {
+    config.setUrl("/upload")
+    config.setProd(true)
+    config.setFiles([
+      {
+        size: 100
+      },
+      {
+        size: 10000
+      },
+    ])
+    await config.executeMiddleware()
+
+    expect(usageQuota.update).toHaveBeenCalledWith("storage", 10100)
+    expect(config.next).toHaveBeenCalled()
+  })
 })