@budibase/server 2.0.33 → 2.0.34-alpha.0

package/src/api/controllers/row/index.ts

@@ -31,8 +31,11 @@ export async function patch(ctx: any): Promise<any> {
     return save(ctx)
   }
   try {
-    const { row, table } = await quotas.addQuery(() =>
-      pickApi(tableId).patch(ctx)
+    const { row, table } = await quotas.addQuery(
+      () => pickApi(tableId).patch(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
     ctx.status = 200
     ctx.eventEmitter &&
@@ -54,7 +57,9 @@ export const save = async (ctx: any) => {
   }
   try {
     const { row, table } = await quotas.addRow(() =>
-      quotas.addQuery(() => pickApi(tableId).save(ctx))
+      quotas.addQuery(() => pickApi(tableId).save(ctx), {
+        datasourceId: tableId,
+      })
     )
     ctx.status = 200
     ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
@@ -68,7 +73,9 @@ export const save = async (ctx: any) => {
 export async function fetchView(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -77,7 +84,9 @@ export async function fetchView(ctx: any) {
 export async function fetch(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -86,7 +95,9 @@ export async function fetch(ctx: any) {
 export async function find(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -98,8 +109,11 @@ export async function destroy(ctx: any) {
   const tableId = getTableId(ctx)
   let response, row
   if (inputs.rows) {
-    let { rows } = await quotas.addQuery(() =>
-      pickApi(tableId).bulkDestroy(ctx)
+    let { rows } = await quotas.addQuery(
+      () => pickApi(tableId).bulkDestroy(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
     await quotas.removeRows(rows.length)
     response = rows
@@ -107,7 +121,9 @@ export async function destroy(ctx: any) {
       ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
     }
   } else {
-    let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx))
+    let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), {
+      datasourceId: tableId,
+    })
     await quotas.removeRow()
     response = resp.response
     row = resp.row
@@ -123,7 +139,9 @@ export async function search(ctx: any) {
   const tableId = getTableId(ctx)
   try {
     ctx.status = 200
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -141,8 +159,11 @@ export async function validate(ctx: any) {
 export async function fetchEnrichedRow(ctx: any) {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() =>
-      pickApi(tableId).fetchEnrichedRow(ctx)
+    ctx.body = await quotas.addQuery(
+      () => pickApi(tableId).fetchEnrichedRow(ctx),
+      {
+        datasourceId: tableId,
+      }
     )
   } catch (err) {
     ctx.throw(400, err)
@@ -152,7 +173,9 @@ export async function fetchEnrichedRow(ctx: any) {
 export const exportRows = async (ctx: any) => {
   const tableId = getTableId(ctx)
   try {
-    ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx))
+    ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
+      datasourceId: tableId,
+    })
   } catch (err) {
     ctx.throw(400, err)
   }

package/src/api/controllers/static/index.ts

@@ -5,7 +5,7 @@ require("svelte/register")
 const send = require("koa-send")
 const { resolve, join } = require("../../../utilities/centralPath")
 const uuid = require("uuid")
-const { ObjectStoreBuckets } = require("../../../constants")
+const { ObjectStoreBuckets, ATTACHMENT_DIR } = require("../../../constants")
 const { processString } = require("@budibase/string-templates")
 const {
   loadHandlebarsFile,
@@ -90,7 +90,7 @@ export const uploadFile = async function (ctx: any) {
 
     return prepareUpload({
       file,
-      s3Key: `${ctx.appId}/attachments/${processedFileName}`,
+      s3Key: `${ctx.appId}/${ATTACHMENT_DIR}/${processedFileName}`,
       bucket: ObjectStoreBuckets.APPS,
     })
   })

package/src/api/controllers/static/templates/preview.hbs

@@ -56,6 +56,16 @@
         return
       }
 
+      // If this is a custom event, try and handle it
+      if (parsed.runtimeEvent) {
+        const { name, payload } = parsed
+        if (window.handleBuilderRuntimeEvent) {
+          window.handleBuilderRuntimeEvent(name, payload)
+        }
+        return
+      }
+
+      // Otherwise this is a full reload message
       // Extract data from message
       const {
         selectedComponentId,

package/src/api/controllers/table/bulkFormula.js

@@ -1,10 +1,11 @@
 const { FieldTypes, FormulaTypes } = require("../../../constants")
-const { getAllInternalTables, clearColumns } = require("./utils")
+const { clearColumns } = require("./utils")
 const { doesContainStrings } = require("@budibase/string-templates")
 const { cloneDeep } = require("lodash/fp")
 const { isEqual, uniq } = require("lodash")
 const { updateAllFormulasInTable } = require("../row/staticFormula")
 const { getAppDB } = require("@budibase/backend-core/context")
+const sdk = require("../../../sdk")
 
 function isStaticFormula(column) {
   return (
@@ -39,7 +40,7 @@ function getFormulaThatUseColumn(table, columnNames) {
  */
 async function checkIfFormulaNeedsCleared(table, { oldTable, deletion }) {
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables()).filter(
+  const tables = (await sdk.tables.getAllInternalTables()).filter(
     tbl => tbl._id !== table._id
   )
   const schemaToUse = oldTable ? oldTable.schema : table.schema
@@ -99,7 +100,7 @@ async function updateRelatedFormulaLinksOnTables(
 ) {
   const db = getAppDB()
   // start by retrieving all tables, remove the current table from the list
-  const tables = (await getAllInternalTables()).filter(
+  const tables = (await sdk.tables.getAllInternalTables()).filter(
     tbl => tbl._id !== table._id
   )
   // clone the tables, so we can compare at end

package/src/api/controllers/table/external.js

@@ -3,7 +3,6 @@ const {
   breakExternalTableId,
 } = require("../../../integrations/utils")
 const {
-  getTable,
   generateForeignKey,
   generateJunctionTableName,
   foreignKeyStructure,
@@ -20,6 +19,7 @@ const csvParser = require("../../../utilities/csvParser")
 const { handleRequest } = require("../row/external")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
+const sdk = require("../../../sdk")
 
 async function makeTableRequest(
   datasource,
@@ -181,7 +181,7 @@ exports.save = async function (ctx) {
 
   let oldTable
   if (ctx.request.body && ctx.request.body._id) {
-    oldTable = await getTable(ctx.request.body._id)
+    oldTable = await sdk.tables.getTable(ctx.request.body._id)
   }
 
   if (hasTypeChanged(tableToSave, oldTable)) {
@@ -281,7 +281,7 @@ exports.save = async function (ctx) {
 }
 
 exports.destroy = async function (ctx) {
-  const tableToDelete = await getTable(ctx.params.tableId)
+  const tableToDelete = await sdk.tables.getTable(ctx.params.tableId)
   if (!tableToDelete || !tableToDelete.created) {
     ctx.throw(400, "Cannot delete tables which weren't created in Budibase.")
   }
@@ -303,7 +303,7 @@ exports.destroy = async function (ctx) {
 }
 
 exports.bulkImport = async function (ctx) {
-  const table = await getTable(ctx.params.tableId)
+  const table = await sdk.tables.getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   if (!dataImport || !dataImport.schema || !dataImport.csvString) {
     ctx.throw(400, "Provided data import information is invalid.")

package/src/api/controllers/table/index.js

@@ -4,8 +4,8 @@ const csvParser = require("../../../utilities/csvParser")
 const { isExternalTable, isSQL } = require("../../../integrations/utils")
 const { getDatasourceParams } = require("../../../db/utils")
 const { getAppDB } = require("@budibase/backend-core/context")
-const { getTable, getAllInternalTables } = require("./utils")
 const { events } = require("@budibase/backend-core")
+const sdk = require("../../../sdk")
 
 function pickApi({ tableId, table }) {
   if (table && !tableId) {
@@ -23,7 +23,7 @@ function pickApi({ tableId, table }) {
 exports.fetch = async function (ctx) {
   const db = getAppDB()
 
-  const internal = await getAllInternalTables()
+  const internal = await sdk.tables.getAllInternalTables()
 
   const externalTables = await db.allDocs(
     getDatasourceParams("plus", {
@@ -50,7 +50,7 @@ exports.fetch = async function (ctx) {
 
 exports.find = async function (ctx) {
   const tableId = ctx.params.tableId
-  ctx.body = await getTable(tableId)
+  ctx.body = await sdk.tables.getTable(tableId)
 }
 
 exports.save = async function (ctx) {
@@ -101,7 +101,7 @@ exports.validateCSVSchema = async function (ctx) {
   const { csvString, schema = {}, tableId } = ctx.request.body
   let existingTable
   if (tableId) {
-    existingTable = await getTable(tableId)
+    existingTable = await sdk.tables.getTable(tableId)
   }
   let result = await csvParser.parse(csvString, schema)
   if (existingTable) {

package/src/api/controllers/table/internal.ts

@@ -1,12 +1,7 @@
 import { updateLinks, EventType } from "../../../db/linkedRows"
 import { getRowParams, generateTableID } from "../../../db/utils"
 import { FieldTypes } from "../../../constants"
-import {
-  TableSaveFunctions,
-  hasTypeChanged,
-  getTable,
-  handleDataImport,
-} from "./utils"
+import { TableSaveFunctions, hasTypeChanged, handleDataImport } from "./utils"
 const { getAppDB } = require("@budibase/backend-core/context")
 import { isTest } from "../../../environment"
 import {
@@ -18,6 +13,8 @@ import { Table } from "@budibase/types"
 import { quotas } from "@budibase/pro"
 import { isEqual } from "lodash"
 import { cloneDeep } from "lodash/fp"
+import env from "../../../environment"
+import sdk from "../../../sdk"
 
 function checkAutoColumns(table: Table, oldTable: Table) {
   if (!table.schema) {
@@ -153,7 +150,9 @@ export async function destroy(ctx: any) {
   await db.bulkDocs(
     rows.rows.map((row: any) => ({ ...row.doc, _deleted: true }))
   )
-  await quotas.removeRows(rows.rows.length)
+  await quotas.removeRows(rows.rows.length, {
+    tableId: ctx.params.tableId,
+  })
 
   // update linked rows
   await updateLinks({
@@ -165,7 +164,7 @@ export async function destroy(ctx: any) {
   await db.remove(tableToDelete)
 
   // remove table search index
-  if (!isTest()) {
+  if (!isTest() || env.COUCH_DB_URL) {
     const currentIndexes = await db.getIndexes()
     const existingIndex = currentIndexes.indexes.find(
       (existing: any) => existing.name === `search:${ctx.params.tableId}`
@@ -185,7 +184,7 @@ export async function destroy(ctx: any) {
 }
 
 export async function bulkImport(ctx: any) {
-  const table = await getTable(ctx.params.tableId)
+  const table = await sdk.tables.getTable(ctx.params.tableId)
   const { dataImport } = ctx.request.body
   await handleDataImport(ctx.user, table, dataImport)
   return table

package/src/api/controllers/table/utils.ts

@@ -1,11 +1,5 @@
 import { transform } from "../../../utilities/csvParser"
-import {
-  getRowParams,
-  generateRowID,
-  InternalTables,
-  getTableParams,
-  BudibaseInternalDB,
-} from "../../../db/utils"
+import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
 import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
 import {
@@ -17,11 +11,6 @@ import {
   SwitchableTypes,
   CanSwitchTypes,
 } from "../../../constants"
-import {
-  isExternalTable,
-  breakExternalTableId,
-  isSQL,
-} from "../../../integrations/utils"
 import { getViews, saveView } from "../view/utils"
 import viewTemplate from "../view/viewBuilder"
 const { getAppDB } = require("@budibase/backend-core/context")
@@ -148,7 +137,9 @@ export async function handleDataImport(user: any, table: any, dataImport: any) {
     finalData.push(row)
   }
 
-  await quotas.addRows(finalData.length, () => db.bulkDocs(finalData))
+  await quotas.addRows(finalData.length, () => db.bulkDocs(finalData), {
+    tableId: table._id,
+  })
   await events.rows.imported(table, "csv", finalData.length)
   return table
 }
@@ -254,46 +245,6 @@ class TableSaveFunctions {
   }
 }
 
-export async function getAllInternalTables() {
-  const db = getAppDB()
-  const internalTables = await db.allDocs(
-    getTableParams(null, {
-      include_docs: true,
-    })
-  )
-  return internalTables.rows.map((tableDoc: any) => ({
-    ...tableDoc.doc,
-    type: "internal",
-    sourceId: BudibaseInternalDB._id,
-  }))
-}
-
-export async function getAllExternalTables(datasourceId: any) {
-  const db = getAppDB()
-  const datasource = await db.get(datasourceId)
-  if (!datasource || !datasource.entities) {
-    throw "Datasource is not configured fully."
-  }
-  return datasource.entities
-}
-
-export async function getExternalTable(datasourceId: any, tableName: any) {
-  const entities = await getAllExternalTables(datasourceId)
-  return entities[tableName]
-}
-
-export async function getTable(tableId: any) {
-  const db = getAppDB()
-  if (isExternalTable(tableId)) {
-    let { datasourceId, tableName } = breakExternalTableId(tableId)
-    const datasource = await db.get(datasourceId)
-    const table = await getExternalTable(datasourceId, tableName)
-    return { ...table, sql: isSQL(datasource) }
-  } else {
-    return db.get(tableId)
-  }
-}
-
 export async function checkForViewUpdates(
   table: any,
   rename: any,

package/src/api/controllers/view/index.js

@@ -3,12 +3,12 @@ const { apiFileReturn } = require("../../../utilities/fileSystem")
 const exporters = require("./exporters")
 const { saveView, getView, getViews, deleteView } = require("./utils")
 const { fetchView } = require("../row")
-const { getTable } = require("../table/utils")
 const { FieldTypes } = require("../../../constants")
 const { getAppDB } = require("@budibase/backend-core/context")
 const { events } = require("@budibase/backend-core")
 const { DocumentType } = require("../../../db/utils")
 const { cloneDeep, isEqual } = require("lodash")
+const sdk = require("../../../sdk")
 
 exports.fetch = async ctx => {
   ctx.body = await getViews()
@@ -144,7 +144,7 @@ exports.exportView = async ctx => {
 
   let schema = view && view.meta && view.meta.schema
   const tableId = ctx.params.tableId || view.meta.tableId
-  const table = await getTable(tableId)
+  const table = await sdk.tables.getTable(tableId)
   if (!schema) {
     schema = table.schema
   }

package/src/api/routes/backup.ts

@@ -0,0 +1,10 @@
+import Router from "@koa/router"
+import * as controller from "../controllers/backup"
+import authorized from "../../middleware/authorized"
+import { BUILDER } from "@budibase/backend-core/permissions"
+
+const router = new Router()
+
+router.get("/api/backups/export", authorized(BUILDER), controller.exportAppDump)
+
+export default router

package/src/api/routes/index.ts

@@ -25,11 +25,17 @@ import devRoutes from "./dev"
 import cloudRoutes from "./cloud"
 import migrationRoutes from "./migrations"
 import pluginRoutes from "./plugin"
+import Router from "@koa/router"
+import { api } from "@budibase/pro"
 
 export { default as staticRoutes } from "./static"
 export { default as publicRoutes } from "./public"
 
-export const mainRoutes = [
+const appBackupRoutes = api.appBackups
+const scheduleRoutes = api.schedules
+export const mainRoutes: Router[] = [
+  appBackupRoutes,
+  backupRoutes,
   authRoutes,
   deployRoutes,
   layoutRoutes,
@@ -49,14 +55,14 @@ export const mainRoutes = [
   permissionRoutes,
   datasourceRoutes,
   queryRoutes,
-  backupRoutes,
   metadataRoutes,
   devRoutes,
   cloudRoutes,
-  // these need to be handled last as they still use /api/:tableId
-  // this could be breaking as koa may recognise other routes as this
-  tableRoutes,
   rowRoutes,
   migrationRoutes,
   pluginRoutes,
+  scheduleRoutes,
+  // these need to be handled last as they still use /api/:tableId
+  // this could be breaking as koa may recognise other routes as this
+  tableRoutes,
 ]

package/src/api/routes/tests/backup.spec.js

@@ -21,7 +21,7 @@ describe("/backups", () => {
         .set(config.defaultHeaders())
         .expect(200)
       expect(res.text).toBeDefined()
-      expect(res.text.includes(`"db_name":"${config.getAppId()}"`)).toEqual(true)
+      expect(res.headers["content-type"]).toEqual("application/gzip")
       expect(events.app.exported.mock.calls.length).toBe(1)
     })
 

package/src/api/routes/tests/{deployment.spec.js → deployment.spec.ts}

@@ -1,5 +1,5 @@
-const setup = require("./utilities")
-const { events } = require("@budibase/backend-core")
+import setup from "./utilities"
+import { events } from "@budibase/backend-core"
 
 describe("/deployments", () => {
   let request = setup.getRequest()
@@ -19,7 +19,7 @@ describe("/deployments", () => {
         .set(config.defaultHeaders())
         .expect("Content-Type", /json/)
         .expect(200)
-      expect(events.app.published.mock.calls.length).toBe(1)
+      expect((events.app.published as jest.Mock).mock.calls.length).toBe(1)
     })
   })
-})
+})

package/src/api/routes/tests/row.spec.js

@@ -34,18 +34,13 @@ describe("/rows", () => {
       .expect(status)
 
   const getRowUsage = async () => {
-    return config.doInContext(null, () =>
-      quotas.getCurrentUsageValue(QuotaUsageType.STATIC, StaticQuotaName.ROWS)
-    )
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.STATIC, StaticQuotaName.ROWS))
+    return total
   }
 
   const getQueryUsage = async () => {
-    return config.doInContext(null, () =>
-      quotas.getCurrentUsageValue(
-        QuotaUsageType.MONTHLY,
-        MonthlyQuotaName.QUERIES
-      )
-    )
+    const { total } = await config.doInContext(null, () => quotas.getCurrentUsageValues(QuotaUsageType.MONTHLY, MonthlyQuotaName.QUERIES))
+    return total
   }
 
   const assertRowUsage = async expected => {
@@ -60,26 +55,26 @@ describe("/rows", () => {
 
   describe("save, load, update", () => {
     it("returns a success message when the row is created", async () => {
-      // const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
-      //
-      // const res = await request
-      //   .post(`/api/${row.tableId}/rows`)
-      //   .send(row)
-      //   .set(config.defaultHeaders())
-      //   .expect('Content-Type', /json/)
-      //   .expect(200)
-      // expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
-      // expect(res.body.name).toEqual("Test Contact")
-      // expect(res.body._rev).toBeDefined()
-      // await assertRowUsage(rowUsage + 1)
-      // await assertQueryUsage(queryUsage + 1)
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
+
+      const res = await request
+        .post(`/api/${row.tableId}/rows`)
+        .send(row)
+        .set(config.defaultHeaders())
+        .expect('Content-Type', /json/)
+        .expect(200)
+      expect(res.res.statusMessage).toEqual(`${table.name} saved successfully`)
+      expect(res.body.name).toEqual("Test Contact")
+      expect(res.body._rev).toBeDefined()
+      await assertRowUsage(rowUsage + 1)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("updates a row successfully", async () => {
       const existing = await config.createRow()
-      // const rowUsage = await getRowUsage()
-      // const queryUsage = await getQueryUsage()
+      const rowUsage = await getRowUsage()
+      const queryUsage = await getQueryUsage()
 
       const res = await request
         .post(`/api/${table._id}/rows`)
@@ -97,8 +92,8 @@ describe("/rows", () => {
         `${table.name} updated successfully.`
       )
       expect(res.body.name).toEqual("Updated Name")
-      // await assertRowUsage(rowUsage)
-      // await assertQueryUsage(queryUsage + 1)
+      await assertRowUsage(rowUsage)
+      await assertQueryUsage(queryUsage + 1)
     })
 
     it("should load a row", async () => {

package/src/app.ts

@@ -37,6 +37,8 @@ import {
 } from "./utilities/workerRequests"
 import { watch } from "./watch"
 import { initialise as initialiseWebsockets } from "./websocket"
+import sdk from "./sdk"
+import * as pro from "@budibase/pro"
 
 const app = new Koa()
 
@@ -102,12 +104,25 @@ server.on("close", async () => {
   }
 })
 
+const initPro = async () => {
+  await pro.init({
+    backups: {
+      processing: {
+        exportAppFn: sdk.backups.exportApp,
+        importAppFn: sdk.backups.importApp,
+        statsFn: sdk.backups.calculateBackupStats,
+      },
+    },
+  })
+}
+
 module.exports = server.listen(env.PORT || 0, async () => {
   console.log(`Budibase running on ${JSON.stringify(server.address())}`)
   env._set("PORT", server.address().port)
   eventEmitter.emitPort(env.PORT)
   fileSystem.init()
   await redis.init()
+  await initPro()
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/automations/automationUtils.js

@@ -1,10 +1,10 @@
-const { getTable } = require("../api/controllers/table/utils")
 const {
   findHBSBlocks,
   decodeJSBinding,
   isJSBinding,
   encodeJSBinding,
 } = require("@budibase/string-templates")
+const sdk = require("../sdk")
 
 /**
  * When values are input to the system generally they will be of type string as this is required for template strings.
@@ -64,7 +64,7 @@ exports.cleanInputValues = (inputs, schema) => {
  * @returns {Promise<Object>} The cleaned up rows object, will should now have all the required primitive types.
  */
 exports.cleanUpRow = async (tableId, row) => {
-  let table = await getTable(tableId)
+  let table = await sdk.tables.getTable(tableId)
   return exports.cleanInputValues(row, { properties: table.schema })
 }