@budibase/server 2.4.40 → 2.4.42-alpha.0

package/src/api/controllers/application.ts

@@ -20,10 +20,10 @@ import {
   cache,
   tenancy,
   context,
-  errors,
   events,
   migrations,
   objectStore,
+  ErrorCode,
 } from "@budibase/backend-core"
 import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
@@ -44,7 +44,6 @@ import {
   Layout,
   Screen,
   MigrationType,
-  BBContext,
   Database,
   UserCtx,
 } from "@budibase/types"
@@ -74,14 +73,14 @@ async function getScreens() {
   ).rows.map((row: any) => row.doc)
 }
 
-function getUserRoleId(ctx: BBContext) {
+function getUserRoleId(ctx: UserCtx) {
   return !ctx.user?.role || !ctx.user.role._id
     ? roles.BUILTIN_ROLE_IDS.PUBLIC
     : ctx.user.role._id
 }
 
 function checkAppUrl(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   url: string,
   currentAppId?: string
@@ -95,7 +94,7 @@ function checkAppUrl(
 }
 
 function checkAppName(
-  ctx: BBContext,
+  ctx: UserCtx,
   apps: App[],
   name: string,
   currentAppId?: string
@@ -160,7 +159,7 @@ async function addDefaultTables(db: Database) {
   await db.bulkDocs([...defaultDbDocs])
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const dev = ctx.query && ctx.query.status === AppStatus.DEV
   const all = ctx.query && ctx.query.status === AppStatus.ALL
   const apps = (await dbCore.getAllApps({ dev, all })) as App[]
@@ -185,7 +184,7 @@ export async function fetch(ctx: BBContext) {
   ctx.body = await checkAppMetadata(apps)
 }
 
-export async function fetchAppDefinition(ctx: BBContext) {
+export async function fetchAppDefinition(ctx: UserCtx) {
   const layouts = await getLayouts()
   const userRoleId = getUserRoleId(ctx)
   const accessController = new roles.AccessController()
@@ -231,7 +230,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
   }
 }
 
-async function performAppCreate(ctx: BBContext) {
+async function performAppCreate(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   const name = ctx.request.body.name,
     possibleUrl = ctx.request.body.url
@@ -360,7 +359,7 @@ async function creationEvents(request: any, app: App) {
   }
 }
 
-async function appPostCreate(ctx: BBContext, app: App) {
+async function appPostCreate(ctx: UserCtx, app: App) {
   const tenantId = tenancy.getTenantId()
   await migrations.backPopulateMigrations({
     type: MigrationType.APP,
@@ -378,7 +377,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
           return quotas.addRows(rowCount)
         })
       } catch (err: any) {
-        if (err.code && err.code === errors.codes.USAGE_LIMIT_EXCEEDED) {
+        if (err.code && err.code === ErrorCode.USAGE_LIMIT_EXCEEDED) {
           // this import resulted in row usage exceeding the quota
           // delete the app
           // skip pre and post-steps as no rows have been added to quotas yet
@@ -391,7 +390,7 @@ async function appPostCreate(ctx: BBContext, app: App) {
   }
 }
 
-export async function create(ctx: BBContext) {
+export async function create(ctx: UserCtx) {
   const newApplication = await quotas.addApp(() => performAppCreate(ctx))
   await appPostCreate(ctx, newApplication)
   await cache.bustCache(cache.CacheKey.CHECKLIST)
@@ -401,7 +400,7 @@ export async function create(ctx: BBContext) {
 
 // This endpoint currently operates as a PATCH rather than a PUT
 // Thus name and url fields are handled only if present
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const apps = (await dbCore.getAllApps({ dev: true })) as App[]
   // validation
   const name = ctx.request.body.name,
@@ -421,7 +420,7 @@ export async function update(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function updateClient(ctx: BBContext) {
+export async function updateClient(ctx: UserCtx) {
   // Get current app version
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -445,7 +444,7 @@ export async function updateClient(ctx: BBContext) {
   ctx.body = app
 }
 
-export async function revertClient(ctx: BBContext) {
+export async function revertClient(ctx: UserCtx) {
   // Check app can be reverted
   const db = context.getAppDB()
   const application = await db.get(DocumentType.APP_METADATA)
@@ -471,7 +470,7 @@ export async function revertClient(ctx: BBContext) {
   ctx.body = app
 }
 
-const unpublishApp = async (ctx: any) => {
+async function unpublishApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
 
@@ -487,7 +486,7 @@ const unpublishApp = async (ctx: any) => {
   return result
 }
 
-async function destroyApp(ctx: BBContext) {
+async function destroyApp(ctx: UserCtx) {
   let appId = ctx.params.appId
   appId = dbCore.getProdAppID(appId)
   const devAppId = dbCore.getDevAppID(appId)
@@ -515,12 +514,12 @@ async function destroyApp(ctx: BBContext) {
   return result
 }
 
-async function preDestroyApp(ctx: BBContext) {
+async function preDestroyApp(ctx: UserCtx) {
   const { rows } = await getUniqueRows([ctx.params.appId])
   ctx.rowCount = rows.length
 }
 
-async function postDestroyApp(ctx: BBContext) {
+async function postDestroyApp(ctx: UserCtx) {
   const rowCount = ctx.rowCount
   await groups.cleanupApp(ctx.params.appId)
   if (rowCount) {
@@ -528,7 +527,7 @@ async function postDestroyApp(ctx: BBContext) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   await preDestroyApp(ctx)
   const result = await destroyApp(ctx)
   await postDestroyApp(ctx)
@@ -536,7 +535,7 @@ export async function destroy(ctx: BBContext) {
   ctx.body = result
 }
 
-export const unpublish = async (ctx: BBContext) => {
+export async function unpublish(ctx: UserCtx) {
   const prodAppId = dbCore.getProdAppID(ctx.params.appId)
   const dbExists = await dbCore.dbExists(prodAppId)
 
@@ -551,7 +550,7 @@ export const unpublish = async (ctx: BBContext) => {
   ctx.status = 204
 }
 
-export async function sync(ctx: BBContext) {
+export async function sync(ctx: UserCtx) {
   const appId = ctx.params.appId
   try {
     ctx.body = await sdk.applications.syncApp(appId)

package/src/api/controllers/datasource.ts

@@ -84,8 +84,9 @@ export async function buildSchemaFromDb(ctx: UserCtx) {
   setDefaultDisplayColumns(datasource)
   const dbResp = await db.put(datasource)
   datasource._rev = dbResp.rev
+  const cleanedDatasource = await sdk.datasources.removeSecretSingle(datasource)
 
-  const response: any = { datasource }
+  const response: any = { datasource: cleanedDatasource }
   if (error) {
     response.error = error
   }

package/src/api/controllers/public/metrics.ts

@@ -0,0 +1,251 @@
+import { Ctx } from "@budibase/types"
+import { users as userCore, db as dbCore } from "@budibase/backend-core"
+import { quotas, licensing } from "@budibase/pro"
+
+import os from "os"
+
+export async function fetch(ctx: Ctx) {
+  // *** OPERATING SYSTEM ***
+  const freeMem = os.freemem()
+  const totalMem = os.totalmem()
+  const usedMem = totalMem - freeMem
+  const uptime = os.uptime()
+
+  // *** APPS ***
+  const allDatabases = await dbCore.getAllDbs()
+  const devAppIDs = await dbCore.getDevAppIDs()
+  const prodAppIDs = await dbCore.getProdAppIDs()
+  const allAppIds = await dbCore.getAllApps({ idsOnly: true })
+
+  // *** USERS ***
+  const usersObject = await userCore.getAllUserIds()
+
+  // *** QUOTAS ***
+  const usage = await quotas.getQuotaUsage()
+  const license = await licensing.cache.getCachedLicense()
+  const appsQuotaUsage = usage.usageQuota.apps
+  const rowsQuotaUsage = usage.usageQuota.rows
+  const pluginsQuotaUsage = usage.usageQuota.plugins
+  const userGroupsQuotaUsage = usage.usageQuota.userGroups
+  const queryQuotaUsage = usage.monthly.current.queries
+  const automationsQuotaUsage = usage.monthly.current.automations
+  const appsQuotaLimit = license.quotas.usage.static.apps.value
+  const rowsQuotaLimit = license.quotas.usage.static.rows.value
+  const userGroupsQuotaLimit = license.quotas.usage.static.userGroups.value
+  const pluginsQuotaLimit = license.quotas.usage.static.plugins.value
+  const queryQuotaLimit = license.quotas.usage.monthly.queries.value
+  const automationsQuotaLimit = license.quotas.usage.monthly.automations.value
+
+  // *** BUILD THE OUTPUT STRING ***
+  var outputString = ""
+
+  // **** budibase_os_uptime ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_uptime",
+    "Time in seconds that the host operating system has been up",
+    "counter",
+    uptime
+  )
+
+  // **** budibase_os_free_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_free_mem",
+    "Bytes of memory free for usage on the host operating system",
+    "gauge",
+    freeMem
+  )
+
+  // **** budibase_os_total_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_total_mem",
+    "Total bytes of memory on the host operating system",
+    "gauge",
+    totalMem
+  )
+
+  // **** budibase_os_used_mem ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_used_mem",
+    "Total bytes of memory in use on the host operating system",
+    "gauge",
+    usedMem
+  )
+
+  // **** budibase_os_load1 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load1",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[0]
+  )
+
+  // **** budibase_os_load5 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load5",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[1]
+  )
+  // **** budibase_os_load15 ****
+  outputString += convertToOpenMetrics(
+    "budibase_os_load15",
+    "Host operating system load average",
+    "gauge",
+    os.loadavg()[2]
+  )
+
+  // **** budibase_tenant_user_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_user_count",
+    "The number of users created",
+    "gauge",
+    usersObject.length
+  )
+
+  // **** budibase_tenant_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_app_count",
+    "The number of apps created by a user",
+    "gauge",
+    allAppIds.length
+  )
+
+  // **** budibase_tenant_production_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_production_app_count",
+    "The number of apps a user has published",
+    "gauge",
+    prodAppIDs.length
+  )
+
+  // **** budibase_tenant_dev_app_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_dev_app_count",
+    "The number of apps a user has unpublished in development",
+    "gauge",
+    devAppIDs.length
+  )
+
+  // **** budibase_tenant_db_count ****
+  outputString += convertToOpenMetrics(
+    "budibase_tenant_db_count",
+    "The number of couchdb databases including global tables such as _users",
+    "gauge",
+    allDatabases.length
+  )
+
+  // **** budibase_quota_usage_apps ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_apps",
+    "The number of apps created",
+    "gauge",
+    appsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_apps ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_apps",
+    "The limit on the number of apps that can be created",
+    "gauge",
+    appsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : appsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_rows ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_rows",
+    "The number of database rows used from the quota",
+    "gauge",
+    rowsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_rows ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_rows",
+    "The limit on the number of rows that can be created",
+    "gauge",
+    rowsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : rowsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_plugins ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_plugins",
+    "The number of plugins in use",
+    "gauge",
+    pluginsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_plugins ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_plugins",
+    "The limit on the number of plugins that can be created",
+    "gauge",
+    pluginsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : pluginsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_user_groups ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_user_groups",
+    "The number of user groups created",
+    "gauge",
+    userGroupsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_user_groups ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_user_groups",
+    "The limit on the number of user groups that can be created",
+    "gauge",
+    userGroupsQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : userGroupsQuotaLimit
+  )
+
+  // **** budibase_quota_usage_queries ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_queries",
+    "The number of queries used in the current month",
+    "gauge",
+    queryQuotaUsage
+  )
+
+  // **** budibase_quota_limit_queries ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_queries",
+    "The limit on the number of queries for the current month",
+    "gauge",
+    queryQuotaLimit == -1 ? Number.MAX_SAFE_INTEGER : queryQuotaLimit
+  )
+
+  // **** budibase_quota_usage_automations ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_usage_automations",
+    "The number of automations used in the current month",
+    "gauge",
+    automationsQuotaUsage
+  )
+
+  // **** budibase_quota_limit_automations ****
+  outputString += convertToOpenMetrics(
+    "budibase_quota_limit_automations",
+    "The limit on the number of automations that can be created",
+    "gauge",
+    automationsQuotaLimit == -1
+      ? Number.MAX_SAFE_INTEGER
+      : automationsQuotaLimit
+  )
+  ctx.body = outputString
+  ctx.set("Content-Type", "text/plain")
+}
+
+export function convertToOpenMetrics(
+  metricName: string,
+  metricHelp: string,
+  metricType: string,
+  metricValue: number
+) {
+  return `# HELP ${metricName} ${metricHelp}.
+# TYPE ${metricName} ${metricType}
+${metricName} ${metricValue}\n`
+}
+
+export default {
+  fetch,
+}

package/src/api/controllers/row/external.ts

@@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Operation,
-  BBContext,
+  UserCtx,
   Row,
   PaginationJson,
   Table,
@@ -21,6 +21,7 @@ import {
   SortJson,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import * as utils from "./utils"
 
 const { cleanExportRows } = require("./utils")
 
@@ -49,12 +50,19 @@ export async function handleRequest(
   )
 }
 
-export async function patch(ctx: BBContext) {
+export async function patch(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
   const id = inputs._id
   // don't save the ID to db
   delete inputs._id
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
     row: inputs,
@@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
   })
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.CREATE, tableId, {
     row: inputs,
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
 }
 
-export async function fetchView(ctx: BBContext) {
+export async function fetchView(ctx: UserCtx) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
   const split = ctx.params.viewName.split("all_")
@@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
   return fetch(ctx)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   return handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const response = (await handleRequest(Operation.READ, tableId, {
@@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
   return response ? response[0] : response
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
@@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
   return { response: { ok: true }, row }
 }
 
-export async function bulkDestroy(ctx: BBContext) {
+export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
   let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: BBContext) {
+export async function search(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const { paginate, query, ...params } = ctx.request.body
   let { bookmark, limit } = params
@@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
   }
 }
 
-export async function validate(ctx: BBContext) {
-  // can't validate external right now - maybe in future
-  return { valid: true }
-}
-
-export async function exportRows(ctx: BBContext) {
+export async function exportRows(ctx: UserCtx) {
   const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
   const format = ctx.query.format
   const { columns } = ctx.request.body
@@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
   return apiFileReturn(exporter(headers, exportRows))
 }
 
-export async function fetchEnrichedRow(ctx: BBContext) {
+export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)

package/src/api/controllers/row/index.ts

@@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTable } from "../../../integrations/utils"
+import { Ctx } from "@budibase/types"
+import * as utils from "./utils"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -129,9 +131,17 @@ export async function search(ctx: any) {
   })
 }
 
-export async function validate(ctx: any) {
+export async function validate(ctx: Ctx) {
   const tableId = getTableId(ctx)
-  ctx.body = await pickApi(tableId).validate(ctx)
+  // external tables are hard to validate currently
+  if (isExternalTable(tableId)) {
+    ctx.body = { valid: true }
+  } else {
+    ctx.body = await utils.validate({
+      row: ctx.request.body,
+      tableId,
+    })
+  }
 }
 
 export async function fetchEnrichedRow(ctx: any) {

package/src/api/controllers/row/internal.ts

@@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function validate(ctx: Ctx) {
-  return utils.validate({
-    tableId: ctx.params.tableId,
-    row: ctx.request.body,
-  })
-}
-
 export async function exportRows(ctx: Ctx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)

package/src/api/controllers/row/utils.ts

@@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
-const validateJs = require("validate.js")
-const { cloneDeep } = require("lodash/fp")
 import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
+const validateJs = require("validate.js")
+const { cloneDeep } = require("lodash/fp")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -56,17 +56,17 @@ export async function validate({
 }) {
   let fetchedTable: Table
   if (!table) {
-    const db = context.getAppDB()
-    fetchedTable = await db.get(tableId)
+    fetchedTable = await sdk.tables.getTable(tableId)
   } else {
     fetchedTable = table
   }
   const errors: any = {}
   for (let fieldName of Object.keys(fetchedTable.schema)) {
-    const constraints = cloneDeep(fetchedTable.schema[fieldName].constraints)
-    const type = fetchedTable.schema[fieldName].type
+    const column = fetchedTable.schema[fieldName]
+    const constraints = cloneDeep(column.constraints)
+    const type = column.type
     // formulas shouldn't validated, data will be deleted anyway
-    if (type === FieldTypes.FORMULA) {
+    if (type === FieldTypes.FORMULA || column.autocolumn) {
       continue
     }
     // special case for options, need to always allow unselected (null)