@budibase/server 2.4.26 → 2.4.27-alpha.0

package/src/api/controllers/row/external.ts

@@ -12,7 +12,7 @@ import * as exporters from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Operation,
-  BBContext,
+  UserCtx,
   Row,
   PaginationJson,
   Table,
@@ -21,6 +21,7 @@ import {
   SortJson,
 } from "@budibase/types"
 import sdk from "../../../sdk"
+import * as utils from "./utils"
 
 const { cleanExportRows } = require("./utils")
 
@@ -49,12 +50,19 @@ export async function handleRequest(
   )
 }
 
-export async function patch(ctx: BBContext) {
+export async function patch(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
   const id = inputs._id
   // don't save the ID to db
   delete inputs._id
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.UPDATE, tableId, {
     id: breakRowIdField(id),
     row: inputs,
@@ -62,16 +70,23 @@ export async function patch(ctx: BBContext) {
   })
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const inputs = ctx.request.body
   const tableId = ctx.params.tableId
+  const validateResult = await utils.validate({
+    row: inputs,
+    tableId,
+  })
+  if (!validateResult.valid) {
+    throw { validation: validateResult.errors }
+  }
   return handleRequest(Operation.CREATE, tableId, {
     row: inputs,
     includeSqlRelationships: IncludeRelationship.EXCLUDE,
   })
 }
 
-export async function fetchView(ctx: BBContext) {
+export async function fetchView(ctx: UserCtx) {
   // there are no views in external datasources, shouldn't ever be called
   // for now just fetch
   const split = ctx.params.viewName.split("all_")
@@ -79,14 +94,14 @@ export async function fetchView(ctx: BBContext) {
   return fetch(ctx)
 }
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   return handleRequest(Operation.READ, tableId, {
     includeSqlRelationships: IncludeRelationship.INCLUDE,
   })
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const response = (await handleRequest(Operation.READ, tableId, {
@@ -96,7 +111,7 @@ export async function find(ctx: BBContext) {
   return response ? response[0] : response
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const id = ctx.request.body._id
   const { row } = (await handleRequest(Operation.DELETE, tableId, {
@@ -106,7 +121,7 @@ export async function destroy(ctx: BBContext) {
   return { response: { ok: true }, row }
 }
 
-export async function bulkDestroy(ctx: BBContext) {
+export async function bulkDestroy(ctx: UserCtx) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
   let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
@@ -122,7 +137,7 @@ export async function bulkDestroy(ctx: BBContext) {
   return { response: { ok: true }, rows: responses.map(resp => resp.row) }
 }
 
-export async function search(ctx: BBContext) {
+export async function search(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   const { paginate, query, ...params } = ctx.request.body
   let { bookmark, limit } = params
@@ -185,12 +200,7 @@ export async function search(ctx: BBContext) {
   }
 }
 
-export async function validate(ctx: BBContext) {
-  // can't validate external right now - maybe in future
-  return { valid: true }
-}
-
-export async function exportRows(ctx: BBContext) {
+export async function exportRows(ctx: UserCtx) {
   const { datasourceId, tableName } = breakExternalTableId(ctx.params.tableId)
   const format = ctx.query.format
   const { columns } = ctx.request.body
@@ -244,7 +254,7 @@ export async function exportRows(ctx: BBContext) {
   return apiFileReturn(exporter(headers, exportRows))
 }
 
-export async function fetchEnrichedRow(ctx: BBContext) {
+export async function fetchEnrichedRow(ctx: UserCtx) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)

package/src/api/controllers/row/index.ts

@@ -2,6 +2,8 @@ import { quotas } from "@budibase/pro"
 import * as internal from "./internal"
 import * as external from "./external"
 import { isExternalTable } from "../../../integrations/utils"
+import { Ctx } from "@budibase/types"
+import * as utils from "./utils"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -129,9 +131,12 @@ export async function search(ctx: any) {
   })
 }
 
-export async function validate(ctx: any) {
+export async function validate(ctx: Ctx) {
   const tableId = getTableId(ctx)
-  ctx.body = await pickApi(tableId).validate(ctx)
+  ctx.body = await utils.validate({
+    row: ctx.request.body,
+    tableId,
+  })
 }
 
 export async function fetchEnrichedRow(ctx: any) {

package/src/api/controllers/row/internal.ts

@@ -387,13 +387,6 @@ export async function search(ctx: Ctx) {
   return response
 }
 
-export async function validate(ctx: Ctx) {
-  return utils.validate({
-    tableId: ctx.params.tableId,
-    row: ctx.request.body,
-  })
-}
-
 export async function exportRows(ctx: Ctx) {
   const db = context.getAppDB()
   const table = await db.get(ctx.params.tableId)

package/src/api/controllers/row/utils.ts

@@ -4,11 +4,11 @@ import { FieldTypes } from "../../../constants"
 import { context } from "@budibase/backend-core"
 import { makeExternalQuery } from "../../../integrations/base/query"
 import { Row, Table } from "@budibase/types"
-const validateJs = require("validate.js")
-const { cloneDeep } = require("lodash/fp")
 import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
+const validateJs = require("validate.js")
+const { cloneDeep } = require("lodash/fp")
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -56,8 +56,7 @@ export async function validate({
 }) {
   let fetchedTable: Table
   if (!table) {
-    const db = context.getAppDB()
-    fetchedTable = await db.get(tableId)
+    fetchedTable = await sdk.tables.getTable(tableId)
   } else {
     fetchedTable = table
   }

package/src/api/controllers/table/external.ts

@@ -7,6 +7,7 @@ import {
   generateJunctionTableName,
   foreignKeyStructure,
   hasTypeChanged,
+  setStaticSchemas,
 } from "./utils"
 import { FieldTypes } from "../../../constants"
 import { makeExternalQuery } from "../../../integrations/base/query"
@@ -20,7 +21,7 @@ import {
   Operation,
   RenameColumn,
   FieldSchema,
-  BBContext,
+  UserCtx,
   TableRequest,
   RelationshipTypes,
 } from "@budibase/types"
@@ -194,20 +195,20 @@ function isRelationshipSetup(column: FieldSchema) {
   return column.foreignKey || column.through
 }
 
-export async function save(ctx: BBContext) {
-  const table: TableRequest = ctx.request.body
-  const renamed = table?._rename
+export async function save(ctx: UserCtx) {
+  const inputs: TableRequest = ctx.request.body
+  const renamed = inputs?._rename
   // can't do this right now
-  delete table.rows
+  delete inputs.rows
   const datasourceId = getDatasourceId(ctx.request.body)!
   // table doesn't exist already, note that it is created
-  if (!table._id) {
-    table.created = true
+  if (!inputs._id) {
+    inputs.created = true
   }
   let tableToSave: TableRequest = {
     type: "table",
-    _id: buildExternalTableId(datasourceId, table.name),
-    ...table,
+    _id: buildExternalTableId(datasourceId, inputs.name),
+    ...inputs,
   }
 
   let oldTable
@@ -224,6 +225,10 @@ export async function save(ctx: BBContext) {
   if (!datasource.entities) {
     datasource.entities = {}
   }
+
+  // GSheets is a specific case - only ever has a static primary key
+  tableToSave = setStaticSchemas(datasource, tableToSave)
+
   const oldTables = cloneDeep(datasource.entities)
   const tables: Record<string, Table> = datasource.entities
 
@@ -246,7 +251,7 @@ export async function save(ctx: BBContext) {
       const junctionTable = generateManyLinkSchema(
         datasource,
         schema,
-        table,
+        tableToSave,
         relatedTable
       )
       if (tables[junctionTable.name]) {
@@ -256,10 +261,12 @@ export async function save(ctx: BBContext) {
       extraTablesToUpdate.push(junctionTable)
     } else {
       const fkTable =
-        relationType === RelationshipTypes.ONE_TO_MANY ? table : relatedTable
+        relationType === RelationshipTypes.ONE_TO_MANY
+          ? tableToSave
+          : relatedTable
       const foreignKey = generateLinkSchema(
         schema,
-        table,
+        tableToSave,
         relatedTable,
         relationType
       )
@@ -271,11 +278,11 @@ export async function save(ctx: BBContext) {
         fkTable.constrained.push(foreignKey)
       }
       // foreign key is in other table, need to save it to external
-      if (fkTable._id !== table._id) {
+      if (fkTable._id !== tableToSave._id) {
         extraTablesToUpdate.push(fkTable)
       }
     }
-    generateRelatedSchema(schema, relatedTable, table, relatedColumnName)
+    generateRelatedSchema(schema, relatedTable, tableToSave, relatedColumnName)
     schema.main = true
   }
 
@@ -313,7 +320,7 @@ export async function save(ctx: BBContext) {
   return tableToSave
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const tableToDelete: TableRequest = await sdk.tables.getTable(
     ctx.params.tableId
   )
@@ -339,7 +346,7 @@ export async function destroy(ctx: BBContext) {
   return tableToDelete
 }
 
-export async function bulkImport(ctx: BBContext) {
+export async function bulkImport(ctx: UserCtx) {
   const table = await sdk.tables.getTable(ctx.params.tableId)
   const { rows }: { rows: unknown } = ctx.request.body
   const schema: unknown = table.schema
@@ -348,7 +355,7 @@ export async function bulkImport(ctx: BBContext) {
     ctx.throw(400, "Provided data import information is invalid.")
   }
 
-  const parsedRows = await parse(rows, schema)
+  const parsedRows = parse(rows, schema)
   await handleRequest(Operation.BULK_CREATE, table._id!, {
     rows: parsedRows,
   })

package/src/api/controllers/table/index.ts

@@ -8,7 +8,7 @@ import {
 import { isExternalTable, isSQL } from "../../../integrations/utils"
 import { getDatasourceParams } from "../../../db/utils"
 import { context, events } from "@budibase/backend-core"
-import { Table, BBContext } from "@budibase/types"
+import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
 import csv from "csvtojson"
 
@@ -25,7 +25,7 @@ function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
 }
 
 // covers both internal and external
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   const db = context.getAppDB()
 
   const internal = await sdk.tables.getAllInternalTables()
@@ -53,12 +53,12 @@ export async function fetch(ctx: BBContext) {
   ctx.body = [...internal, ...external]
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   ctx.body = await sdk.tables.getTable(tableId)
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const appId = ctx.appId
   const table = ctx.request.body
   const isImport = table.rows
@@ -79,7 +79,7 @@ export async function save(ctx: BBContext) {
   ctx.body = savedTable
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const appId = ctx.appId
   const tableId = ctx.params.tableId
   const deletedTable = await pickApi({ tableId }).destroy(ctx)
@@ -91,7 +91,7 @@ export async function destroy(ctx: BBContext) {
   ctx.body = { message: `Table ${tableId} deleted.` }
 }
 
-export async function bulkImport(ctx: BBContext) {
+export async function bulkImport(ctx: UserCtx) {
   const tableId = ctx.params.tableId
   await pickApi({ tableId }).bulkImport(ctx)
   // right now we don't trigger anything for bulk import because it
@@ -101,7 +101,7 @@ export async function bulkImport(ctx: BBContext) {
   ctx.body = { message: `Bulk rows created.` }
 }
 
-export async function csvToJson(ctx: BBContext) {
+export async function csvToJson(ctx: UserCtx) {
   const { csvString } = ctx.request.body
 
   const result = await csv().fromString(csvString)
@@ -110,7 +110,7 @@ export async function csvToJson(ctx: BBContext) {
   ctx.body = result
 }
 
-export async function validateNewTableImport(ctx: BBContext) {
+export async function validateNewTableImport(ctx: UserCtx) {
   const { rows, schema }: { rows: unknown; schema: unknown } = ctx.request.body
 
   if (isRows(rows) && isSchema(schema)) {
@@ -121,7 +121,7 @@ export async function validateNewTableImport(ctx: BBContext) {
   }
 }
 
-export async function validateExistingTableImport(ctx: BBContext) {
+export async function validateExistingTableImport(ctx: UserCtx) {
   const { rows, tableId }: { rows: unknown; tableId: unknown } =
     ctx.request.body
 

package/src/api/controllers/table/utils.ts

@@ -1,7 +1,11 @@
 import { parse, isSchema, isRows } from "../../../utilities/schema"
 import { getRowParams, generateRowID, InternalTables } from "../../../db/utils"
 import { isEqual } from "lodash"
-import { AutoFieldSubTypes, FieldTypes } from "../../../constants"
+import {
+  AutoFieldSubTypes,
+  FieldTypes,
+  GOOGLE_SHEETS_PRIMARY_KEY,
+} from "../../../constants"
 import {
   inputProcessing,
   cleanupAttachments,
@@ -16,7 +20,7 @@ import viewTemplate from "../view/viewBuilder"
 import { cloneDeep } from "lodash/fp"
 import { quotas } from "@budibase/pro"
 import { events, context } from "@budibase/backend-core"
-import { Database } from "@budibase/types"
+import { Database, Datasource, SourceName, Table } from "@budibase/types"
 
 export async function clearColumns(table: any, columnNames: any) {
   const db: Database = context.getAppDB()
@@ -392,5 +396,17 @@ export function hasTypeChanged(table: any, oldTable: any) {
   return false
 }
 
+// used for external tables, some of them will have static schemas that need
+// to be hard set
+export function setStaticSchemas(datasource: Datasource, table: Table) {
+  // GSheets is a specific case - only ever has a static primary key
+  if (table && datasource.source === SourceName.GOOGLE_SHEETS) {
+    table.primary = [GOOGLE_SHEETS_PRIMARY_KEY]
+    // if there is an id column, remove it, should never exist in GSheets
+    delete table.schema?.id
+  }
+  return table
+}
+
 const _TableSaveFunctions = TableSaveFunctions
 export { _TableSaveFunctions as TableSaveFunctions }

package/src/api/routes/public/index.ts

@@ -1,4 +1,5 @@
 import appEndpoints from "./applications"
+import metricEndpoints from "./metrics"
 import queryEndpoints from "./queries"
 import tableEndpoints from "./tables"
 import rowEndpoints from "./rows"
@@ -12,7 +13,7 @@ import env from "../../../environment"
 // below imports don't have declaration files
 const Router = require("@koa/router")
 const { RateLimit, Stores } = require("koa2-ratelimit")
-import { redis, permissions } from "@budibase/backend-core"
+import { middleware, redis, permissions } from "@budibase/backend-core"
 const { PermissionType, PermissionLevel } = permissions
 
 const PREFIX = "/api/public/v1"
@@ -91,6 +92,13 @@ function addToRouter(endpoints: any) {
   }
 }
 
+function applyAdminRoutes(endpoints: any) {
+  addMiddleware(endpoints.read, middleware.builderOrAdmin)
+  addMiddleware(endpoints.write, middleware.builderOrAdmin)
+  addToRouter(endpoints.read)
+  addToRouter(endpoints.write)
+}
+
 function applyRoutes(
   endpoints: any,
   permType: string,
@@ -119,6 +127,7 @@ function applyRoutes(
   addToRouter(endpoints.write)
 }
 
+applyAdminRoutes(metricEndpoints)
 applyRoutes(appEndpoints, PermissionType.APP, "appId")
 applyRoutes(tableEndpoints, PermissionType.TABLE, "tableId")
 applyRoutes(userEndpoints, PermissionType.USER, "userId")

package/src/api/routes/public/metrics.ts

@@ -0,0 +1,28 @@
+import controller from "../../controllers/public/metrics"
+import Endpoint from "./utils/Endpoint"
+
+const read = []
+
+/**
+ * @openapi
+ * /metrics:
+ *   get:
+ *     operationId: metricsGet
+ *     summary: Retrieve Budibase tenant metrics
+ *     description: Output metrics in OpenMetrics format compatible with Prometheus
+ *     tags:
+ *       - metrics
+ *     responses:
+ *       200:
+ *         description: Returns tenant metrics.
+ *         content:
+ *           text/plain:
+ *             schema:
+ *               type: string
+ *             examples:
+ *               metrics:
+ *                 $ref: '#/components/examples/metrics'
+ */
+read.push(new Endpoint("get", "/metrics", controller.fetch))
+
+export default { read }

package/src/api/routes/public/tests/metrics.spec.js

@@ -0,0 +1,34 @@
+const setup = require("../../tests/utilities")
+
+jest.setTimeout(30000)
+
+describe("/metrics", () => {
+  let request = setup.getRequest()
+  let config = setup.getConfig()
+
+  afterAll(setup.afterAll)
+
+  // For some reason this cannot be a beforeAll or the test "should be able to update the user" fail
+  beforeEach(async () => {
+    await config.init()
+  })
+
+  describe("get", () => {
+    it("returns a list of metrics", async () => {
+      const res = await request
+        .get(`/api/public/v1/metrics`)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /text\/plain/)
+        .expect(200)
+      expect(res.text).toContain("budibase_tenant_user_count")
+    })
+
+    it("endpoint should not be publicly exposed", async () => {
+      await request
+      .get(`/api/public/v1/metrics`)
+      .set(config.publicHeaders())
+      .expect(403)
+    })
+  })
+
+})

package/src/constants/index.ts

@@ -180,3 +180,4 @@ export enum AutomationErrors {
 // pass through the list from the auth/core lib
 export const ObjectStoreBuckets = objectStore.ObjectStoreBuckets
 export const MAX_AUTOMATION_RECURRING_ERRORS = 5
+export const GOOGLE_SHEETS_PRIMARY_KEY = "rowNumber"

package/src/definitions/openapi.ts

@@ -22,6 +22,10 @@ export interface paths {
     /** Based on application properties (currently only name) search for applications. */
     post: operations["appSearch"];
   };
+  "/metrics": {
+    /** Output metrics in OpenMetrics format compatible with Prometheus */
+    get: operations["metricsGet"];
+  };
   "/queries/{queryId}": {
     /** Queries which have been created within a Budibase app can be executed using this, */
     post: operations["queryExecute"];
@@ -844,6 +848,17 @@ export interface operations {
       };
     };
   };
+  /** Output metrics in OpenMetrics format compatible with Prometheus */
+  metricsGet: {
+    responses: {
+      /** Returns tenant metrics. */
+      200: {
+        content: {
+          "text/plain": string;
+        };
+      };
+    };
+  };
   /** Queries which have been created within a Budibase app can be executed using this, */
   queryExecute: {
     parameters: {