@budibase/server 2.2.12-alpha.6 → 2.2.12-alpha.60

package/src/api/controllers/datasource.ts

@@ -12,9 +12,11 @@ import { getIntegration } from "../../integrations"
 import { getDatasourceAndQuery } from "./row/utils"
 import { invalidateDynamicVariables } from "../../threads/utils"
 import { db as dbCore, context, events } from "@budibase/backend-core"
-import { BBContext, Datasource, Row } from "@budibase/types"
+import { UserCtx, Datasource, Row } from "@budibase/types"
+import sdk from "../../sdk"
+import { mergeConfigs } from "../../sdk/app/datasources/datasources"
 
-export async function fetch(ctx: BBContext) {
+export async function fetch(ctx: UserCtx) {
   // Get internal tables
   const db = context.getAppDB()
   const internalTables = await db.allDocs(
@@ -43,25 +45,23 @@ export async function fetch(ctx: BBContext) {
     )
   ).rows.map(row => row.doc)
 
-  const allDatasources = [bbInternalDb, ...datasources]
+  const allDatasources: Datasource[] = await sdk.datasources.removeSecrets([
+    bbInternalDb,
+    ...datasources,
+  ])
 
   for (let datasource of allDatasources) {
-    if (datasource.config && datasource.config.auth) {
-      // strip secrets from response so they don't show in the network request
-      delete datasource.config.auth
-    }
-
     if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
-      datasource.entities = internal[datasource._id]
+      datasource.entities = internal[datasource._id!]
     }
   }
 
   ctx.body = [bbInternalDb, ...datasources]
 }
 
-export async function buildSchemaFromDb(ctx: BBContext) {
+export async function buildSchemaFromDb(ctx: UserCtx) {
   const db = context.getAppDB()
-  const datasource = await db.get(ctx.params.datasourceId)
+  const datasource = await sdk.datasources.get(ctx.params.datasourceId)
   const tablesFilter = ctx.request.body.tablesFilter
 
   let { tables, error } = await buildSchemaHelper(datasource)
@@ -146,11 +146,11 @@ async function invalidateVariables(
   await invalidateDynamicVariables(toInvalidate)
 }
 
-export async function update(ctx: BBContext) {
+export async function update(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
-  let datasource = await db.get(datasourceId)
-  const auth = datasource.config.auth
+  let datasource = await sdk.datasources.get(datasourceId)
+  const auth = datasource.config?.auth
   await invalidateVariables(datasource, ctx.request.body)
 
   const isBudibaseSource = datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE
@@ -159,10 +159,13 @@ export async function update(ctx: BBContext) {
     ? { name: ctx.request.body?.name }
     : ctx.request.body
 
-  datasource = { ...datasource, ...dataSourceBody }
+  datasource = {
+    ...datasource,
+    ...sdk.datasources.mergeConfigs(dataSourceBody, datasource),
+  }
   if (auth && !ctx.request.body.auth) {
     // don't strip auth config from DB
-    datasource.config.auth = auth
+    datasource.config!.auth = auth
   }
 
   const response = await db.put(datasource)
@@ -179,10 +182,12 @@ export async function update(ctx: BBContext) {
 
   ctx.status = 200
   ctx.message = "Datasource saved successfully."
-  ctx.body = { datasource }
+  ctx.body = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
 }
 
-export async function save(ctx: BBContext) {
+export async function save(ctx: UserCtx) {
   const db = context.getAppDB()
   const plus = ctx.request.body.datasource.plus
   const fetchSchema = ctx.request.body.fetchSchema
@@ -213,7 +218,9 @@ export async function save(ctx: BBContext) {
     }
   }
 
-  const response: any = { datasource }
+  const response: any = {
+    datasource: await sdk.datasources.removeSecretSingle(datasource),
+  }
   if (schemaError) {
     response.error = schemaError
   }
@@ -251,11 +258,11 @@ async function destroyInternalTablesBySourceId(datasourceId: string) {
   }
 }
 
-export async function destroy(ctx: BBContext) {
+export async function destroy(ctx: UserCtx) {
   const db = context.getAppDB()
   const datasourceId = ctx.params.datasourceId
 
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   // Delete all queries for the datasource
 
   if (datasource.type === dbCore.BUDIBASE_DATASOURCE_TYPE) {
@@ -279,13 +286,14 @@ export async function destroy(ctx: BBContext) {
   ctx.status = 200
 }
 
-export async function find(ctx: BBContext) {
+export async function find(ctx: UserCtx) {
   const database = context.getAppDB()
-  ctx.body = await database.get(ctx.params.datasourceId)
+  const datasource = await database.get(ctx.params.datasourceId)
+  ctx.body = await sdk.datasources.removeSecretSingle(datasource)
 }
 
 // dynamic query functionality
-export async function query(ctx: BBContext) {
+export async function query(ctx: UserCtx) {
   const queryJson = ctx.request.body
   try {
     ctx.body = await getDatasourceAndQuery(queryJson)
@@ -313,7 +321,7 @@ function updateError(error: any, newError: any, tables: string[]) {
 
 async function buildSchemaHelper(datasource: Datasource) {
   const Connector = await getIntegration(datasource.source)
-
+  datasource = await sdk.datasources.enrich(datasource)
   // Connect to the DB and build the schema
   const connector = new Connector(datasource.config)
   await connector.buildSchema(datasource._id, datasource.entities)

package/src/api/controllers/query/index.ts

@@ -7,6 +7,8 @@ import { invalidateDynamicVariables } from "../../../threads/utils"
 import env from "../../../environment"
 import { quotas } from "@budibase/pro"
 import { events, context, utils, constants } from "@budibase/backend-core"
+import sdk from "../../../sdk"
+import { QueryEvent } from "../../../threads/definitions"
 
 const Runner = new Thread(ThreadType.QUERY, {
   timeoutMs: env.QUERY_THREAD_TIMEOUT || 10000,
@@ -81,7 +83,7 @@ export async function save(ctx: any) {
   const db = context.getAppDB()
   const query = ctx.request.body
 
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
 
   let eventFn
   if (!query._id) {
@@ -126,9 +128,9 @@ function getAuthConfig(ctx: any) {
 }
 
 export async function preview(ctx: any) {
-  const db = context.getAppDB()
-
-  const datasource = await db.get(ctx.request.body.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    ctx.request.body.datasourceId
+  )
   const query = ctx.request.body
   // preview may not have a queryId as it hasn't been saved, but if it does
   // this stops dynamic variables from calling the same query
@@ -137,20 +139,22 @@ export async function preview(ctx: any) {
   const authConfigCtx: any = getAuthConfig(ctx)
 
   try {
-    const runFn = () =>
-      Runner.run({
-        appId: ctx.appId,
-        datasource,
-        queryVerb,
-        fields,
-        parameters,
-        transformer,
-        queryId,
-        ctx: {
-          user: ctx.user,
-          auth: { ...authConfigCtx },
-        },
-      })
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb,
+      fields,
+      parameters,
+      transformer,
+      queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, keys, info, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -201,7 +205,9 @@ async function execute(
   const db = context.getAppDB()
 
   const query = await db.get(ctx.params.queryId)
-  const datasource = await db.get(query.datasourceId)
+  const { datasource, envVars } = await sdk.datasources.getWithEnvVars(
+    query.datasourceId
+  )
 
   let authConfigCtx: any = {}
   if (!opts.isAutomation) {
@@ -219,21 +225,23 @@ async function execute(
 
   // call the relevant CRUD method on the integration class
   try {
-    const runFn = () =>
-      Runner.run({
-        appId: ctx.appId,
-        datasource,
-        queryVerb: query.queryVerb,
-        fields: query.fields,
-        pagination: ctx.request.body.pagination,
-        parameters: enrichedParameters,
-        transformer: query.transformer,
-        queryId: ctx.params.queryId,
-        ctx: {
-          user: ctx.user,
-          auth: { ...authConfigCtx },
-        },
-      })
+    const inputs: QueryEvent = {
+      appId: ctx.appId,
+      datasource,
+      queryVerb: query.queryVerb,
+      fields: query.fields,
+      pagination: ctx.request.body.pagination,
+      parameters: enrichedParameters,
+      transformer: query.transformer,
+      queryId: ctx.params.queryId,
+      // have to pass down to the thread runner - can't put into context now
+      environmentVariables: envVars,
+      ctx: {
+        user: ctx.user,
+        auth: { ...authConfigCtx },
+      },
+    }
+    const runFn = () => Runner.run(inputs)
 
     const { rows, pagination, extra } = await quotas.addQuery(runFn, {
       datasourceId: datasource._id,
@@ -266,18 +274,18 @@ export async function executeV2(
 const removeDynamicVariables = async (queryId: any) => {
   const db = context.getAppDB()
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
-  const dynamicVariables = datasource.config.dynamicVariables
+  const datasource = await sdk.datasources.get(query.datasourceId)
+  const dynamicVariables = datasource.config?.dynamicVariables as any[]
 
   if (dynamicVariables) {
     // delete dynamic variables from the datasource
-    datasource.config.dynamicVariables = dynamicVariables.filter(
+    datasource.config!.dynamicVariables = dynamicVariables!.filter(
       (dv: any) => dv.queryId !== queryId
     )
     await db.put(datasource)
 
     // invalidate the deleted variables
-    const variablesToDelete = dynamicVariables.filter(
+    const variablesToDelete = dynamicVariables!.filter(
       (dv: any) => dv.queryId === queryId
     )
     await invalidateDynamicVariables(variablesToDelete)
@@ -289,7 +297,7 @@ export async function destroy(ctx: any) {
   const queryId = ctx.params.queryId
   await removeDynamicVariables(queryId)
   const query = await db.get(queryId)
-  const datasource = await db.get(query.datasourceId)
+  const datasource = await sdk.datasources.get(query.datasourceId)
   await db.remove(ctx.params.queryId, ctx.params.revId)
   ctx.message = `Query deleted.`
   ctx.status = 200

package/src/api/controllers/row/ExternalRequest.ts

@@ -25,6 +25,7 @@ import { cloneDeep } from "lodash/fp"
 import { processFormulas, processDates } from "../../../utilities/rowProcessor"
 import { context } from "@budibase/backend-core"
 import { removeKeyNumbering } from "./utils"
+import sdk from "../../../sdk"
 
 export interface ManyRelationship {
   tableId?: string
@@ -664,8 +665,7 @@ export class ExternalRequest {
       throw "Unable to run without a table name"
     }
     if (!this.datasource) {
-      const db = context.getAppDB()
-      this.datasource = await db.get(datasourceId)
+      this.datasource = await sdk.datasources.get(datasourceId!)
       if (!this.datasource || !this.datasource.entities) {
         throw "No tables found, fetch tables before query."
       }
@@ -681,6 +681,12 @@ export class ExternalRequest {
       config,
       table
     )
+    //if the sort column is a formula, remove it
+    for (let sortColumn of Object.keys(sort || {})) {
+      if (table.schema[sortColumn]?.type === "formula") {
+        delete sort?.[sortColumn]
+      }
+    }
     filters = buildFilters(id, filters || {}, table)
     const relationships = this.buildRelationships(table)
     // clean up row on ingress using schema

package/src/api/controllers/row/external.ts

@@ -19,6 +19,9 @@ import {
   Table,
   Datasource,
 } from "@budibase/types"
+import sdk from "../../../sdk"
+
+const { cleanExportRows } = require("./utils")
 
 export async function handleRequest(
   operation: Operation,
@@ -99,7 +102,7 @@ export async function destroy(ctx: BBContext) {
 export async function bulkDestroy(ctx: BBContext) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
-  let promises = []
+  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -179,27 +182,30 @@ export async function validate(ctx: BBContext) {
 
 export async function exportRows(ctx: BBContext) {
   const { datasourceId } = breakExternalTableId(ctx.params.tableId)
-  const db = context.getAppDB()
   const format = ctx.query.format
   const { columns } = ctx.request.body
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId!)
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
-  ctx.request.body = {
-    query: {
-      oneOf: {
-        _id: ctx.request.body.rows.map(
-          (row: string) => JSON.parse(decodeURI(row))[0]
-        ),
+
+  if (ctx.request.body.rows) {
+    ctx.request.body = {
+      query: {
+        oneOf: {
+          _id: ctx.request.body.rows.map(
+            (row: string) => JSON.parse(decodeURI(row))[0]
+          ),
+        },
       },
-    },
+    }
   }
 
   let result = await search(ctx)
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
+
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {
       rows[i] = {}
@@ -211,22 +217,26 @@ export async function exportRows(ctx: BBContext) {
     rows = result.rows
   }
 
-  let headers = Object.keys(rows[0])
+  // @ts-ignore
+  let schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
   // @ts-ignore
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  return apiFileReturn(exporter(headers, exportRows))
 }
 
 export async function fetchEnrichedRow(ctx: BBContext) {
   const id = ctx.params.rowId
   const tableId = ctx.params.tableId
   const { datasourceId, tableName } = breakExternalTableId(tableId)
-  const db = context.getAppDB()
-  const datasource: Datasource = await db.get(datasourceId)
+  const datasource: Datasource = await sdk.datasources.get(datasourceId!)
   if (!tableName) {
     ctx.throw(400, "Unable to find table.")
   }

package/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import * as exporters from "../view/exporters"
+import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -38,6 +38,8 @@ import {
   Table,
 } from "@budibase/types"
 
+const { cleanExportRows } = require("./utils")
+
 const CALCULATION_TYPES = {
   SUM: "sum",
   COUNT: "count",
@@ -357,6 +359,14 @@ export async function search(ctx: Ctx) {
   params.version = ctx.version
   params.tableId = tableId
 
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type == "number" ? "number" : "string"
+  }
+
   let response
   if (paginate) {
     response = await paginatedSearch(query, params)
@@ -370,7 +380,7 @@ export async function search(ctx: Ctx) {
     if (tableId === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
-    const table = await db.get(tableId)
+    table = table || (await db.get(tableId))
     response.rows = await outputProcessing(table, response.rows)
   }
 
@@ -389,16 +399,25 @@ export async function exportRows(ctx: Ctx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
-  const { columns } = ctx.request.body
-  let response = (
-    await db.allDocs({
-      include_docs: true,
-      keys: rowIds,
-    })
-  ).rows.map(row => row.doc)
+  const { columns, query } = ctx.request.body
+
+  let result
+  if (rowIds) {
+    let response = (
+      await db.allDocs({
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await exports.search(ctx)
+    result = searchResponse.rows
+  }
 
-  let result = (await outputProcessing(table, response)) as Row[]
   let rows: Row[] = []
+  let schema = table.schema
 
   // Filter data to only specified columns if required
   if (columns && columns.length) {
@@ -412,14 +431,19 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
-  let headers = Object.keys(rows[0])
-  // @ts-ignore
-  const exporter = exporters[format]
-  const filename = `export.${format}`
-
-  // send down the file
-  ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+  if (format === Format.CSV) {
+    ctx.attachment("export.csv")
+    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
+  } else if (format === Format.JSON) {
+    ctx.attachment("export.json")
+    return apiFileReturn(json(exportRows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment("export.json")
+    return apiFileReturn(jsonWithSchema(schema, exportRows))
+  } else {
+    throw "Format not recognised"
+  }
 }
 
 export async function fetchEnrichedRow(ctx: Ctx) {

package/src/api/controllers/row/utils.ts

@@ -7,7 +7,9 @@ import { BBContext, Row, Table } from "@budibase/types"
 export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
+import sdk from "../../../sdk"
 
 validateJs.extend(validateJs.validators.datetime, {
   parse: function (value: string) {
@@ -21,8 +23,7 @@ validateJs.extend(validateJs.validators.datetime, {
 
 export async function getDatasourceAndQuery(json: any) {
   const datasourceId = json.endpoint.datasourceId
-  const db = context.getAppDB()
-  const datasource = await db.get(datasourceId)
+  const datasource = await sdk.datasources.get(datasourceId)
   return makeExternalQuery(datasource, json)
 }
 
@@ -117,3 +118,40 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+export function cleanExportRows(
+  rows: any[],
+  schema: any,
+  format: string,
+  columns: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  // Intended to avoid 'undefined' in export
+  if (format === Format.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

package/src/api/controllers/static/index.ts

@@ -1,20 +1,21 @@
 require("svelte/register")
 
-const send = require("koa-send")
-const { resolve, join } = require("../../../utilities/centralPath")
+import { resolve, join } from "../../../utilities/centralPath"
 const uuid = require("uuid")
 import { ObjectStoreBuckets } from "../../../constants"
-const { processString } = require("@budibase/string-templates")
-const {
+import { processString } from "@budibase/string-templates"
+import {
   loadHandlebarsFile,
   NODE_MODULES_PATH,
   TOP_LEVEL_PATH,
-} = require("../../../utilities/fileSystem")
+} from "../../../utilities/fileSystem"
 import env from "../../../environment"
-const { DocumentType } = require("../../../db/utils")
-const { context, objectStore, utils } = require("@budibase/backend-core")
-const AWS = require("aws-sdk")
-const fs = require("fs")
+import { DocumentType } from "../../../db/utils"
+import { context, objectStore, utils } from "@budibase/backend-core"
+import AWS from "aws-sdk"
+import fs from "fs"
+import sdk from "../../../sdk"
+const send = require("koa-send")
 
 async function prepareUpload({ s3Key, bucket, metadata, file }: any) {
   const response = await objectStore.upload({
@@ -110,7 +111,7 @@ export const serveApp = async function (ctx: any) {
       title: appInfo.name,
       production: env.isProd(),
       appId,
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
       usedPlugins: plugins,
     })
 
@@ -135,7 +136,7 @@ export const serveBuilderPreview = async function (ctx: any) {
     let appId = context.getAppId()
     const previewHbs = loadHandlebarsFile(`${__dirname}/templates/preview.hbs`)
     ctx.body = await processString(previewHbs, {
-      clientLibPath: objectStore.clientLibraryUrl(appId, appInfo.version),
+      clientLibPath: objectStore.clientLibraryUrl(appId!, appInfo.version),
     })
   } else {
     // just return the app info for jest to assert on
@@ -150,13 +151,11 @@ export const serveClientLibrary = async function (ctx: any) {
 }
 
 export const getSignedUploadURL = async function (ctx: any) {
-  const database = context.getAppDB()
-
   // Ensure datasource is valid
   let datasource
   try {
     const { datasourceId } = ctx.params
-    datasource = await database.get(datasourceId)
+    datasource = await sdk.datasources.get(datasourceId, { enriched: true })
     if (!datasource) {
       ctx.throw(400, "The specified datasource could not be found")
     }
@@ -172,8 +171,8 @@ export const getSignedUploadURL = async function (ctx: any) {
   // Determine type of datasource and generate signed URL
   let signedUrl
   let publicUrl
-  const awsRegion = datasource?.config?.region || "eu-west-1"
-  if (datasource.source === "S3") {
+  const awsRegion = (datasource?.config?.region || "eu-west-1") as string
+  if (datasource?.source === "S3") {
     const { bucket, key } = ctx.request.body || {}
     if (!bucket || !key) {
       ctx.throw(400, "bucket and key values are required")
@@ -182,8 +181,8 @@ export const getSignedUploadURL = async function (ctx: any) {
     try {
       const s3 = new AWS.S3({
         region: awsRegion,
-        accessKeyId: datasource?.config?.accessKeyId,
-        secretAccessKey: datasource?.config?.secretAccessKey,
+        accessKeyId: datasource?.config?.accessKeyId as string,
+        secretAccessKey: datasource?.config?.secretAccessKey as string,
         apiVersion: "2006-03-01",
         signatureVersion: "v4",
       })

package/src/api/controllers/static/templates/BudibaseApp.svelte

@@ -2,7 +2,6 @@
   export let title = ""
   export let favicon = ""
   export let metaImage = ""
-  export let url = ""
 
   export let clientLibPath
   export let usedPlugins