@budibase/server 2.2.12-alpha.47 → 2.2.12-alpha.49

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.2.12-alpha.47",
+  "version": "2.2.12-alpha.49",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -43,11 +43,11 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.2.12-alpha.47",
-    "@budibase/client": "2.2.12-alpha.47",
-    "@budibase/pro": "2.2.12-alpha.46",
-    "@budibase/string-templates": "2.2.12-alpha.47",
-    "@budibase/types": "2.2.12-alpha.47",
+    "@budibase/backend-core": "2.2.12-alpha.49",
+    "@budibase/client": "2.2.12-alpha.49",
+    "@budibase/pro": "2.2.12-alpha.48",
+    "@budibase/string-templates": "2.2.12-alpha.49",
+    "@budibase/types": "2.2.12-alpha.49",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -169,5 +169,5 @@
   "optionalDependencies": {
     "oracledb": "5.3.0"
   },
-  "gitHead": "a2996efe7a94822856ddfb51151bd1d4ecff311a"
+  "gitHead": "df1d853fe0990446eec4a4e08128257d35cc438f"
 }

package/src/api/controllers/row/external.ts

@@ -21,6 +21,8 @@ import {
 } from "@budibase/types"
 import sdk from "../../../sdk"
 
+const { cleanExportRows } = require("./utils")
+
 export async function handleRequest(
   operation: Operation,
   tableId: string,
@@ -100,7 +102,7 @@ export async function destroy(ctx: BBContext) {
 export async function bulkDestroy(ctx: BBContext) {
   const { rows } = ctx.request.body
   const tableId = ctx.params.tableId
-  let promises = []
+  let promises: Promise<Row[] | { row: Row; table: Table }>[] = []
   for (let row of rows) {
     promises.push(
       handleRequest(Operation.DELETE, tableId, {
@@ -186,20 +188,24 @@ export async function exportRows(ctx: BBContext) {
   if (!datasource || !datasource.entities) {
     ctx.throw(400, "Datasource has not been configured for plus API.")
   }
-  ctx.request.body = {
-    query: {
-      oneOf: {
-        _id: ctx.request.body.rows.map(
-          (row: string) => JSON.parse(decodeURI(row))[0]
-        ),
+
+  if (ctx.request.body.rows) {
+    ctx.request.body = {
+      query: {
+        oneOf: {
+          _id: ctx.request.body.rows.map(
+            (row: string) => JSON.parse(decodeURI(row))[0]
+          ),
+        },
       },
-    },
+    }
   }
 
   let result = await search(ctx)
   let rows: Row[] = []
 
   // Filter data to only specified columns if required
+
   if (columns && columns.length) {
     for (let i = 0; i < result.rows.length; i++) {
       rows[i] = {}
@@ -211,14 +217,19 @@ export async function exportRows(ctx: BBContext) {
     rows = result.rows
   }
 
-  let headers = Object.keys(rows[0])
+  // @ts-ignore
+  let schema = datasource.entities[tableName].schema
+  let exportRows = cleanExportRows(rows, schema, format, columns)
+
+  let headers = Object.keys(schema)
+
   // @ts-ignore
   const exporter = exporters[format]
   const filename = `export.${format}`
 
   // send down the file
   ctx.attachment(filename)
-  return apiFileReturn(exporter(headers, rows))
+  return apiFileReturn(exporter(headers, exportRows))
 }
 
 export async function fetchEnrichedRow(ctx: BBContext) {

package/src/api/controllers/row/internal.ts

@@ -27,7 +27,7 @@ import {
 import { cloneDeep } from "lodash/fp"
 import { context, db as dbCore } from "@budibase/backend-core"
 import { finaliseRow, updateRelatedFormula } from "./staticFormula"
-import { csv, json, jsonWithSchema, Format, isFormat } from "../view/exporters"
+import { csv, json, jsonWithSchema, Format } from "../view/exporters"
 import { apiFileReturn } from "../../../utilities/fileSystem"
 import {
   Ctx,
@@ -38,6 +38,8 @@ import {
   Table,
 } from "@budibase/types"
 
+const { cleanExportRows } = require("./utils")
+
 const CALCULATION_TYPES = {
   SUM: "sum",
   COUNT: "count",
@@ -357,6 +359,14 @@ export async function search(ctx: Ctx) {
   params.version = ctx.version
   params.tableId = tableId
 
+  let table
+  if (params.sort && !params.sortType) {
+    table = await db.get(tableId)
+    const schema = table.schema
+    const sortField = schema[params.sort]
+    params.sortType = sortField.type == "number" ? "number" : "string"
+  }
+
   let response
   if (paginate) {
     response = await paginatedSearch(query, params)
@@ -370,7 +380,7 @@ export async function search(ctx: Ctx) {
     if (tableId === InternalTables.USER_METADATA) {
       response.rows = await getGlobalUsersFromMetadata(response.rows)
     }
-    const table = await db.get(tableId)
+    table = table || (await db.get(tableId))
     response.rows = await outputProcessing(table, response.rows)
   }
 
@@ -389,16 +399,25 @@ export async function exportRows(ctx: Ctx) {
   const table = await db.get(ctx.params.tableId)
   const rowIds = ctx.request.body.rows
   let format = ctx.query.format
-  const { columns } = ctx.request.body
-  let response = (
-    await db.allDocs({
-      include_docs: true,
-      keys: rowIds,
-    })
-  ).rows.map(row => row.doc)
+  const { columns, query } = ctx.request.body
+
+  let result
+  if (rowIds) {
+    let response = (
+      await db.allDocs({
+        include_docs: true,
+        keys: rowIds,
+      })
+    ).rows.map(row => row.doc)
+
+    result = await outputProcessing(table, response)
+  } else if (query) {
+    let searchResponse = await exports.search(ctx)
+    result = searchResponse.rows
+  }
 
-  let result = (await outputProcessing(table, response)) as Row[]
   let rows: Row[] = []
+  let schema = table.schema
 
   // Filter data to only specified columns if required
   if (columns && columns.length) {
@@ -412,12 +431,16 @@ export async function exportRows(ctx: Ctx) {
     rows = result
   }
 
+  let exportRows = cleanExportRows(rows, schema, format, columns)
   if (format === Format.CSV) {
     ctx.attachment("export.csv")
-    return apiFileReturn(csv(Object.keys(rows[0]), rows))
+    return apiFileReturn(csv(Object.keys(rows[0]), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment("export.json")
-    return apiFileReturn(json(rows))
+    return apiFileReturn(json(exportRows))
+  } else if (format === Format.JSON_WITH_SCHEMA) {
+    ctx.attachment("export.json")
+    return apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

package/src/api/controllers/row/utils.ts

@@ -7,6 +7,7 @@ import { BBContext, Row, Table } from "@budibase/types"
 export { removeKeyNumbering } from "../../../integrations/base/utils"
 const validateJs = require("validate.js")
 const { cloneDeep } = require("lodash/fp")
+import { Format } from "../view/exporters"
 import { Ctx } from "@budibase/types"
 import sdk from "../../../sdk"
 
@@ -117,3 +118,40 @@ export async function validate({
   }
   return { valid: Object.keys(errors).length === 0, errors }
 }
+
+export function cleanExportRows(
+  rows: any[],
+  schema: any,
+  format: string,
+  columns: string[]
+) {
+  let cleanRows = [...rows]
+
+  const relationships = Object.entries(schema)
+    .filter((entry: any[]) => entry[1].type === FieldTypes.LINK)
+    .map(entry => entry[0])
+
+  relationships.forEach(column => {
+    cleanRows.forEach(row => {
+      delete row[column]
+    })
+    delete schema[column]
+  })
+
+  // Intended to avoid 'undefined' in export
+  if (format === Format.CSV) {
+    const schemaKeys = Object.keys(schema)
+    for (let key of schemaKeys) {
+      if (columns?.length && columns.indexOf(key) > 0) {
+        continue
+      }
+      for (let row of cleanRows) {
+        if (row[key] == null) {
+          row[key] = ""
+        }
+      }
+    }
+  }
+
+  return cleanRows
+}

package/src/api/controllers/view/index.ts

@@ -3,7 +3,6 @@ import { apiFileReturn } from "../../../utilities/fileSystem"
 import { csv, json, jsonWithSchema, Format, isFormat } from "./exporters"
 import { deleteView, getView, getViews, saveView } from "./utils"
 import { fetchView } from "../row"
-import { FieldTypes } from "../../../constants"
 import { context, events } from "@budibase/backend-core"
 import { DocumentType } from "../../../db/utils"
 import sdk from "../../../sdk"
@@ -15,6 +14,7 @@ import {
   TableSchema,
   View,
 } from "@budibase/types"
+import { cleanExportRows } from "../row/utils"
 
 const { cloneDeep, isEqual } = require("lodash")
 
@@ -162,39 +162,17 @@ export async function exportView(ctx: BBContext) {
     schema = table.schema
   }
 
-  // remove any relationships
-  const relationships = Object.entries(schema)
-    .filter(entry => entry[1].type === FieldTypes.LINK)
-    .map(entry => entry[0])
-  // iterate relationship columns and remove from and row and schema
-  relationships.forEach(column => {
-    rows.forEach(row => {
-      delete row[column]
-    })
-    delete schema[column]
-  })
-
-  // make sure no "undefined" entries appear in the CSV
-  if (format === Format.CSV) {
-    const schemaKeys = Object.keys(schema)
-    for (let key of schemaKeys) {
-      for (let row of rows) {
-        if (row[key] == null) {
-          row[key] = ""
-        }
-      }
-    }
-  }
+  let exportRows = cleanExportRows(rows, schema, format, [])
 
   if (format === Format.CSV) {
     ctx.attachment(`${viewName}.csv`)
-    ctx.body = apiFileReturn(csv(Object.keys(schema), rows))
+    ctx.body = apiFileReturn(csv(Object.keys(schema), exportRows))
   } else if (format === Format.JSON) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(json(rows))
+    ctx.body = apiFileReturn(json(exportRows))
   } else if (format === Format.JSON_WITH_SCHEMA) {
     ctx.attachment(`${viewName}.json`)
-    ctx.body = apiFileReturn(jsonWithSchema(schema, rows))
+    ctx.body = apiFileReturn(jsonWithSchema(schema, exportRows))
   } else {
     throw "Format not recognised"
   }

package/src/definitions/datasource.ts

@@ -7,44 +7,3 @@
 export interface QueryOptions {
   disableReturning?: boolean
 }
-
-export enum AuthType {
-  BASIC = "basic",
-  BEARER = "bearer",
-}
-
-interface AuthConfig {
-  _id: string
-  name: string
-  type: AuthType
-  config: BasicAuthConfig | BearerAuthConfig
-}
-
-export interface BasicAuthConfig {
-  username: string
-  password: string
-}
-
-export interface BearerAuthConfig {
-  token: string
-}
-
-export interface RestConfig {
-  url: string
-  rejectUnauthorized: boolean
-  defaultHeaders: {
-    [key: string]: any
-  }
-  legacyHttpParser: boolean
-  authConfigs: AuthConfig[]
-  staticVariables: {
-    [key: string]: string
-  }
-  dynamicVariables: [
-    {
-      name: string
-      queryId: string
-      value: string
-    }
-  ]
-}

package/src/integrations/rest.ts

@@ -6,13 +6,11 @@ import {
   IntegrationBase,
   PaginationValues,
   RestQueryFields as RestQuery,
-} from "@budibase/types"
-import {
   RestConfig,
-  AuthType,
-  BasicAuthConfig,
-  BearerAuthConfig,
-} from "../definitions/datasource"
+  RestAuthType,
+  RestBasicAuthConfig,
+  RestBearerAuthConfig,
+} from "@budibase/types"
 import { get } from "lodash"
 import * as https from "https"
 import qs from "querystring"
@@ -331,14 +329,14 @@ class RestIntegration implements IntegrationBase {
       if (authConfig) {
         let config
         switch (authConfig.type) {
-          case AuthType.BASIC:
-            config = authConfig.config as BasicAuthConfig
+          case RestAuthType.BASIC:
+            config = authConfig.config as RestBasicAuthConfig
             headers.Authorization = `Basic ${Buffer.from(
               `${config.username}:${config.password}`
             ).toString("base64")}`
             break
-          case AuthType.BEARER:
-            config = authConfig.config as BearerAuthConfig
+          case RestAuthType.BEARER:
+            config = authConfig.config as RestBearerAuthConfig
             headers.Authorization = `Bearer ${config.token}`
             break
         }
@@ -428,5 +426,4 @@ class RestIntegration implements IntegrationBase {
 export default {
   schema: SCHEMA,
   integration: RestIntegration,
-  AuthType,
 }

package/src/integrations/tests/rest.spec.ts

@@ -15,6 +15,7 @@ jest.mock("node-fetch", () => {
 
 import fetch from "node-fetch"
 import { default as RestIntegration } from "../rest"
+import { RestAuthType } from "@budibase/types"
 const FormData = require("form-data")
 const { URLSearchParams } = require("url")
 
@@ -229,7 +230,7 @@ describe("REST Integration", () => {
     const basicAuth = {
       _id: "c59c14bd1898a43baa08da68959b24686",
       name: "basic-1",
-      type: RestIntegration.AuthType.BASIC,
+      type: RestAuthType.BASIC,
       config: {
         username: "user",
         password: "password",
@@ -239,7 +240,7 @@ describe("REST Integration", () => {
     const bearerAuth = {
       _id: "0d91d732f34e4befabeff50b392a8ff3",
       name: "bearer-1",
-      type: RestIntegration.AuthType.BEARER,
+      type: RestAuthType.BEARER,
       config: {
         token: "mytoken",
       },
@@ -581,6 +582,7 @@ describe("REST Integration", () => {
     })
     await config.integration.read({})
 
+    // @ts-ignore
     const calls: any = fetch.mock.calls[0]
     const url = calls[0]
     expect(url).toBe(`${BASE_URL}/`)

package/src/sdk/app/datasources/datasources.ts

@@ -1,17 +1,19 @@
 import { context } from "@budibase/backend-core"
-import { processObjectSync, findHBSBlocks } from "@budibase/string-templates"
+import { findHBSBlocks, processObjectSync } from "@budibase/string-templates"
 import {
   Datasource,
   DatasourceFieldType,
-  Integration,
   PASSWORD_REPLACEMENT,
+  RestAuthConfig,
+  RestAuthType,
+  RestBasicAuthConfig,
+  SourceName,
 } from "@budibase/types"
 import { cloneDeep } from "lodash/fp"
 import { getEnvironmentVariables } from "../../utils"
 import { getDefinitions } from "../../../integrations"
 
 const ENV_VAR_PREFIX = "env."
-const USER_PREFIX = "user"
 
 async function enrichDatasourceWithValues(datasource: Datasource) {
   const cloned = cloneDeep(datasource)
@@ -47,6 +49,18 @@ export async function getWithEnvVars(datasourceId: string) {
   return enrichDatasourceWithValues(datasource)
 }
 
+function hasAuthConfigs(datasource: Datasource) {
+  return datasource.source === SourceName.REST && datasource.config?.authConfigs
+}
+
+function useEnvVars(str: any) {
+  if (typeof str !== "string") {
+    return false
+  }
+  const blocks = findHBSBlocks(str)
+  return blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+}
+
 export async function removeSecrets(datasources: Datasource[]) {
   const definitions = await getDefinitions()
   for (let datasource of datasources) {
@@ -56,17 +70,24 @@ export async function removeSecrets(datasources: Datasource[]) {
       if (datasource.config.auth) {
         delete datasource.config.auth
       }
-      // remove passwords
-      for (let key of Object.keys(datasource.config)) {
-        if (typeof datasource.config[key] !== "string") {
-          continue
+      // specific to REST datasources, contains passwords
+      if (hasAuthConfigs(datasource)) {
+        const configs = datasource.config.authConfigs as RestAuthConfig[]
+        for (let config of configs) {
+          if (config.type !== RestAuthType.BASIC) {
+            continue
+          }
+          const basic = config.config as RestBasicAuthConfig
+          if (!useEnvVars(basic.password)) {
+            basic.password = PASSWORD_REPLACEMENT
+          }
         }
-        const blocks = findHBSBlocks(datasource.config[key] as string)
-        const usesEnvVars =
-          blocks.find(block => block.includes(ENV_VAR_PREFIX)) != null
+      }
+      // remove general passwords
+      for (let key of Object.keys(datasource.config)) {
         if (
-          !usesEnvVars &&
-          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD
+          schema.datasource?.[key]?.type === DatasourceFieldType.PASSWORD &&
+          !useEnvVars(datasource.config[key])
         ) {
           datasource.config[key] = PASSWORD_REPLACEMENT
         }
@@ -84,6 +105,23 @@ export function mergeConfigs(update: Datasource, old: Datasource) {
   if (!update.config) {
     return update
   }
+  // specific to REST datasources, fix the auth configs again if required
+  if (hasAuthConfigs(update)) {
+    const configs = update.config.authConfigs as RestAuthConfig[]
+    const oldConfigs = old.config?.authConfigs as RestAuthConfig[]
+    for (let config of configs) {
+      if (config.type !== RestAuthType.BASIC) {
+        continue
+      }
+      const basic = config.config as RestBasicAuthConfig
+      const oldBasic = oldConfigs.find(old => old.name === config.name)
+        ?.config as RestBasicAuthConfig
+      if (basic.password === PASSWORD_REPLACEMENT) {
+        basic.password = oldBasic.password
+      }
+    }
+  }
+  // update back to actual passwords for everything else
   for (let [key, value] of Object.entries(update.config)) {
     if (value !== PASSWORD_REPLACEMENT) {
       continue