@budibase/server 2.2.12-alpha.7 → 2.2.12-alpha.71

package/src/api/routes/table.ts

@@ -67,10 +67,7 @@ router
    * structure, and the "updated", new column name should also be supplied. The schema should also be updated, this field
    * lets the server know that a field hasn't just been deleted, that the data has moved to a new name, this will fix
    * the rows in the table. This functionality is only available for internal tables.
-   * @apiParam (Body) {object} [dataImport] When creating an internal table it can be built from a CSV, by using the
-   * CSV validation endpoint. Send the CSV data to the validation endpoint, then put the results of that call
-   * into this property, along with the CSV and a table/rows will be built from it. This is not supported when updating
-   * or for external tables.
+   * @apiParam (Body) {object[]} [rows] When creating a table using a compatible data source, an array of objects to be imported into the new table can be provided.
    *
    * @apiParamExample {json} Example:
    * {
@@ -99,15 +96,7 @@ router
    *     "old": "columnName",
    *     "updated": "newColumnName",
    *   },
-   *   "dataImport": {
-   *     "csvString": "column\nvalue",
-   *     "primaryDisplay": "column",
-   *     "schema": {
-   *       "column": {
-   *         "type": "string"
-   *       }
-   *     }
-   *   }
+   *   "rows": []
    * }
    *
    * @apiSuccess {object} table The response body will contain the table structure after being cleaned up and
@@ -121,30 +110,20 @@ router
     tableValidator(),
     tableController.save
   )
-  /**
-   * @api {post} /api/tables/csv/validate Validate a CSV for a table
-   * @apiName Validate a CSV for a table
-   * @apiGroup tables
-   * @apiPermission builder
-   * @apiDescription When creating a new table, or importing a CSV to an existing table the CSV must be validated and
-   * converted into a Budibase schema; this endpoint does this.
-   *
-   * @apiParam (Body) {string} csvString The CSV which is to be validated as a string.
-   * @apiParam (Body) {object} [schema] When a CSV has been validated it is possible to re-validate after changing the
-   * type of a field, by default everything will be strings as there is no way to infer types. The returned schema can
-   * be updated and then returned to the endpoint to re-validate and check if the type will work for the CSV, e.g.
-   * using a number instead of strings.
-   * @apiParam (Body) {string} [tableId] If importing data to an existing table this will pull the current table and
-   * remove any fields from the CSV schema which do not exist on the table/don't match the type of the table. When
-   * importing a CSV to an existing table only fields that are present on the table can be imported.
-   *
-   * @apiSuccess {object} schema The response body will contain a "schema" object that represents the schema found for
-   * the CSV - this will be in the same format used for table schema.s
-   */
   .post(
-    "/api/tables/csv/validate",
+    "/api/convert/csvToJson",
+    authorized(BUILDER),
+    tableController.csvToJson
+  )
+  .post(
+    "/api/tables/validateNewTableImport",
+    authorized(BUILDER),
+    tableController.validateNewTableImport
+  )
+  .post(
+    "/api/tables/validateExistingTableImport",
     authorized(BUILDER),
-    tableController.validateCSVSchema
+    tableController.validateExistingTableImport
   )
   /**
    * @api {post} /api/tables/:tableId/:revId Delete a table
@@ -177,9 +156,7 @@ router
    *
    * @apiParam {string} tableId The ID of the table which the data should be imported to.
    *
-   * @apiParam (Body) {object} dataImport This is the same as the structure used when creating an internal table with
-   * a CSV, it will have the "schema" returned from the CSV validation endpoint and the "csvString" which is to be
-   * turned into rows.
+   * @apiParam (Body) {object[]} rows An array of objects representing the rows to be imported, key-value pairs not matching the table schema will be ignored.
    *
    * @apiSuccess {string} message A message stating that the data was imported successfully.
    */

package/src/api/routes/tests/datasource.spec.ts

@@ -2,7 +2,8 @@ jest.mock("pg")
 import * as setup from "./utilities"
 import { checkBuilderEndpoint } from "./utilities/TestFunctions"
 import { checkCacheForDynamicVariable } from "../../../threads/utils"
-import { events } from "@budibase/backend-core"
+import { context, events } from "@budibase/backend-core"
+import sdk from "../../../sdk"
 
 let { basicDatasource } = setup.structures
 const pg = require("pg")
@@ -184,4 +185,37 @@ describe("/datasources", () => {
       })
     })
   })
+
+  describe("check secret replacement", () => {
+    async function makeDatasource() {
+      datasource = basicDatasource()
+      datasource.datasource.config.password = "testing"
+      const res = await request
+        .post(`/api/datasources`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      return res.body.datasource
+    }
+
+    it("should save a datasource with password", async () => {
+      const datasource = await makeDatasource()
+      expect(datasource.config.password).toBe("--secret-value--")
+    })
+
+    it("should not the password on update with the --secret-value--", async () => {
+      const datasource = await makeDatasource()
+      await request
+        .put(`/api/datasources/${datasource._id}`)
+        .send(datasource)
+        .set(config.defaultHeaders())
+        .expect("Content-Type", /json/)
+        .expect(200)
+      await context.doInAppContext(config.getAppId(), async () => {
+        const dbDatasource: any = await sdk.datasources.get(datasource._id)
+        expect(dbDatasource.config.password).toBe("testing")
+      })
+    })
+  })
 })

package/src/api/routes/tests/misc.spec.js

@@ -42,7 +42,7 @@ describe("run misc tests", () => {
   })
 
   describe("test table utilities", () => {
-    it("should be able to import a CSV", async () => {
+    it("should be able to import data", async () => {
       return config.doInContext(null, async () => {
         const table = await config.createTable({
           name: "table",
@@ -75,17 +75,11 @@ describe("run misc tests", () => {
             },
           },
         })
-        const dataImport = {
-          csvString: "a,b,c,d\n1,2,3,4",
-          schema: {},
-        }
-        for (let col of ["a", "b", "c", "d"]) {
-          dataImport.schema[col] = { type: "string" }
-        }
+
         await tableUtils.handleDataImport(
           { userId: "test" },
           table,
-          dataImport
+          [{ a: '1', b: '2', c: '3', d: '4'}]
         )
         const rows = await config.getRows()
         expect(rows[0].a).toEqual("1")
@@ -94,4 +88,4 @@ describe("run misc tests", () => {
       })
     })
   })
-})
+})

package/src/api/routes/tests/table.spec.js

@@ -43,21 +43,18 @@ describe("/tables", () => {
       expect(events.table.created).toBeCalledWith(res.body)
     })
 
-    it("creates a table via data import CSV", async () => {
+    it("creates a table via data import", async () => {
       const table = basicTable()
-      table.dataImport = {
-        csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-      }
-      table.dataImport.schema = table.schema
+      table.rows = [{ name: 'test-name', description: 'test-desc' }]
 
       const res = await createTable(table)
 
       expect(events.table.created).toBeCalledTimes(1)
       expect(events.table.created).toBeCalledWith(res.body)
       expect(events.table.imported).toBeCalledTimes(1)
-      expect(events.table.imported).toBeCalledWith(res.body, "csv")
+      expect(events.table.imported).toBeCalledWith(res.body)
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(res.body, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(res.body, 1)
     })
 
     it("should apply authorization to endpoint", async () => {
@@ -155,11 +152,10 @@ describe("/tables", () => {
     it("imports rows successfully", async () => {
       const table = await config.createTable()
       const importRequest = {
-        dataImport: {
-          csvString: "\"name\",\"description\"\n\"test-name\",\"test-desc\"",
-          schema: table.schema
-        }
+        schema: table.schema,
+        rows: [{ name: 'test-name', description: 'test-desc' }]
       }
+
       jest.clearAllMocks()
 
       await request
@@ -171,7 +167,7 @@ describe("/tables", () => {
 
       expect(events.table.created).not.toHaveBeenCalled()
       expect(events.rows.imported).toBeCalledTimes(1)
-      expect(events.rows.imported).toBeCalledWith(table, "csv", 1)
+      expect(events.rows.imported).toBeCalledWith(table, 1)
     })
   })
 
@@ -206,24 +202,6 @@ describe("/tables", () => {
     })
   })
 
-  describe("validate csv", () => {
-    it("should be able to validate a CSV layout", async () => {
-      const res = await request
-        .post(`/api/tables/csv/validate`)
-        .send({
-          csvString: "a,b,c,d\n1,2,3,4"
-        })
-        .set(config.defaultHeaders())
-        .expect('Content-Type', /json/)
-        .expect(200)
-      expect(res.body.schema).toBeDefined()
-      expect(res.body.schema.a).toEqual({
-        type: "string",
-        success: true,
-      })
-    })
-  })
-
   describe("indexing", () => {
     it("should be able to create a table with indexes", async () => {
       await context.doInAppContext(appId, async () => {

package/src/api/routes/tests/user.spec.js

@@ -171,9 +171,28 @@ describe("/users", () => {
         .expect("Content-Type", /json/)
       expect(res.body.message).toEqual('User synced.')
     })
-  })
 
 
+    it("should sync the user when a previous user is specified", async () => {
+      const app1 = await config.createApp('App 1')
+      const app2 = await config.createApp('App 2')
 
+      let user = await config.createUser(
+        undefined,
+        undefined,
+        undefined,
+        undefined,
+        false,
+        true,
+        { [app1.appId]: 'ADMIN' })
+      let res = await request
+        .post(`/api/users/metadata/sync/${user._id}`)
+        .set(config.defaultHeaders())
+        .send({ previousUser: { ...user, roles: { ...user.roles, [app2.appId]: 'BASIC' } } })
+        .expect(200)
+        .expect("Content-Type", /json/)
 
+      expect(res.body.message).toEqual('User synced.')
+    })
+  })
 })

package/src/api/routes/utils/validators.ts

@@ -18,7 +18,7 @@ export function tableValidator() {
     schema: Joi.object().required(),
     name: Joi.string().required(),
     views: Joi.object(),
-    dataImport: Joi.object(),
+    rows: Joi.array(),
   }).unknown(true))
 }
 

package/src/app.ts

@@ -19,7 +19,7 @@ import * as api from "./api"
 import * as automations from "./automations"
 import { Thread } from "./threads"
 import * as redis from "./utilities/redis"
-import { events, logging } from "@budibase/backend-core"
+import { events, logging, middleware } from "@budibase/backend-core"
 import { initialise as initialiseWebsockets } from "./websocket"
 import { startup } from "./startup"
 const Sentry = require("@sentry/node")
@@ -40,6 +40,8 @@ app.use(
   })
 )
 
+app.use(middleware.logging)
+
 if (env.isProd()) {
   env._set("NODE_ENV", "production")
   Sentry.init()

package/src/automations/tests/loop.spec.ts

@@ -0,0 +1,45 @@
+import * as automation from "../index"
+import * as triggers from "../triggers"
+import { loopAutomation } from "../../tests/utilities/structures"
+import { context } from "@budibase/backend-core"
+import * as setup from "./utilities"
+
+describe("Attempt to run a basic loop automation", () => {
+  let config = setup.getConfig(),
+    table: any,
+    row: any
+
+  beforeEach(async () => {
+    await automation.init()
+    await config.init()
+    table = await config.createTable()
+    row = await config.createRow()
+  })
+
+  afterAll(setup.afterAll)
+
+  async function runLoop(loopOpts?: any) {
+    const appId = config.getAppId()
+    return await context.doInAppContext(appId, async () => {
+      const params = { fields: { appId } }
+      return await triggers.externalTrigger(
+        loopAutomation(table._id, loopOpts),
+        params,
+        { getResponses: true }
+      )
+    })
+  }
+
+  it("attempt to run a basic loop", async () => {
+    const resp = await runLoop()
+    expect(resp.steps[2].outputs.iterations).toBe(1)
+  })
+
+  it("test a loop with a string", async () => {
+    const resp = await runLoop({
+      type: "String",
+      binding: "a,b,c",
+    })
+    expect(resp.steps[2].outputs.iterations).toBe(3)
+  })
+})

package/src/automations/triggers.ts

@@ -109,8 +109,13 @@ export async function externalTrigger(
     }
     params.fields = coercedFields
   }
-  const data = { automation, event: params }
+  const data: Record<string, any> = { automation, event: params }
   if (getResponses) {
+    data.event = {
+      ...data.event,
+      appId: context.getAppId(),
+      automation,
+    }
     return utils.processEvent({ data })
   } else {
     return automationQueue.add(data, JOB_OPTS)

package/src/definitions/automations.ts

@@ -24,6 +24,7 @@ export interface TriggerOutput {
 
 export interface AutomationContext extends AutomationResults {
   steps: any[]
+  env?: Record<string, string>
   trigger: any
 }
 

package/src/definitions/datasource.ts

@@ -7,44 +7,3 @@
 export interface QueryOptions {
   disableReturning?: boolean
 }
-
-export enum AuthType {
-  BASIC = "basic",
-  BEARER = "bearer",
-}
-
-interface AuthConfig {
-  _id: string
-  name: string
-  type: AuthType
-  config: BasicAuthConfig | BearerAuthConfig
-}
-
-export interface BasicAuthConfig {
-  username: string
-  password: string
-}
-
-export interface BearerAuthConfig {
-  token: string
-}
-
-export interface RestConfig {
-  url: string
-  rejectUnauthorized: boolean
-  defaultHeaders: {
-    [key: string]: any
-  }
-  legacyHttpParser: boolean
-  authConfigs: AuthConfig[]
-  staticVariables: {
-    [key: string]: string
-  }
-  dynamicVariables: [
-    {
-      name: string
-      queryId: string
-      value: string
-    }
-  ]
-}

package/src/index.ts

@@ -1,6 +1,6 @@
 import { bootstrap } from "global-agent"
 const fixPath = require("fix-path")
-const { checkDevelopmentEnvironment } = require("./utilities/fileSystem")
+import { checkDevelopmentEnvironment } from "./utilities/fileSystem"
 
 function runServer() {
   // this will shutdown the system if development environment not ready

package/src/integrations/base/query.ts

@@ -1,10 +1,12 @@
 import { QueryJson, Datasource } from "@budibase/types"
-const { getIntegration } = require("../index")
+import { getIntegration } from "../index"
+import sdk from "../../sdk"
 
 export async function makeExternalQuery(
   datasource: Datasource,
   json: QueryJson
 ) {
+  datasource = await sdk.datasources.enrich(datasource)
   const Integration = await getIntegration(datasource.source)
   // query is the opinionated function
   if (Integration.prototype.query) {

package/src/integrations/base/sqlTable.ts

@@ -3,7 +3,7 @@ import { Operation, QueryJson, RenameColumn, Table } from "@budibase/types"
 import { breakExternalTableId } from "../utils"
 import SchemaBuilder = Knex.SchemaBuilder
 import CreateTableBuilder = Knex.CreateTableBuilder
-const { FieldTypes, RelationshipTypes } = require("../../constants")
+import { FieldTypes, RelationshipTypes } from "../../constants"
 
 function generateSchema(
   schema: CreateTableBuilder,

package/src/integrations/couchdb.ts

@@ -1,11 +1,11 @@
 import {
-  Integration,
   DatasourceFieldType,
-  QueryType,
+  Document,
+  Integration,
   IntegrationBase,
+  QueryType,
 } from "@budibase/types"
-
-const PouchDB = require("pouchdb")
+import { db as dbCore } from "@budibase/backend-core"
 
 interface CouchDBConfig {
   url: string
@@ -39,6 +39,15 @@ const SCHEMA: Integration = {
     update: {
       type: QueryType.JSON,
     },
+    get: {
+      type: QueryType.FIELDS,
+      fields: {
+        id: {
+          type: DatasourceFieldType.STRING,
+          required: true,
+        },
+      },
+    },
     delete: {
       type: QueryType.FIELDS,
       fields: {
@@ -57,7 +66,7 @@ class CouchDBIntegration implements IntegrationBase {
 
   constructor(config: CouchDBConfig) {
     this.config = config
-    this.client = new PouchDB(`${config.url}/${config.database}`)
+    this.client = dbCore.DatabaseWithConnection(config.database, config.url)
   }
 
   async query(
@@ -66,31 +75,48 @@ class CouchDBIntegration implements IntegrationBase {
     query: { json?: object; id?: string }
   ) {
     try {
-      const response = await this.client[command](query.id || query.json)
-      await this.client.close()
-      return response
+      return await this.client[command](query.id || query.json)
     } catch (err) {
       console.error(errorMsg, err)
       throw err
     }
   }
 
-  async create(query: { json: object }) {
-    return this.query("post", "Error writing to couchDB", query)
+  private parse(query: { json: string | object }) {
+    return typeof query.json === "string" ? JSON.parse(query.json) : query.json
+  }
+
+  async create(query: { json: string | object }) {
+    const parsed = this.parse(query)
+    return this.query("post", "Error writing to couchDB", { json: parsed })
   }
 
-  async read(query: { json: object }) {
+  async read(query: { json: string | object }) {
+    const parsed = this.parse(query)
     const result = await this.query("allDocs", "Error querying couchDB", {
       json: {
         include_docs: true,
-        ...query.json,
+        ...parsed,
       },
     })
     return result.rows.map((row: { doc: object }) => row.doc)
   }
 
-  async update(query: { json: object }) {
-    return this.query("put", "Error updating couchDB document", query)
+  async update(query: { json: string | object }) {
+    const parsed: Document = this.parse(query)
+    if (!parsed?._rev && parsed?._id) {
+      const oldDoc = await this.get({ id: parsed._id })
+      parsed._rev = oldDoc._rev
+    }
+    return this.query("put", "Error updating couchDB document", {
+      json: parsed,
+    })
+  }
+
+  async get(query: { id: string }) {
+    return this.query("get", "Error retrieving couchDB document by ID", {
+      id: query.id,
+    })
   }
 
   async delete(query: { id: string }) {

package/src/integrations/dynamodb.ts

@@ -5,8 +5,8 @@ import {
   IntegrationBase,
 } from "@budibase/types"
 
-const AWS = require("aws-sdk")
-const { AWS_REGION } = require("../db/dynamoClient")
+import AWS from "aws-sdk"
+import { AWS_REGION } from "../db/dynamoClient"
 
 interface DynamoDBConfig {
   region: string
@@ -182,7 +182,7 @@ class DynamoDBIntegration implements IntegrationBase {
     return response
   }
 
-  async describe(query: { table: string }) {
+  async describe(query: { table: string }): Promise<any> {
     const params = {
       TableName: query.table,
     }

package/src/integrations/elasticsearch.ts

@@ -80,11 +80,11 @@ const SCHEMA: Integration = {
     delete: {
       type: QueryType.FIELDS,
       fields: {
-        index: {
+        id: {
           type: DatasourceFieldType.STRING,
           required: true,
         },
-        id: {
+        index: {
           type: DatasourceFieldType.STRING,
           required: true,
         },
@@ -164,9 +164,13 @@ class ElasticSearchIntegration implements IntegrationBase {
     }
   }
 
-  async delete(query: object) {
+  async delete(query: { id: string; index: string }) {
+    const { id, index } = query
     try {
-      const result = await this.client.delete(query)
+      const result = await this.client.delete({
+        id,
+        index,
+      })
       return result.body
     } catch (err) {
       console.error("Error deleting from elasticsearch", err)

package/src/integrations/index.ts

@@ -17,8 +17,8 @@ import oracle from "./oracle"
 import { getPlugins } from "../api/controllers/plugin"
 import { SourceName, Integration, PluginType } from "@budibase/types"
 import { getDatasourcePlugin } from "../utilities/fileSystem"
-const environment = require("../environment")
-const { cloneDeep } = require("lodash")
+import env from "../environment"
+import { cloneDeep } from "lodash"
 
 const DEFINITIONS: { [key: string]: Integration } = {
   [SourceName.POSTGRES]: postgres.schema,
@@ -69,7 +69,7 @@ if (
 
 export async function getDefinitions() {
   const pluginSchemas: { [key: string]: Integration } = {}
-  if (environment.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
     const plugins = await getPlugins(PluginType.DATASOURCE)
     // extract the actual schema from each custom
     for (let plugin of plugins) {
@@ -93,7 +93,7 @@ export async function getIntegration(integration: string) {
   if (INTEGRATIONS[integration]) {
     return INTEGRATIONS[integration]
   }
-  if (environment.SELF_HOSTED) {
+  if (env.SELF_HOSTED) {
     const plugins = await getPlugins(PluginType.DATASOURCE)
     for (let plugin of plugins) {
       if (plugin.name === integration) {