@budibase/server 2.6.19-alpha.41 → 2.6.19-alpha.42

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.41",
+  "version": "2.6.19-alpha.42",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.41",
-    "@budibase/client": "2.6.19-alpha.41",
-    "@budibase/pro": "2.6.19-alpha.41",
-    "@budibase/shared-core": "2.6.19-alpha.41",
-    "@budibase/string-templates": "2.6.19-alpha.41",
-    "@budibase/types": "2.6.19-alpha.41",
+    "@budibase/backend-core": "2.6.19-alpha.42",
+    "@budibase/client": "2.6.19-alpha.42",
+    "@budibase/pro": "2.6.19-alpha.42",
+    "@budibase/shared-core": "2.6.19-alpha.42",
+    "@budibase/string-templates": "2.6.19-alpha.42",
+    "@budibase/types": "2.6.19-alpha.42",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -80,6 +80,7 @@
     "global-agent": "3.0.0",
     "google-auth-library": "7.12.0",
     "google-spreadsheet": "3.2.0",
+    "ioredis": "5.3.2",
     "jimp": "0.16.1",
     "joi": "17.6.0",
     "js-yaml": "4.1.0",
@@ -137,7 +138,6 @@
     "@types/bson": "4.2.0",
     "@types/global-agent": "2.1.1",
     "@types/google-spreadsheet": "3.1.5",
-    "@types/ioredis": "4.28.10",
     "@types/jest": "29.5.0",
     "@types/koa": "2.13.4",
     "@types/koa__router": "8.0.8",
@@ -157,7 +157,6 @@
     "copyfiles": "2.4.1",
     "docker-compose": "0.23.17",
     "eslint": "6.8.0",
-    "ioredis-mock": "7.2.0",
     "is-wsl": "2.2.0",
     "jest": "29.5.0",
     "jest-openapi": "0.14.2",
@@ -196,5 +195,5 @@
       }
     }
   },
-  "gitHead": "b45483f4b33be53615d8e5d1e3df8915f06bf0dd"
+  "gitHead": "2f80b0b4cc54ed4f18a711651ae262b85ad88906"
 }

package/src/api/controllers/datasource.ts

@@ -21,6 +21,7 @@ import {
   CreateDatasourceRequest,
   VerifyDatasourceRequest,
   VerifyDatasourceResponse,
+  FetchDatasourceInfoRequest,
   FetchDatasourceInfoResponse,
   IntegrationBase,
   DatasourcePlus,
@@ -57,6 +58,21 @@ async function getConnector(
   return new Connector(datasource.config)
 }
 
+async function getAndMergeDatasource(datasource: Datasource) {
+  let existingDatasource: undefined | Datasource
+  if (datasource._id) {
+    existingDatasource = await sdk.datasources.get(datasource._id)
+  }
+  let enrichedDatasource = datasource
+  if (existingDatasource) {
+    enrichedDatasource = sdk.datasources.mergeConfigs(
+      datasource,
+      existingDatasource
+    )
+  }
+  return await sdk.datasources.enrich(enrichedDatasource)
+}
+
 async function buildSchemaHelper(datasource: Datasource) {
   const connector = (await getConnector(datasource)) as DatasourcePlus
   await connector.buildSchema(datasource._id!, datasource.entities!)
@@ -132,17 +148,7 @@ export async function verify(
   ctx: UserCtx<VerifyDatasourceRequest, VerifyDatasourceResponse>
 ) {
   const { datasource } = ctx.request.body
-  let existingDatasource: undefined | Datasource
-  if (datasource._id) {
-    existingDatasource = await sdk.datasources.get(datasource._id)
-  }
-  let enrichedDatasource = datasource
-  if (existingDatasource) {
-    enrichedDatasource = sdk.datasources.mergeConfigs(
-      datasource,
-      existingDatasource
-    )
-  }
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
   const connector = await getConnector(enrichedDatasource)
   if (!connector.testConnection) {
     ctx.throw(400, "Connection information verification not supported")
@@ -156,11 +162,11 @@ export async function verify(
 }
 
 export async function information(
-  ctx: UserCtx<void, FetchDatasourceInfoResponse>
+  ctx: UserCtx<FetchDatasourceInfoRequest, FetchDatasourceInfoResponse>
 ) {
-  const datasourceId = ctx.params.datasourceId
-  const datasource = await sdk.datasources.get(datasourceId, { enriched: true })
-  const connector = (await getConnector(datasource)) as DatasourcePlus
+  const { datasource } = ctx.request.body
+  const enrichedDatasource = await getAndMergeDatasource(datasource)
+  const connector = (await getConnector(enrichedDatasource)) as DatasourcePlus
   if (!connector.getTableNames) {
     ctx.throw(400, "Table name fetching not supported by datasource")
   }
@@ -297,7 +303,7 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
-  builderSocket.emitDatasourceUpdate(ctx, datasource)
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 export async function save(
@@ -340,7 +346,7 @@ export async function save(
     response.error = schemaError
   }
   ctx.body = response
-  builderSocket.emitDatasourceUpdate(ctx, datasource)
+  builderSocket?.emitDatasourceUpdate(ctx, datasource)
 }
 
 async function destroyInternalTablesBySourceId(datasourceId: string) {
@@ -400,7 +406,7 @@ export async function destroy(ctx: UserCtx) {
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200
-  builderSocket.emitDatasourceDeletion(ctx, datasourceId)
+  builderSocket?.emitDatasourceDeletion(ctx, datasourceId)
 }
 
 export async function find(ctx: UserCtx) {

package/src/api/controllers/plugin/index.ts

@@ -71,7 +71,7 @@ export async function create(ctx: any) {
 
     const doc = await pro.plugins.storePlugin(metadata, directory, source)
 
-    clientAppSocket.emit("plugins-update", { name, hash: doc.hash })
+    clientAppSocket?.emit("plugins-update", { name, hash: doc.hash })
     ctx.body = {
       message: "Plugin uploaded successfully",
       plugins: [doc],

package/src/api/controllers/table/index.ts

@@ -78,7 +78,7 @@ export async function save(ctx: UserCtx) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
   ctx.body = savedTable
-  builderSocket.emitTableUpdate(ctx, savedTable)
+  builderSocket?.emitTableUpdate(ctx, savedTable)
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -91,7 +91,7 @@ export async function destroy(ctx: UserCtx) {
   ctx.status = 200
   ctx.table = deletedTable
   ctx.body = { message: `Table ${tableId} deleted.` }
-  builderSocket.emitTableDeletion(ctx, tableId)
+  builderSocket?.emitTableDeletion(ctx, tableId)
 }
 
 export async function bulkImport(ctx: UserCtx) {

package/src/api/controllers/view/index.ts

@@ -58,7 +58,7 @@ export async function save(ctx: Ctx) {
   await handleViewEvents(existingTable.views[viewName], table.views[viewName])
 
   ctx.body = table.views[viewName]
-  builderSocket.emitTableUpdate(ctx, table)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function calculationEvents(existingView: View, newView: View) {
@@ -127,7 +127,7 @@ export async function destroy(ctx: Ctx) {
   await events.view.deleted(view)
 
   ctx.body = view
-  builderSocket.emitTableUpdate(ctx, table)
+  builderSocket?.emitTableUpdate(ctx, table)
 }
 
 export async function exportView(ctx: Ctx) {

package/src/api/routes/datasource.ts

@@ -20,8 +20,8 @@ router
     authorized(permissions.BUILDER),
     datasourceController.verify
   )
-  .get(
-    "/api/datasources/:datasourceId/info",
+  .post(
+    "/api/datasources/info",
     authorized(permissions.BUILDER),
     datasourceController.information
   )

package/src/api/routes/tests/view.spec.js

@@ -114,8 +114,8 @@ describe("/views", () => {
       expect(res.body.tableId).toBe(table._id)
 
       const updatedTable = await config.getTable(table._id)
-      expect(updatedTable.views).toEqual({
-        TestView: {
+      const expectedObj = expect.objectContaining({
+        TestView: expect.objectContaining({
           field: "Price",
           calculation: "stats",
           tableId: table._id,
@@ -143,8 +143,9 @@ describe("/views", () => {
               type: "string",
             },
           },
-        },
+        }),
       })
+      expect(updatedTable.views).toEqual(expectedObj)
     })
   })
 

package/src/integration-test/postgres.spec.ts

@@ -25,6 +25,7 @@ const config = setup.getConfig()!
 jest.setTimeout(30000)
 
 jest.unmock("pg")
+jest.mock("../websockets")
 
 describe("postgres integrations", () => {
   let makeRequest: MakeRequestResponse,
@@ -1055,13 +1056,12 @@ describe("postgres integrations", () => {
     })
   })
 
-  describe("GET /api/datasources/:datasourceId/info", () => {
+  describe("POST /api/datasources/info", () => {
     it("should fetch information about postgres datasource", async () => {
       const primaryName = primaryPostgresTable.name
-      const response = await makeRequest(
-        "get",
-        `/api/datasources/${postgresDatasource._id}/info`
-      )
+      const response = await makeRequest("post", "/api/datasources/info", {
+        datasource: postgresDatasource,
+      })
       expect(response.status).toBe(200)
       expect(response.body.tableNames).toBeDefined()
       expect(response.body.tableNames.indexOf(primaryName)).not.toBe(-1)

package/src/integrations/redis.ts

@@ -177,7 +177,13 @@ class RedisIntegration {
       const pipeline = this.client.pipeline(pipelineCommands)
       const result = await pipeline.exec()
 
-      return result.map((output: string | string[]) => output[1])
+      return result?.map((output: any) => {
+        if (typeof output === "string") {
+          return output
+        } else if (Array.isArray(output)) {
+          return output[1]
+        }
+      })
     })
   }
 }

package/src/sdk/plugins/plugins.ts

@@ -36,6 +36,6 @@ export async function processUploaded(plugin: FileType, source?: PluginSource) {
   }
 
   const doc = await pro.plugins.storePlugin(metadata, directory, source)
-  clientAppSocket.emit("plugin-update", { name: doc.name, hash: doc.hash })
+  clientAppSocket?.emit("plugin-update", { name: doc.name, hash: doc.hash })
   return doc
 }

package/src/threads/automation.ts

@@ -44,10 +44,18 @@ function getLoopIterations(loopStep: LoopStep) {
   if (!binding) {
     return 0
   }
+  const isString = typeof binding === "string"
+  try {
+    if (isString) {
+      binding = JSON.parse(binding)
+    }
+  } catch (err) {
+    // ignore error - wasn't able to parse
+  }
   if (Array.isArray(binding)) {
     return binding.length
   }
-  if (typeof binding === "string") {
+  if (isString) {
     return automationUtils.stringSplit(binding).length
   }
   return 0

package/src/utilities/redis.ts

@@ -1,13 +1,16 @@
-import { redis } from "@budibase/backend-core"
+import { redis, RedisClient } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import { ContextUser } from "@budibase/types"
+import env from "../environment"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient: RedisClient,
+  debounceClient: RedisClient,
+  flagClient: RedisClient
 
 // We need to maintain a duplicate client for socket.io pub/sub
-let socketClient: any
+let socketClient: RedisClient
 let socketSubClient: any
 
 // We init this as we want to keep the connection open all the time
@@ -22,7 +25,9 @@ export async function init() {
 
   // Duplicate the socket client for pub/sub
   socketClient = await redis.clients.getSocketClient()
-  socketSubClient = socketClient.getClient().duplicate()
+  if (!env.isTest()) {
+    socketSubClient = socketClient.getClient().duplicate()
+  }
 }
 
 export async function shutdown() {

package/src/websockets/builder.ts

@@ -14,9 +14,9 @@ export default class BuilderSocket extends BaseSocket {
     super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
   }
 
-  async onConnect(socket: Socket) {
+  async onConnect(socket?: Socket) {
     // Initial identification of selected app
-    socket.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
+    socket?.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
       await this.joinRoom(socket, appId)
 
       // Reply with all users in current room
@@ -49,14 +49,14 @@ export default class BuilderSocket extends BaseSocket {
     this.io
       .in(ctx.appId)
       .emit(BuilderSocketEvent.TableChange, { id: table._id, table })
-    gridSocket.emitTableUpdate(table)
+    gridSocket?.emitTableUpdate(table)
   }
 
   emitTableDeletion(ctx: any, id: string) {
     this.io
       .in(ctx.appId)
       .emit(BuilderSocketEvent.TableChange, { id, table: null })
-    gridSocket.emitTableDeletion(id)
+    gridSocket?.emitTableDeletion(id)
   }
 
   emitDatasourceUpdate(ctx: any, datasource: Datasource) {

package/src/websockets/index.ts

@@ -3,15 +3,19 @@ import Koa from "koa"
 import ClientAppSocket from "./client"
 import GridSocket from "./grid"
 import BuilderSocket from "./builder"
+import env from "../environment"
 
-let clientAppSocket: ClientAppSocket
-let gridSocket: GridSocket
-let builderSocket: BuilderSocket
+let clientAppSocket: ClientAppSocket | undefined
+let gridSocket: GridSocket | undefined
+let builderSocket: BuilderSocket | undefined
 
 export const initialise = (app: Koa, server: http.Server) => {
-  clientAppSocket = new ClientAppSocket(app, server)
-  gridSocket = new GridSocket(app, server)
-  builderSocket = new BuilderSocket(app, server)
+  // have to remove these for testing until ioredis-mock can be fully removed
+  if (!env.isTest()) {
+    clientAppSocket = new ClientAppSocket(app, server)
+    gridSocket = new GridSocket(app, server)
+    builderSocket = new BuilderSocket(app, server)
+  }
 }
 
 export { clientAppSocket, gridSocket, builderSocket }