@budibase/server 2.6.19-alpha.26 → 2.6.19-alpha.27

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.26",
+  "version": "2.6.19-alpha.27",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.26",
-    "@budibase/client": "2.6.19-alpha.26",
-    "@budibase/pro": "2.6.19-alpha.26",
-    "@budibase/shared-core": "2.6.19-alpha.26",
-    "@budibase/string-templates": "2.6.19-alpha.26",
-    "@budibase/types": "2.6.19-alpha.26",
+    "@budibase/backend-core": "2.6.19-alpha.27",
+    "@budibase/client": "2.6.19-alpha.27",
+    "@budibase/pro": "2.6.19-alpha.27",
+    "@budibase/shared-core": "2.6.19-alpha.27",
+    "@budibase/string-templates": "2.6.19-alpha.27",
+    "@budibase/types": "2.6.19-alpha.27",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -59,6 +59,7 @@
     "@koa/router": "8.0.8",
     "@sendgrid/mail": "7.1.1",
     "@sentry/node": "6.17.7",
+    "@socket.io/redis-adapter": "^8.2.1",
     "airtable": "0.10.1",
     "arangojs": "7.2.0",
     "aws-sdk": "2.1030.0",
@@ -194,5 +195,5 @@
       }
     }
   },
-  "gitHead": "dbeebc571d598a9a6696dbc15e9b1e38e66a80a3"
+  "gitHead": "84334ce650f8782d49ca9c7b5a7186e5232ad91a"
 }

package/src/api/controllers/application.ts

@@ -29,7 +29,7 @@ import { USERS_TABLE_SCHEMA } from "../../constants"
 import { buildDefaultDocs } from "../../db/defaultData/datasource_bb_default"
 import { removeAppFromUserRoles } from "../../utilities/workerRequests"
 import { stringToReadStream, isQsTrue } from "../../utilities"
-import { getLocksById } from "../../utilities/redis"
+import { getLocksById, doesUserHaveLock } from "../../utilities/redis"
 import {
   updateClientLibrary,
   backupClientLibrary,
@@ -227,6 +227,7 @@ export async function fetchAppPackage(ctx: UserCtx) {
     screens,
     layouts,
     clientLibPath,
+    hasLock: await doesUserHaveLock(application.appId, ctx.user),
   }
 }
 

package/src/api/controllers/datasource.ts

@@ -26,6 +26,7 @@ import {
   DatasourcePlus,
 } from "@budibase/types"
 import sdk from "../../sdk"
+import { builderSocket } from "../../websockets"
 
 function getErrorTables(errors: any, errorType: string) {
   return Object.entries(errors)
@@ -296,6 +297,7 @@ export async function update(ctx: UserCtx<any, UpdateDatasourceResponse>) {
   ctx.body = {
     datasource: await sdk.datasources.removeSecretSingle(datasource),
   }
+  builderSocket.emitDatasourceUpdate(ctx, datasource)
 }
 
 export async function save(
@@ -338,6 +340,7 @@ export async function save(
     response.error = schemaError
   }
   ctx.body = response
+  builderSocket.emitDatasourceUpdate(ctx, datasource)
 }
 
 async function destroyInternalTablesBySourceId(datasourceId: string) {
@@ -397,6 +400,7 @@ export async function destroy(ctx: UserCtx) {
 
   ctx.message = `Datasource deleted.`
   ctx.status = 200
+  builderSocket.emitDatasourceDeletion(ctx, datasourceId)
 }
 
 export async function find(ctx: UserCtx) {

package/src/api/controllers/row/index.ts

@@ -4,6 +4,7 @@ import * as external from "./external"
 import { isExternalTable } from "../../../integrations/utils"
 import { Ctx } from "@budibase/types"
 import * as utils from "./utils"
+import { gridSocket } from "../../../websockets"
 
 function pickApi(tableId: any) {
   if (isExternalTable(tableId)) {
@@ -12,21 +13,9 @@ function pickApi(tableId: any) {
   return internal
 }
 
-function getTableId(ctx: any) {
-  if (ctx.request.body && ctx.request.body.tableId) {
-    return ctx.request.body.tableId
-  }
-  if (ctx.params && ctx.params.tableId) {
-    return ctx.params.tableId
-  }
-  if (ctx.params && ctx.params.viewName) {
-    return ctx.params.viewName
-  }
-}
-
 export async function patch(ctx: any): Promise<any> {
   const appId = ctx.appId
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   const body = ctx.request.body
   // if it doesn't have an _id then its save
   if (body && !body._id) {
@@ -47,6 +36,7 @@ export async function patch(ctx: any): Promise<any> {
       ctx.eventEmitter.emitRow(`row:update`, appId, row, table)
     ctx.message = `${table.name} updated successfully.`
     ctx.body = row
+    gridSocket?.emitRowUpdate(ctx, row)
   } catch (err) {
     ctx.throw(400, err)
   }
@@ -54,7 +44,7 @@ export async function patch(ctx: any): Promise<any> {
 
 export const save = async (ctx: any) => {
   const appId = ctx.appId
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   const body = ctx.request.body
   // if it has an ID already then its a patch
   if (body && body._id) {
@@ -69,23 +59,24 @@ export const save = async (ctx: any) => {
   ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:save`, appId, row, table)
   ctx.message = `${table.name} saved successfully`
   ctx.body = row
+  gridSocket?.emitRowUpdate(ctx, row)
 }
 export async function fetchView(ctx: any) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).fetchView(ctx), {
     datasourceId: tableId,
   })
 }
 
 export async function fetch(ctx: any) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).fetch(ctx), {
     datasourceId: tableId,
   })
 }
 
 export async function find(ctx: any) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).find(ctx), {
     datasourceId: tableId,
   })
@@ -94,7 +85,7 @@ export async function find(ctx: any) {
 export async function destroy(ctx: any) {
   const appId = ctx.appId
   const inputs = ctx.request.body
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   let response, row
   if (inputs.rows) {
     let { rows } = await quotas.addQuery(
@@ -107,6 +98,7 @@ export async function destroy(ctx: any) {
     response = rows
     for (let row of rows) {
       ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
+      gridSocket?.emitRowDeletion(ctx, row._id)
     }
   } else {
     let resp = await quotas.addQuery(() => pickApi(tableId).destroy(ctx), {
@@ -116,6 +108,7 @@ export async function destroy(ctx: any) {
     response = resp.response
     row = resp.row
     ctx.eventEmitter && ctx.eventEmitter.emitRow(`row:delete`, appId, row)
+    gridSocket?.emitRowDeletion(ctx, row._id)
   }
   ctx.status = 200
   // for automations include the row that was deleted
@@ -124,7 +117,7 @@ export async function destroy(ctx: any) {
 }
 
 export async function search(ctx: any) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.status = 200
   ctx.body = await quotas.addQuery(() => pickApi(tableId).search(ctx), {
     datasourceId: tableId,
@@ -132,7 +125,7 @@ export async function search(ctx: any) {
 }
 
 export async function validate(ctx: Ctx) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   // external tables are hard to validate currently
   if (isExternalTable(tableId)) {
     ctx.body = { valid: true }
@@ -145,7 +138,7 @@ export async function validate(ctx: Ctx) {
 }
 
 export async function fetchEnrichedRow(ctx: any) {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(
     () => pickApi(tableId).fetchEnrichedRow(ctx),
     {
@@ -155,7 +148,7 @@ export async function fetchEnrichedRow(ctx: any) {
 }
 
 export const exportRows = async (ctx: any) => {
-  const tableId = getTableId(ctx)
+  const tableId = utils.getTableId(ctx)
   ctx.body = await quotas.addQuery(() => pickApi(tableId).exportRows(ctx), {
     datasourceId: tableId,
   })

package/src/api/controllers/row/utils.ts

@@ -154,3 +154,15 @@ export function cleanExportRows(
 
   return cleanRows
 }
+
+export function getTableId(ctx: any) {
+  if (ctx.request.body && ctx.request.body.tableId) {
+    return ctx.request.body.tableId
+  }
+  if (ctx.params && ctx.params.tableId) {
+    return ctx.params.tableId
+  }
+  if (ctx.params && ctx.params.viewName) {
+    return ctx.params.viewName
+  }
+}

package/src/api/controllers/table/index.ts

@@ -11,6 +11,7 @@ import { context, events } from "@budibase/backend-core"
 import { Table, UserCtx } from "@budibase/types"
 import sdk from "../../../sdk"
 import { jsonFromCsvString } from "../../../utilities/csv"
+import { builderSocket } from "../../../websockets"
 
 function pickApi({ tableId, table }: { tableId?: string; table?: Table }) {
   if (table && !tableId) {
@@ -77,6 +78,7 @@ export async function save(ctx: UserCtx) {
   ctx.eventEmitter &&
     ctx.eventEmitter.emitTable(`table:save`, appId, savedTable)
   ctx.body = savedTable
+  builderSocket.emitTableUpdate(ctx, savedTable)
 }
 
 export async function destroy(ctx: UserCtx) {
@@ -89,6 +91,7 @@ export async function destroy(ctx: UserCtx) {
   ctx.status = 200
   ctx.table = deletedTable
   ctx.body = { message: `Table ${tableId} deleted.` }
+  builderSocket.emitTableDeletion(ctx, tableId)
 }
 
 export async function bulkImport(ctx: UserCtx) {

package/src/api/controllers/view/index.ts

@@ -16,6 +16,7 @@ import {
   View,
 } from "@budibase/types"
 import { cleanExportRows } from "../row/utils"
+import { builderSocket } from "../../../websockets"
 
 const { cloneDeep, isEqual } = require("lodash")
 
@@ -48,7 +49,7 @@ export async function save(ctx: Ctx) {
   if (!view.meta.schema) {
     view.meta.schema = table.schema
   }
-  table.views[viewName] = view.meta
+  table.views[viewName] = { ...view.meta, name: viewName }
   if (originalName) {
     delete table.views[originalName]
     existingTable.views[viewName] = existingTable.views[originalName]
@@ -56,10 +57,8 @@ export async function save(ctx: Ctx) {
   await db.put(table)
   await handleViewEvents(existingTable.views[viewName], table.views[viewName])
 
-  ctx.body = {
-    ...table.views[viewToSave.name],
-    name: viewToSave.name,
-  }
+  ctx.body = table.views[viewName]
+  builderSocket.emitTableUpdate(ctx, table)
 }
 
 export async function calculationEvents(existingView: View, newView: View) {
@@ -128,6 +127,7 @@ export async function destroy(ctx: Ctx) {
   await events.view.deleted(view)
 
   ctx.body = view
+  builderSocket.emitTableUpdate(ctx, table)
 }
 
 export async function exportView(ctx: Ctx) {

package/src/app.ts

@@ -61,7 +61,6 @@ if (env.isProd()) {
 
 const server = http.createServer(app.callback())
 destroyable(server)
-initialiseWebsockets(app, server)
 
 let shuttingDown = false,
   errCode = 0

package/src/middleware/builder.ts

@@ -35,12 +35,11 @@ async function checkDevAppLocks(ctx: BBContext) {
   if (!appId || !appId.startsWith(APP_DEV_PREFIX)) {
     return
   }
-  if (!(await doesUserHaveLock(appId, ctx.user))) {
-    ctx.throw(400, "User does not hold app lock.")
-  }
 
-  // they do have lock, update it
-  await updateLock(appId, ctx.user)
+  // If this user already owns the lock, then update it
+  if (await doesUserHaveLock(appId, ctx.user)) {
+    await updateLock(appId, ctx.user)
+  }
 }
 
 async function updateAppUpdatedAt(ctx: BBContext) {

package/src/startup.ts

@@ -16,6 +16,7 @@ import * as bullboard from "./automations/bullboard"
 import * as pro from "@budibase/pro"
 import * as api from "./api"
 import sdk from "./sdk"
+import { initialise as initialiseWebsockets } from "./websockets"
 
 let STARTUP_RAN = false
 
@@ -64,6 +65,7 @@ export async function startup(app?: any, server?: any) {
   fileSystem.init()
   await redis.init()
   eventInit()
+  initialiseWebsockets(app, server)
 
   // run migrations on startup if not done via http
   // not recommended in a clustered environment

package/src/utilities/redis.ts

@@ -4,23 +4,33 @@ import { ContextUser } from "@budibase/types"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any
 
-// we init this as we want to keep the connection open all the time
+// We need to maintain a duplicate client for socket.io pub/sub
+let socketSubClient: any
+
+// We init this as we want to keep the connection open all the time
 // reduces the performance hit
 export async function init() {
   devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
   debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
   flagClient = new redis.Client(redis.utils.Databases.FLAGS)
+  socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
+  await socketClient.init()
+
+  // Duplicate the socket client for pub/sub
+  socketSubClient = socketClient.getClient().duplicate()
 }
 
 export async function shutdown() {
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  if (socketClient) await socketClient.finish()
+  if (socketSubClient) socketSubClient.disconnect()
   // shutdown core clients
   await redis.clients.shutdown()
   console.log("Redis shutdown")
@@ -86,3 +96,10 @@ export async function checkTestFlag(id: string) {
 export async function clearTestFlag(id: string) {
   await devAppClient.delete(id)
 }
+
+export function getSocketPubSubClients() {
+  return {
+    pub: socketClient.getClient(),
+    sub: socketSubClient,
+  }
+}

package/src/websockets/builder.ts

@@ -0,0 +1,69 @@
+import authorized from "../middleware/authorized"
+import Socket from "./websocket"
+import { permissions } from "@budibase/backend-core"
+import http from "http"
+import Koa from "koa"
+import { Datasource, Table } from "@budibase/types"
+import { gridSocket } from "./index"
+import { clearLock } from "../utilities/redis"
+
+export default class BuilderSocket extends Socket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
+
+    this.io.on("connection", socket => {
+      // Join a room for this app
+      const user = socket.data.user
+      const appId = socket.data.appId
+      socket.join(appId)
+      socket.to(appId).emit("user-update", user)
+
+      // Initial identification of connected spreadsheet
+      socket.on("get-users", async (payload, callback) => {
+        const sockets = await this.io.in(appId).fetchSockets()
+        callback({
+          users: sockets.map(socket => socket.data.user),
+        })
+      })
+
+      // Disconnection cleanup
+      socket.on("disconnect", async () => {
+        socket.to(appId).emit("user-disconnect", user)
+
+        // Remove app lock from this user if they have no other connections
+        try {
+          const sockets = await this.io.in(appId).fetchSockets()
+          const hasOtherConnection = sockets.some(socket => {
+            const { _id, sessionId } = socket.data.user
+            return _id === user._id && sessionId !== user.sessionId
+          })
+          if (!hasOtherConnection) {
+            await clearLock(appId, user)
+          }
+        } catch (e) {
+          // This is fine, just means this user didn't hold the lock
+        }
+      })
+    })
+  }
+
+  emitTableUpdate(ctx: any, table: Table) {
+    this.io.in(ctx.appId).emit("table-change", { id: table._id, table })
+    gridSocket.emitTableUpdate(table)
+  }
+
+  emitTableDeletion(ctx: any, id: string) {
+    this.io.in(ctx.appId).emit("table-change", { id, table: null })
+    gridSocket.emitTableDeletion(id)
+  }
+
+  emitDatasourceUpdate(ctx: any, datasource: Datasource) {
+    this.io
+      .in(ctx.appId)
+      .emit("datasource-change", { id: datasource._id, datasource })
+  }
+
+  emitDatasourceDeletion(ctx: any, id: string) {
+    this.io.in(ctx.appId).emit("datasource-change", { id, datasource: null })
+  }
+}

package/src/websockets/grid.ts

@@ -3,6 +3,8 @@ import Socket from "./websocket"
 import { permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
+import { getTableId } from "../api/controllers/row/utils"
+import { Row, Table } from "@budibase/types"
 
 export default class GridSocket extends Socket {
   constructor(app: Koa, server: http.Server) {
@@ -10,7 +12,6 @@ export default class GridSocket extends Socket {
 
     this.io.on("connection", socket => {
       const user = socket.data.user
-      console.log(`Spreadsheet user connected: ${user?.id}`)
 
       // Socket state
       let currentRoom: string
@@ -19,37 +20,54 @@ export default class GridSocket extends Socket {
       socket.on("select-table", async (tableId, callback) => {
         // Leave current room
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+          socket.to(currentRoom).emit("user-disconnect", user)
           socket.leave(currentRoom)
         }
 
         // Join new room
         currentRoom = tableId
         socket.join(currentRoom)
-        socket.to(currentRoom).emit("user-update", socket.data.user)
+        socket.to(currentRoom).emit("user-update", user)
 
         // Reply with all users in current room
         const sockets = await this.io.in(currentRoom).fetchSockets()
         callback({
           users: sockets.map(socket => socket.data.user),
-          id: user.id,
         })
       })
 
       // Handle users selecting a new cell
       socket.on("select-cell", cellId => {
-        socket.data.user.selectedCellId = cellId
+        socket.data.user.focusedCellId = cellId
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-update", socket.data.user)
+          socket.to(currentRoom).emit("user-update", user)
         }
       })
 
       // Disconnection cleanup
       socket.on("disconnect", () => {
         if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
+          socket.to(currentRoom).emit("user-disconnect", user)
         }
       })
     })
   }
+
+  emitRowUpdate(ctx: any, row: Row) {
+    const tableId = getTableId(ctx)
+    this.io.in(tableId).emit("row-change", { id: row._id, row })
+  }
+
+  emitRowDeletion(ctx: any, id: string) {
+    const tableId = getTableId(ctx)
+    this.io.in(tableId).emit("row-change", { id, row: null })
+  }
+
+  emitTableUpdate(table: Table) {
+    this.io.in(table._id!).emit("table-change", { id: table._id, table })
+  }
+
+  emitTableDeletion(id: string) {
+    this.io.in(id).emit("table-change", { id, table: null })
+  }
 }

package/src/websockets/index.ts

@@ -1,14 +1,17 @@
 import http from "http"
 import Koa from "koa"
-import GridSocket from "./grid"
 import ClientAppSocket from "./client"
+import GridSocket from "./grid"
+import BuilderSocket from "./builder"
 
 let clientAppSocket: ClientAppSocket
 let gridSocket: GridSocket
+let builderSocket: BuilderSocket
 
 export const initialise = (app: Koa, server: http.Server) => {
   clientAppSocket = new ClientAppSocket(app, server)
   gridSocket = new GridSocket(app, server)
+  builderSocket = new BuilderSocket(app, server)
 }
 
-export { clientAppSocket, gridSocket }
+export { clientAppSocket, gridSocket, builderSocket }

package/src/websockets/websocket.ts

@@ -5,6 +5,9 @@ import Cookies from "cookies"
 import { userAgent } from "koa-useragent"
 import { auth } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
+import { createAdapter } from "@socket.io/redis-adapter"
+import { getSocketPubSubClients } from "../utilities/redis"
+import uuid from "uuid"
 
 export default class Socket {
   io: Server
@@ -12,7 +15,7 @@ export default class Socket {
   constructor(
     app: Koa,
     server: http.Server,
-    path: string,
+    path: string = "/",
     additionalMiddlewares?: any[]
   ) {
     this.io = new Server(server, {
@@ -59,13 +62,21 @@ export default class Socket {
         for (let [idx, middleware] of middlewares.entries()) {
           await middleware(ctx, () => {
             if (idx === middlewares.length - 1) {
-              // Middlewares are finished.
+              // Middlewares are finished
               // Extract some data from our enriched koa context to persist
               // as metadata for the socket
+              // Add user info, including a deterministic color and label
+              const { _id, email, firstName, lastName } = ctx.user
               socket.data.user = {
-                id: ctx.user._id,
-                email: ctx.user.email,
+                _id,
+                email,
+                firstName,
+                lastName,
+                sessionId: uuid.v4(),
               }
+
+              // Add app ID to help split sockets into rooms
+              socket.data.appId = ctx.appId
               next()
             }
           })
@@ -74,6 +85,11 @@ export default class Socket {
         next(error)
       }
     })
+
+    // Instantiate redis adapter
+    const { pub, sub } = getSocketPubSubClients()
+    const opts = { key: `socket.io-${path}` }
+    this.io.adapter(createAdapter(pub, sub, opts))
   }
 
   // Emit an event to all sockets