@budibase/server 2.6.19-alpha.36 → 2.6.19-alpha.38

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@budibase/server",
   "email": "hi@budibase.com",
-  "version": "2.6.19-alpha.36",
+  "version": "2.6.19-alpha.38",
   "description": "Budibase Web Server",
   "main": "src/index.ts",
   "repository": {
@@ -46,12 +46,12 @@
   "license": "GPL-3.0",
   "dependencies": {
     "@apidevtools/swagger-parser": "10.0.3",
-    "@budibase/backend-core": "2.6.19-alpha.36",
-    "@budibase/client": "2.6.19-alpha.36",
-    "@budibase/pro": "2.6.19-alpha.36",
-    "@budibase/shared-core": "2.6.19-alpha.36",
-    "@budibase/string-templates": "2.6.19-alpha.36",
-    "@budibase/types": "2.6.19-alpha.36",
+    "@budibase/backend-core": "2.6.19-alpha.38",
+    "@budibase/client": "2.6.19-alpha.38",
+    "@budibase/pro": "2.6.19-alpha.38",
+    "@budibase/shared-core": "2.6.19-alpha.38",
+    "@budibase/string-templates": "2.6.19-alpha.38",
+    "@budibase/types": "2.6.19-alpha.38",
     "@bull-board/api": "3.7.0",
     "@bull-board/koa": "3.9.4",
     "@elastic/elasticsearch": "7.10.0",
@@ -196,5 +196,5 @@
       }
     }
   },
-  "gitHead": "2bcc1449a8531d74fe138fb63575d244817d1918"
+  "gitHead": "67e4eb65b65c13813fbbb5e21e180234e3328755"
 }

package/src/api/controllers/deploy/index.ts

@@ -104,6 +104,7 @@ export async function fetchDeployments(ctx: any) {
     }
     ctx.body = Object.values(deployments.history).reverse()
   } catch (err) {
+    console.error(err)
     ctx.body = []
   }
 }

package/src/utilities/redis.ts

@@ -4,9 +4,10 @@ import { ContextUser } from "@budibase/types"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any, socketClient: any
+let devAppClient: any, debounceClient: any, flagClient: any
 
 // We need to maintain a duplicate client for socket.io pub/sub
+let socketClient: any
 let socketSubClient: any
 
 // We init this as we want to keep the connection open all the time
@@ -15,21 +16,20 @@ export async function init() {
   devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
   debounceClient = new redis.Client(redis.utils.Databases.DEBOUNCE)
   flagClient = new redis.Client(redis.utils.Databases.FLAGS)
-  socketClient = new redis.Client(redis.utils.Databases.SOCKET_IO)
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
-  await socketClient.init()
 
   // Duplicate the socket client for pub/sub
+  socketClient = await redis.clients.getSocketClient()
   socketSubClient = socketClient.getClient().duplicate()
 }
 
 export async function shutdown() {
+  console.log("REDIS SHUTDOWN")
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
-  if (socketClient) await socketClient.finish()
   if (socketSubClient) socketSubClient.disconnect()
   // shutdown core clients
   await redis.clients.shutdown()

package/src/websockets/builder.ts

@@ -1,69 +1,74 @@
 import authorized from "../middleware/authorized"
-import Socket from "./websocket"
+import { BaseSocket } from "./websocket"
 import { permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
-import { Datasource, Table } from "@budibase/types"
+import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types"
 import { gridSocket } from "./index"
 import { clearLock } from "../utilities/redis"
+import { Socket } from "socket.io"
+import { BuilderSocketEvent } from "@budibase/shared-core"
 
-export default class BuilderSocket extends Socket {
+export default class BuilderSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
     super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
+  }
 
-    this.io.on("connection", socket => {
-      // Join a room for this app
-      const user = socket.data.user
-      const appId = socket.data.appId
-      socket.join(appId)
-      socket.to(appId).emit("user-update", user)
-
-      // Initial identification of connected spreadsheet
-      socket.on("get-users", async (payload, callback) => {
-        const sockets = await this.io.in(appId).fetchSockets()
-        callback({
-          users: sockets.map(socket => socket.data.user),
-        })
-      })
+  async onConnect(socket: Socket) {
+    // Initial identification of selected app
+    socket.on(BuilderSocketEvent.SelectApp, async (appId, callback) => {
+      await this.joinRoom(socket, appId)
 
-      // Disconnection cleanup
-      socket.on("disconnect", async () => {
-        socket.to(appId).emit("user-disconnect", user)
+      // Reply with all users in current room
+      const sessions = await this.getRoomSessions(appId)
+      callback({ users: sessions })
+    })
+  }
 
-        // Remove app lock from this user if they have no other connections
-        try {
-          const sockets = await this.io.in(appId).fetchSockets()
-          const hasOtherConnection = sockets.some(socket => {
-            const { _id, sessionId } = socket.data.user
-            return _id === user._id && sessionId !== user.sessionId
-          })
-          if (!hasOtherConnection) {
-            await clearLock(appId, user)
-          }
-        } catch (e) {
-          // This is fine, just means this user didn't hold the lock
-        }
+  async onDisconnect(socket: Socket) {
+    // Remove app lock from this user if they have no other connections
+    try {
+      // @ts-ignore
+      const session: SocketSession = socket.data
+      const { _id, sessionId, room } = session
+      const sessions = await this.getRoomSessions(room)
+      const hasOtherSession = sessions.some(otherSession => {
+        return _id === otherSession._id && sessionId !== otherSession.sessionId
       })
-    })
+      if (!hasOtherSession && room) {
+        // @ts-ignore
+        const user: ContextUser = { _id: socket.data._id }
+        await clearLock(room, user)
+      }
+    } catch (e) {
+      // This is fine, just means this user didn't hold the lock
+    }
   }
 
   emitTableUpdate(ctx: any, table: Table) {
-    this.io.in(ctx.appId).emit("table-change", { id: table._id, table })
+    this.io
+      .in(ctx.appId)
+      .emit(BuilderSocketEvent.TableChange, { id: table._id, table })
     gridSocket.emitTableUpdate(table)
   }
 
   emitTableDeletion(ctx: any, id: string) {
-    this.io.in(ctx.appId).emit("table-change", { id, table: null })
+    this.io
+      .in(ctx.appId)
+      .emit(BuilderSocketEvent.TableChange, { id, table: null })
     gridSocket.emitTableDeletion(id)
   }
 
   emitDatasourceUpdate(ctx: any, datasource: Datasource) {
-    this.io
-      .in(ctx.appId)
-      .emit("datasource-change", { id: datasource._id, datasource })
+    this.io.in(ctx.appId).emit(BuilderSocketEvent.DatasourceChange, {
+      id: datasource._id,
+      datasource,
+    })
   }
 
   emitDatasourceDeletion(ctx: any, id: string) {
-    this.io.in(ctx.appId).emit("datasource-change", { id, datasource: null })
+    this.io
+      .in(ctx.appId)
+      .emit(BuilderSocketEvent.DatasourceChange, { id, datasource: null })
   }
 }

package/src/websockets/client.ts

@@ -1,10 +1,10 @@
-import Socket from "./websocket"
+import { BaseSocket } from "./websocket"
 import authorized from "../middleware/authorized"
 import http from "http"
 import Koa from "koa"
 import { permissions } from "@budibase/backend-core"
 
-export default class ClientAppWebsocket extends Socket {
+export default class ClientAppWebsocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
     super(app, server, "/socket/client", [authorized(permissions.BUILDER)])
   }

package/src/websockets/grid.ts

@@ -1,73 +1,51 @@
 import authorized from "../middleware/authorized"
-import Socket from "./websocket"
+import { BaseSocket } from "./websocket"
 import { permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
 import { getTableId } from "../api/controllers/row/utils"
 import { Row, Table } from "@budibase/types"
+import { Socket } from "socket.io"
+import { GridSocketEvent } from "@budibase/shared-core"
 
-export default class GridSocket extends Socket {
+export default class GridSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
     super(app, server, "/socket/grid", [authorized(permissions.BUILDER)])
+  }
 
-    this.io.on("connection", socket => {
-      const user = socket.data.user
-
-      // Socket state
-      let currentRoom: string
-
-      // Initial identification of connected spreadsheet
-      socket.on("select-table", async (tableId, callback) => {
-        // Leave current room
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", user)
-          socket.leave(currentRoom)
-        }
-
-        // Join new room
-        currentRoom = tableId
-        socket.join(currentRoom)
-        socket.to(currentRoom).emit("user-update", user)
-
-        // Reply with all users in current room
-        const sockets = await this.io.in(currentRoom).fetchSockets()
-        callback({
-          users: sockets.map(socket => socket.data.user),
-        })
-      })
+  async onConnect(socket: Socket) {
+    // Initial identification of connected spreadsheet
+    socket.on(GridSocketEvent.SelectTable, async (tableId, callback) => {
+      await this.joinRoom(socket, tableId)
 
-      // Handle users selecting a new cell
-      socket.on("select-cell", cellId => {
-        socket.data.user.focusedCellId = cellId
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-update", user)
-        }
-      })
+      // Reply with all users in current room
+      const sessions = await this.getRoomSessions(tableId)
+      callback({ users: sessions })
+    })
 
-      // Disconnection cleanup
-      socket.on("disconnect", () => {
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", user)
-        }
-      })
+    // Handle users selecting a new cell
+    socket.on(GridSocketEvent.SelectCell, cellId => {
+      this.updateUser(socket, { focusedCellId: cellId })
     })
   }
 
   emitRowUpdate(ctx: any, row: Row) {
     const tableId = getTableId(ctx)
-    this.io.in(tableId).emit("row-change", { id: row._id, row })
+    this.io.in(tableId).emit(GridSocketEvent.RowChange, { id: row._id, row })
   }
 
   emitRowDeletion(ctx: any, id: string) {
     const tableId = getTableId(ctx)
-    this.io.in(tableId).emit("row-change", { id, row: null })
+    this.io.in(tableId).emit(GridSocketEvent.RowChange, { id, row: null })
   }
 
   emitTableUpdate(table: Table) {
-    this.io.in(table._id!).emit("table-change", { id: table._id, table })
+    this.io
+      .in(table._id!)
+      .emit(GridSocketEvent.TableChange, { id: table._id, table })
   }
 
   emitTableDeletion(id: string) {
-    this.io.in(id).emit("table-change", { id, table: null })
+    this.io.in(id).emit(GridSocketEvent.TableChange, { id, table: null })
   }
 }

package/src/websockets/websocket.ts

@@ -3,14 +3,18 @@ import http from "http"
 import Koa from "koa"
 import Cookies from "cookies"
 import { userAgent } from "koa-useragent"
-import { auth } from "@budibase/backend-core"
+import { auth, redis } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
 import { createAdapter } from "@socket.io/redis-adapter"
+import { Socket } from "socket.io"
 import { getSocketPubSubClients } from "../utilities/redis"
-import uuid from "uuid"
+import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core"
+import { SocketSession } from "@budibase/types"
 
-export default class Socket {
+export class BaseSocket {
   io: Server
+  path: string
+  redisClient?: redis.Client
 
   constructor(
     app: Koa,
@@ -18,6 +22,7 @@ export default class Socket {
     path: string = "/",
     additionalMiddlewares?: any[]
   ) {
+    this.path = path
     this.io = new Server(server, {
       path,
     })
@@ -65,18 +70,14 @@ export default class Socket {
               // Middlewares are finished
               // Extract some data from our enriched koa context to persist
               // as metadata for the socket
-              // Add user info, including a deterministic color and label
               const { _id, email, firstName, lastName } = ctx.user
-              socket.data.user = {
+              socket.data = {
                 _id,
                 email,
                 firstName,
                 lastName,
-                sessionId: uuid.v4(),
+                sessionId: socket.id,
               }
-
-              // Add app ID to help split sockets into rooms
-              socket.data.appId = ctx.appId
               next()
             }
           })
@@ -86,10 +87,184 @@ export default class Socket {
       }
     })
 
-    // Instantiate redis adapter
+    // Initialise redis before handling connections
+    this.initialise().then(() => {
+      this.io.on("connection", async socket => {
+        // Add built in handler for heartbeats
+        socket.on(SocketEvent.Heartbeat, async () => {
+          console.log(socket.data.email, "heartbeat received")
+          await this.extendSessionTTL(socket.data.sessionId)
+        })
+
+        // Add early disconnection handler to clean up and leave room
+        socket.on("disconnect", async () => {
+          // Run any custom disconnection logic before we leave the room,
+          // so that we have access to their room etc before disconnection
+          await this.onDisconnect(socket)
+
+          // Leave the current room when the user disconnects if we're in one
+          await this.leaveRoom(socket)
+        })
+
+        // Add handlers for this socket
+        await this.onConnect(socket)
+      })
+    })
+  }
+
+  async initialise() {
+    // Instantiate redis adapter.
+    // We use a fully qualified key name here as this bypasses the normal
+    // redis client#s key prefixing.
     const { pub, sub } = getSocketPubSubClients()
-    const opts = { key: `socket.io-${path}` }
+    const opts = {
+      key: `${redis.utils.Databases.SOCKET_IO}-${this.path}-pubsub`,
+    }
     this.io.adapter(createAdapter(pub, sub, opts))
+
+    // Fetch redis client
+    this.redisClient = await redis.clients.getSocketClient()
+  }
+
+  // Gets the redis key for a certain session ID
+  getSessionKey(sessionId: string) {
+    return `${this.path}-session:${sessionId}`
+  }
+
+  // Gets the redis key for certain room name
+  getRoomKey(room: string) {
+    return `${this.path}-room:${room}`
+  }
+
+  async extendSessionTTL(sessionId: string) {
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.setExpiry(key, SocketSessionTTL)
+  }
+
+  // Gets an array of all redis keys of users inside a certain room
+  async getRoomSessionIds(room: string): Promise<string[]> {
+    const keys = await this.redisClient?.get(this.getRoomKey(room))
+    return keys || []
+  }
+
+  // Sets the list of redis keys for users inside a certain room.
+  // There is no TTL on the actual room key map itself.
+  async setRoomSessionIds(room: string, ids: string[]) {
+    await this.redisClient?.store(this.getRoomKey(room), ids)
+  }
+
+  // Gets a list of all users inside a certain room
+  async getRoomSessions(room?: string): Promise<SocketSession[]> {
+    if (room) {
+      const sessionIds = await this.getRoomSessionIds(room)
+      const keys = sessionIds.map(this.getSessionKey.bind(this))
+      const sessions = await this.redisClient?.bulkGet(keys)
+      return Object.values(sessions || {})
+    } else {
+      return []
+    }
+  }
+
+  // Detects keys which have been pruned from redis due to TTL expiry in a certain
+  // room and broadcasts disconnection messages to ensure clients are aware
+  async pruneRoom(room: string) {
+    const sessionIds = await this.getRoomSessionIds(room)
+    const sessionsExist = await Promise.all(
+      sessionIds.map(id => this.redisClient?.exists(this.getSessionKey(id)))
+    )
+    const prunedSessionIds = sessionIds.filter((id, idx) => {
+      if (!sessionsExist[idx]) {
+        this.io.to(room).emit(SocketEvent.UserDisconnect, sessionIds[idx])
+        return false
+      }
+      return true
+    })
+
+    // Store new pruned keys
+    await this.setRoomSessionIds(room, prunedSessionIds)
+  }
+
+  // Adds a user to a certain room
+  async joinRoom(socket: Socket, room: string) {
+    if (!room) {
+      return
+    }
+    // Prune room before joining
+    await this.pruneRoom(room)
+
+    // Check if we're already in a room, as we'll need to leave if we are before we
+    // can join a different room
+    const oldRoom = socket.data.room
+    if (oldRoom && oldRoom !== room) {
+      await this.leaveRoom(socket)
+    }
+
+    // Join new room
+    if (!oldRoom || oldRoom !== room) {
+      socket.join(room)
+      socket.data.room = room
+    }
+
+    // Store in redis
+    // @ts-ignore
+    let user: SocketSession = socket.data
+    const { sessionId } = user
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.store(key, user, SocketSessionTTL)
+    const sessionIds = await this.getRoomSessionIds(room)
+    if (!sessionIds.includes(sessionId)) {
+      await this.setRoomSessionIds(room, [...sessionIds, sessionId])
+    }
+
+    // Notify other users
+    socket.to(room).emit(SocketEvent.UserUpdate, user)
+  }
+
+  // Disconnects a socket from its current room
+  async leaveRoom(socket: Socket) {
+    // @ts-ignore
+    let user: SocketSession = socket.data
+    const { room, sessionId } = user
+    if (!room) {
+      return
+    }
+
+    // Leave room
+    socket.leave(room)
+    socket.data.room = undefined
+
+    // Delete from redis
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.delete(key)
+    const sessionIds = await this.getRoomSessionIds(room)
+    await this.setRoomSessionIds(
+      room,
+      sessionIds.filter(id => id !== sessionId)
+    )
+
+    // Notify other users
+    socket.to(room).emit(SocketEvent.UserDisconnect, sessionId)
+  }
+
+  // Updates a connected user's metadata, assuming a room change is not required.
+  async updateUser(socket: Socket, patch: Object) {
+    socket.data = {
+      ...socket.data,
+      ...patch,
+    }
+
+    // If we're in a room, notify others of this change and update redis
+    if (socket.data.room) {
+      await this.joinRoom(socket, socket.data.room)
+    }
+  }
+
+  async onConnect(socket: Socket) {
+    // Override
+  }
+
+  async onDisconnect(socket: Socket) {
+    // Override
   }
 
   // Emit an event to all sockets