@budibase/server 2.6.23 → 2.6.24-alpha.1

package/src/utilities/redis.ts

@@ -1,12 +1,19 @@
-import { redis } from "@budibase/backend-core"
+import { redis, RedisClient } from "@budibase/backend-core"
 import { getGlobalIDFromUserMetadataID } from "../db/utils"
 import { ContextUser } from "@budibase/types"
+import env from "../environment"
 
 const APP_DEV_LOCK_SECONDS = 600
 const AUTOMATION_TEST_FLAG_SECONDS = 60
-let devAppClient: any, debounceClient: any, flagClient: any
+let devAppClient: RedisClient,
+  debounceClient: RedisClient,
+  flagClient: RedisClient
 
-// we init this as we want to keep the connection open all the time
+// We need to maintain a duplicate client for socket.io pub/sub
+let socketClient: RedisClient
+let socketSubClient: any
+
+// We init this as we want to keep the connection open all the time
 // reduces the performance hit
 export async function init() {
   devAppClient = new redis.Client(redis.utils.Databases.DEV_LOCKS)
@@ -15,12 +22,20 @@ export async function init() {
   await devAppClient.init()
   await debounceClient.init()
   await flagClient.init()
+
+  // Duplicate the socket client for pub/sub
+  socketClient = await redis.clients.getSocketClient()
+  if (!env.isTest()) {
+    socketSubClient = socketClient.getClient().duplicate()
+  }
 }
 
 export async function shutdown() {
+  console.log("REDIS SHUTDOWN")
   if (devAppClient) await devAppClient.finish()
   if (debounceClient) await debounceClient.finish()
   if (flagClient) await flagClient.finish()
+  if (socketSubClient) socketSubClient.disconnect()
   // shutdown core clients
   await redis.clients.shutdown()
   console.log("Redis shutdown")
@@ -86,3 +101,10 @@ export async function checkTestFlag(id: string) {
 export async function clearTestFlag(id: string) {
   await devAppClient.delete(id)
 }
+
+export function getSocketPubSubClients() {
+  return {
+    pub: socketClient.getClient(),
+    sub: socketSubClient,
+  }
+}

package/src/utilities/rowProcessor/index.ts

@@ -216,7 +216,10 @@ export async function outputProcessing(
     }
   }
   if (opts.squash) {
-    enriched = await linkRows.squashLinksToPrimaryDisplay(table, enriched)
+    enriched = (await linkRows.squashLinksToPrimaryDisplay(
+      table,
+      enriched
+    )) as Row[]
   }
   return wasArray ? enriched : enriched[0]
 }

package/src/utilities/rowProcessor/map.ts

@@ -12,7 +12,7 @@ const parseArrayString = value => {
       result = JSON.parse(value.replace(/'/g, '"'))
       return result
     } catch (e) {
-      logging.logAlert("Could not parse row value", e)
+      logging.logWarn("Could not parse row value", e)
     }
   }
   return value

package/src/websockets/builder.ts

@@ -0,0 +1,95 @@
+import authorized from "../middleware/authorized"
+import { BaseSocket } from "./websocket"
+import { permissions } from "@budibase/backend-core"
+import http from "http"
+import Koa from "koa"
+import { Datasource, Table, SocketSession, ContextUser } from "@budibase/types"
+import { gridSocket } from "./index"
+import { clearLock, updateLock } from "../utilities/redis"
+import { Socket } from "socket.io"
+import { BuilderSocketEvent } from "@budibase/shared-core"
+
+export default class BuilderSocket extends BaseSocket {
+  constructor(app: Koa, server: http.Server) {
+    super(app, server, "/socket/builder", [authorized(permissions.BUILDER)])
+  }
+
+  async onConnect(socket?: Socket) {
+    // Initial identification of selected app
+    socket?.on(BuilderSocketEvent.SelectApp, async ({ appId }, callback) => {
+      await this.joinRoom(socket, appId)
+
+      // Reply with all users in current room
+      const sessions = await this.getRoomSessions(appId)
+      callback({ users: sessions })
+    })
+  }
+
+  async onDisconnect(socket: Socket) {
+    // Remove app lock from this user if they have no other connections,
+    // and transfer it to someone else if possible
+    try {
+      // @ts-ignore
+      const session: SocketSession = socket.data
+      const { _id, sessionId, room } = session
+      const sessions = await this.getRoomSessions(room)
+      const hasOtherSession = sessions.some(otherSession => {
+        return _id === otherSession._id && sessionId !== otherSession.sessionId
+      })
+      if (!hasOtherSession && room) {
+        // Clear the lock from this user since they had no other sessions
+        // @ts-ignore
+        const user: ContextUser = { _id: socket.data._id }
+        await clearLock(room, user)
+
+        // Transfer lock ownership to the next oldest user
+        let otherSessions = sessions.filter(x => x._id !== _id).slice()
+        otherSessions.sort((a, b) => {
+          return a.connectedAt < b.connectedAt ? -1 : 1
+        })
+        const nextSession = otherSessions[0]
+        if (nextSession) {
+          const { _id, email, firstName, lastName } = nextSession
+          // @ts-ignore
+          const nextUser: ContextUser = { _id, email, firstName, lastName }
+          await updateLock(room, nextUser)
+          this.io.to(room).emit(BuilderSocketEvent.LockTransfer, {
+            userId: _id,
+          })
+        }
+      }
+    } catch (e) {
+      // This is fine, just means this user didn't hold the lock
+    }
+  }
+
+  emitTableUpdate(ctx: any, table: Table) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, {
+      id: table._id,
+      table,
+    })
+    gridSocket?.emitTableUpdate(ctx, table)
+  }
+
+  emitTableDeletion(ctx: any, id: string) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.TableChange, {
+      id,
+      table: null,
+    })
+    gridSocket?.emitTableDeletion(ctx, id)
+  }
+
+  emitDatasourceUpdate(ctx: any, datasource: Datasource) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, {
+      id: datasource._id,
+      datasource,
+    })
+  }
+
+  emitDatasourceDeletion(ctx: any, id: string) {
+    this.emitToRoom(ctx, ctx.appId, BuilderSocketEvent.DatasourceChange, {
+      id,
+      datasource: null,
+    })
+  }
+}

package/src/websockets/client.ts

@@ -1,10 +1,10 @@
-import Socket from "./websocket"
+import { BaseSocket } from "./websocket"
 import authorized from "../middleware/authorized"
 import http from "http"
 import Koa from "koa"
 import { permissions } from "@budibase/backend-core"
 
-export default class ClientAppWebsocket extends Socket {
+export default class ClientAppWebsocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
     super(app, server, "/socket/client", [authorized(permissions.BUILDER)])
   }

package/src/websockets/grid.ts

@@ -1,55 +1,55 @@
 import authorized from "../middleware/authorized"
-import Socket from "./websocket"
+import { BaseSocket } from "./websocket"
 import { permissions } from "@budibase/backend-core"
 import http from "http"
 import Koa from "koa"
+import { getTableId } from "../api/controllers/row/utils"
+import { Row, Table } from "@budibase/types"
+import { Socket } from "socket.io"
+import { GridSocketEvent } from "@budibase/shared-core"
 
-export default class GridSocket extends Socket {
+export default class GridSocket extends BaseSocket {
   constructor(app: Koa, server: http.Server) {
     super(app, server, "/socket/grid", [authorized(permissions.BUILDER)])
+  }
+
+  async onConnect(socket: Socket) {
+    // Initial identification of connected spreadsheet
+    socket.on(GridSocketEvent.SelectTable, async ({ tableId }, callback) => {
+      await this.joinRoom(socket, tableId)
+
+      // Reply with all users in current room
+      const sessions = await this.getRoomSessions(tableId)
+      callback({ users: sessions })
+    })
+
+    // Handle users selecting a new cell
+    socket.on(GridSocketEvent.SelectCell, ({ cellId }) => {
+      this.updateUser(socket, { focusedCellId: cellId })
+    })
+  }
 
-    this.io.on("connection", socket => {
-      const user = socket.data.user
-      console.log(`Spreadsheet user connected: ${user?.id}`)
-
-      // Socket state
-      let currentRoom: string
-
-      // Initial identification of connected spreadsheet
-      socket.on("select-table", async (tableId, callback) => {
-        // Leave current room
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
-          socket.leave(currentRoom)
-        }
-
-        // Join new room
-        currentRoom = tableId
-        socket.join(currentRoom)
-        socket.to(currentRoom).emit("user-update", socket.data.user)
-
-        // Reply with all users in current room
-        const sockets = await this.io.in(currentRoom).fetchSockets()
-        callback({
-          users: sockets.map(socket => socket.data.user),
-          id: user.id,
-        })
-      })
-
-      // Handle users selecting a new cell
-      socket.on("select-cell", cellId => {
-        socket.data.user.selectedCellId = cellId
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-update", socket.data.user)
-        }
-      })
-
-      // Disconnection cleanup
-      socket.on("disconnect", () => {
-        if (currentRoom) {
-          socket.to(currentRoom).emit("user-disconnect", socket.data.user)
-        }
-      })
+  emitRowUpdate(ctx: any, row: Row) {
+    const tableId = getTableId(ctx)
+    this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, {
+      id: row._id,
+      row,
     })
   }
+
+  emitRowDeletion(ctx: any, id: string) {
+    const tableId = getTableId(ctx)
+    this.emitToRoom(ctx, tableId, GridSocketEvent.RowChange, { id, row: null })
+  }
+
+  emitTableUpdate(ctx: any, table: Table) {
+    this.emitToRoom(ctx, table._id!, GridSocketEvent.TableChange, {
+      id: table._id,
+      table,
+    })
+  }
+
+  emitTableDeletion(ctx: any, id: string) {
+    this.emitToRoom(ctx, id, GridSocketEvent.TableChange, { id, table: null })
+  }
 }

package/src/websockets/index.ts

@@ -1,14 +1,21 @@
 import http from "http"
 import Koa from "koa"
-import GridSocket from "./grid"
 import ClientAppSocket from "./client"
+import GridSocket from "./grid"
+import BuilderSocket from "./builder"
+import env from "../environment"
 
-let clientAppSocket: ClientAppSocket
-let gridSocket: GridSocket
+let clientAppSocket: ClientAppSocket | undefined
+let gridSocket: GridSocket | undefined
+let builderSocket: BuilderSocket | undefined
 
 export const initialise = (app: Koa, server: http.Server) => {
-  clientAppSocket = new ClientAppSocket(app, server)
-  gridSocket = new GridSocket(app, server)
+  // have to remove these for testing until ioredis-mock can be fully removed
+  if (!env.isTest()) {
+    clientAppSocket = new ClientAppSocket(app, server)
+    gridSocket = new GridSocket(app, server)
+    builderSocket = new BuilderSocket(app, server)
+  }
 }
 
-export { clientAppSocket, gridSocket }
+export { clientAppSocket, gridSocket, builderSocket }

package/src/websockets/websocket.ts

@@ -3,18 +3,26 @@ import http from "http"
 import Koa from "koa"
 import Cookies from "cookies"
 import { userAgent } from "koa-useragent"
-import { auth } from "@budibase/backend-core"
+import { auth, Header, redis } from "@budibase/backend-core"
 import currentApp from "../middleware/currentapp"
+import { createAdapter } from "@socket.io/redis-adapter"
+import { Socket } from "socket.io"
+import { getSocketPubSubClients } from "../utilities/redis"
+import { SocketEvent, SocketSessionTTL } from "@budibase/shared-core"
+import { SocketSession } from "@budibase/types"
 
-export default class Socket {
+export class BaseSocket {
   io: Server
+  path: string
+  redisClient?: redis.Client
 
   constructor(
     app: Koa,
     server: http.Server,
-    path: string,
+    path: string = "/",
     additionalMiddlewares?: any[]
   ) {
+    this.path = path
     this.io = new Server(server, {
       path,
     })
@@ -59,12 +67,17 @@ export default class Socket {
         for (let [idx, middleware] of middlewares.entries()) {
           await middleware(ctx, () => {
             if (idx === middlewares.length - 1) {
-              // Middlewares are finished.
+              // Middlewares are finished
               // Extract some data from our enriched koa context to persist
               // as metadata for the socket
-              socket.data.user = {
-                id: ctx.user._id,
-                email: ctx.user.email,
+              const { _id, email, firstName, lastName } = ctx.user
+              socket.data = {
+                _id,
+                email,
+                firstName,
+                lastName,
+                sessionId: socket.id,
+                connectedAt: Date.now(),
               }
               next()
             }
@@ -74,10 +87,201 @@ export default class Socket {
         next(error)
       }
     })
+
+    // Initialise redis before handling connections
+    this.initialise().then(() => {
+      this.io.on("connection", async socket => {
+        // Add built in handler for heartbeats
+        socket.on(SocketEvent.Heartbeat, async () => {
+          await this.extendSessionTTL(socket.data.sessionId)
+        })
+
+        // Add early disconnection handler to clean up and leave room
+        socket.on("disconnect", async () => {
+          // Run any custom disconnection logic before we leave the room,
+          // so that we have access to their room etc before disconnection
+          await this.onDisconnect(socket)
+
+          // Leave the current room when the user disconnects if we're in one
+          await this.leaveRoom(socket)
+        })
+
+        // Add handlers for this socket
+        await this.onConnect(socket)
+      })
+    })
+  }
+
+  async initialise() {
+    // Instantiate redis adapter.
+    // We use a fully qualified key name here as this bypasses the normal
+    // redis client#s key prefixing.
+    const { pub, sub } = getSocketPubSubClients()
+    const opts = {
+      key: `${redis.utils.Databases.SOCKET_IO}-${this.path}-pubsub`,
+    }
+    this.io.adapter(createAdapter(pub, sub, opts))
+
+    // Fetch redis client
+    this.redisClient = await redis.clients.getSocketClient()
+  }
+
+  // Gets the redis key for a certain session ID
+  getSessionKey(sessionId: string) {
+    return `${this.path}-session:${sessionId}`
+  }
+
+  // Gets the redis key for certain room name
+  getRoomKey(room: string) {
+    return `${this.path}-room:${room}`
+  }
+
+  async extendSessionTTL(sessionId: string) {
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.setExpiry(key, SocketSessionTTL)
+  }
+
+  // Gets an array of all redis keys of users inside a certain room
+  async getRoomSessionIds(room: string): Promise<string[]> {
+    const keys = await this.redisClient?.get(this.getRoomKey(room))
+    return keys || []
+  }
+
+  // Sets the list of redis keys for users inside a certain room.
+  // There is no TTL on the actual room key map itself.
+  async setRoomSessionIds(room: string, ids: string[]) {
+    await this.redisClient?.store(this.getRoomKey(room), ids)
+  }
+
+  // Gets a list of all users inside a certain room
+  async getRoomSessions(room?: string): Promise<SocketSession[]> {
+    if (room) {
+      const sessionIds = await this.getRoomSessionIds(room)
+      const keys = sessionIds.map(this.getSessionKey.bind(this))
+      const sessions = await this.redisClient?.bulkGet(keys)
+      return Object.values(sessions || {})
+    } else {
+      return []
+    }
+  }
+
+  // Detects keys which have been pruned from redis due to TTL expiry in a certain
+  // room and broadcasts disconnection messages to ensure clients are aware
+  async pruneRoom(room: string) {
+    const sessionIds = await this.getRoomSessionIds(room)
+    const sessionsExist = await Promise.all(
+      sessionIds.map(id => this.redisClient?.exists(this.getSessionKey(id)))
+    )
+    const prunedSessionIds = sessionIds.filter((id, idx) => {
+      if (!sessionsExist[idx]) {
+        this.io.to(room).emit(SocketEvent.UserDisconnect, {
+          sessionId: sessionIds[idx],
+        })
+        return false
+      }
+      return true
+    })
+
+    // Store new pruned keys
+    await this.setRoomSessionIds(room, prunedSessionIds)
+  }
+
+  // Adds a user to a certain room
+  async joinRoom(socket: Socket, room: string) {
+    if (!room) {
+      return
+    }
+    // Prune room before joining
+    await this.pruneRoom(room)
+
+    // Check if we're already in a room, as we'll need to leave if we are before we
+    // can join a different room
+    const oldRoom = socket.data.room
+    if (oldRoom && oldRoom !== room) {
+      await this.leaveRoom(socket)
+    }
+
+    // Join new room
+    if (!oldRoom || oldRoom !== room) {
+      socket.join(room)
+      socket.data.room = room
+    }
+
+    // Store in redis
+    // @ts-ignore
+    let user: SocketSession = socket.data
+    const { sessionId } = user
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.store(key, user, SocketSessionTTL)
+    const sessionIds = await this.getRoomSessionIds(room)
+    if (!sessionIds.includes(sessionId)) {
+      await this.setRoomSessionIds(room, [...sessionIds, sessionId])
+    }
+
+    // Notify other users
+    socket.to(room).emit(SocketEvent.UserUpdate, {
+      user,
+    })
+  }
+
+  // Disconnects a socket from its current room
+  async leaveRoom(socket: Socket) {
+    // @ts-ignore
+    let user: SocketSession = socket.data
+    const { room, sessionId } = user
+    if (!room) {
+      return
+    }
+
+    // Leave room
+    socket.leave(room)
+    socket.data.room = undefined
+
+    // Delete from redis
+    const key = this.getSessionKey(sessionId)
+    await this.redisClient?.delete(key)
+    const sessionIds = await this.getRoomSessionIds(room)
+    await this.setRoomSessionIds(
+      room,
+      sessionIds.filter(id => id !== sessionId)
+    )
+
+    // Notify other users
+    socket.to(room).emit(SocketEvent.UserDisconnect, { sessionId })
+  }
+
+  // Updates a connected user's metadata, assuming a room change is not required.
+  async updateUser(socket: Socket, patch: Object) {
+    socket.data = {
+      ...socket.data,
+      ...patch,
+    }
+
+    // If we're in a room, notify others of this change and update redis
+    if (socket.data.room) {
+      await this.joinRoom(socket, socket.data.room)
+    }
+  }
+
+  async onConnect(socket: Socket) {
+    // Override
+  }
+
+  async onDisconnect(socket: Socket) {
+    // Override
   }
 
   // Emit an event to all sockets
   emit(event: string, payload: any) {
     this.io.sockets.emit(event, payload)
   }
+
+  // Emit an event to everyone in a room, including metadata of whom
+  // the originator of the request was
+  emitToRoom(ctx: any, room: string, event: string, payload: any) {
+    this.io.in(room).emit(event, {
+      ...payload,
+      apiSessionId: ctx.headers?.[Header.SESSION_ID],
+    })
+  }
 }

package/tsconfig.build.json

@@ -10,7 +10,15 @@
     "incremental": true,
     "types": ["node", "jest"],
     "outDir": "dist/src",
-    "skipLibCheck": true
+    "skipLibCheck": true,
+    "baseUrl": ".",
+    "paths": {
+      "@budibase/types": ["../types/src"],
+      "@budibase/backend-core": ["../backend-core/src"],
+      "@budibase/backend-core/*": ["../backend-core/*"],
+      "@budibase/shared-core": ["../shared-core/src"],
+      "@budibase/pro": ["../pro/packages/pro/src"]
+    }
   },
   "include": ["src/**/*"],
   "exclude": [

package/tsconfig.json

@@ -5,25 +5,12 @@
     "declaration": true,
     "sourceMap": true,
     "baseUrl": ".",
-    "outDir": "dist",
-    "paths": {
-      "@budibase/types": ["../types/src"],
-      "@budibase/backend-core": ["../backend-core/src"],
-      "@budibase/backend-core/*": ["../backend-core/*"],
-      "@budibase/shared-core": ["../shared-core/src"],
-      "@budibase/pro": ["../../../budibase-pro/packages/pro/src"]
-    }
+    "outDir": "dist"
   },
   "ts-node": {
     "require": ["tsconfig-paths/register"],
     "swc": true
   },
-  "references": [
-    { "path": "../types" },
-    { "path": "../backend-core" },
-    { "path": "../shared-core" },
-    { "path": "../../../budibase-pro/packages/pro" }
-  ],
   "include": ["src/**/*", "specs"],
   "exclude": ["node_modules", "dist"]
 }