@prsm/realtime 1.0.0

package/src/devtools/client/vite.config.js

@@ -0,0 +1,17 @@
+import { defineConfig } from 'vite'
+import vue from '@vitejs/plugin-vue'
+
+export default defineConfig({
+  plugins: [vue()],
+  base: './',
+  build: {
+    outDir: 'dist',
+    emptyOutDir: true
+  },
+  server: {
+    port: 4173,
+    proxy: {
+      '/api': 'http://localhost:3399/devtools'
+    }
+  }
+})

package/src/devtools/demo/server.js

@@ -0,0 +1,144 @@
+import express from "express"
+import http from "node:http"
+import { RealtimeServer } from "../../../src/index.js"
+import { RealtimeClient } from "../../../src/client/index.js"
+import { createDevtools } from "../index.js"
+
+const PORT = 3399
+
+const app = express()
+const httpServer = http.createServer(app)
+
+const mesh = new RealtimeServer({
+  redis: { host: "127.0.0.1", port: 6379, db: 15 }
+})
+
+mesh.exposeChannel(/^chat:.*/)
+mesh.exposeChannel("notifications")
+mesh.exposeRecord(/^user:.*/)
+mesh.exposeWritableRecord(/^doc:.*/)
+mesh.trackPresence(/^room:.*/)
+
+mesh.exposeCollection("users:online", async () => {
+  return await mesh.listRecordsMatching("user:*")
+})
+
+mesh.exposeCollection("docs:shared", async () => {
+  return await mesh.listRecordsMatching("doc:*")
+})
+
+mesh.exposeCollection(/^users-by-role:.*/, async (_conn, collectionId) => {
+  const role = collectionId.split(":")[1]
+  const users = await mesh.listRecordsMatching("user:*")
+  return users.filter(u => u.role === role)
+})
+
+mesh.exposeCommand("echo", (ctx) => ctx.payload)
+
+app.use("/devtools", createDevtools(mesh))
+
+const redis = mesh.redisManager.redis
+await redis.flushdb()
+
+await mesh.attach(httpServer, { port: PORT })
+
+const names = ["alice", "bob", "charlie", "diana", "eve"]
+const statuses = ["online", "away", "busy", "idle"]
+const rooms = ["room:lobby", "room:general", "room:random"]
+const channels = ["chat:general", "chat:random", "notifications"]
+
+const clients = []
+
+for (let i = 0; i < names.length; i++) {
+  const client = new RealtimeClient(`ws://localhost:${PORT}`)
+  await client.connect()
+
+  await mesh.setConnectionMetadata(client.connectionId, {
+    name: names[i],
+    role: i === 0 ? "admin" : "member",
+    joinedAt: new Date().toISOString()
+  })
+
+  const role = i === 0 ? "admin" : "member"
+  await mesh.writeRecord(`user:${names[i]}`, {
+    id: `user:${names[i]}`,
+    name: names[i],
+    email: `${names[i]}@example.com`,
+    role,
+    active: true
+  })
+
+  if (i < 3) {
+    await mesh.writeRecord(`doc:doc-${i}`, {
+      id: `doc:doc-${i}`,
+      title: `Document ${i}`,
+      content: "lorem ipsum",
+      author: names[i]
+    })
+  }
+
+  const clientRooms = rooms.slice(0, 1 + (i % rooms.length))
+  for (const room of clientRooms) {
+    await client.joinRoom(room)
+    await client.subscribePresence(room, () => {})
+  }
+  await new Promise(r => setTimeout(r, 100))
+  for (const room of clientRooms) {
+    await client.publishPresenceState(room, {
+      state: {
+        status: statuses[i % statuses.length],
+        cursor: { x: Math.floor(Math.random() * 800), y: Math.floor(Math.random() * 600) }
+      }
+    })
+  }
+
+  const clientChannels = channels.slice(0, 1 + (i % channels.length))
+  for (const ch of clientChannels) {
+    await client.subscribeChannel(ch, () => {})
+  }
+
+  await client.subscribeRecord(`user:${names[i]}`, () => {})
+  if (i < 3) await client.subscribeRecord(`doc:doc-${i}`, () => {})
+
+  clients.push(client)
+}
+
+await new Promise(r => setTimeout(r, 300))
+
+for (const client of clients) {
+  await client.subscribeCollection("users:online", { onDiff: () => {} })
+}
+
+for (const client of clients.slice(0, 3)) {
+  await client.subscribeCollection("docs:shared", { onDiff: () => {} })
+}
+
+await clients[0].subscribeCollection("users-by-role:admin", { onDiff: () => {} })
+await clients[1].subscribeCollection("users-by-role:member", { onDiff: () => {} })
+await clients[2].subscribeCollection("users-by-role:admin", { onDiff: () => {} })
+
+setInterval(async () => {
+  const i = Math.floor(Math.random() * clients.length)
+  const room = rooms[Math.floor(Math.random() * rooms.length)]
+  const clientRooms = rooms.slice(0, 1 + (i % rooms.length))
+  if (!clientRooms.includes(room)) return
+
+  await clients[i].publishPresenceState(room, {
+    state: {
+      status: statuses[Math.floor(Math.random() * statuses.length)],
+      cursor: { x: Math.floor(Math.random() * 800), y: Math.floor(Math.random() * 600) }
+    }
+  })
+}, 3000)
+
+setInterval(async () => {
+  const ch = channels[Math.floor(Math.random() * channels.length)]
+  await mesh.writeChannel(ch, {
+    from: names[Math.floor(Math.random() * names.length)],
+    text: `message at ${new Date().toISOString()}`,
+  }, 20)
+}, 5000)
+
+console.log(`mesh devtools demo running on http://localhost:${PORT}/devtools`)
+console.log(`${names.length} clients connected, ${rooms.length} rooms, ${channels.length} channels`)
+console.log(`vite dev server: cd packages/devtools/client && npm run dev`)

package/src/devtools/index.js

@@ -0,0 +1,186 @@
+import express from "express"
+import { fileURLToPath } from "node:url"
+import { dirname, join } from "node:path"
+import { existsSync } from "node:fs"
+
+function patternToString(p) {
+  if (typeof p === "string") return p
+  if (p instanceof RegExp) return p.toString()
+  return String(p)
+}
+
+export function createDevtools(server) {
+  const router = express.Router()
+  const __dirname = dirname(fileURLToPath(import.meta.url))
+  const distPath = join(__dirname, "client", "dist")
+
+  router.get("/api/state", async (_req, res) => {
+    try {
+      const connIds = await server.connectionManager.getAllConnectionIds()
+      const allMeta = await server.connectionManager._getMetadataForConnectionIds(connIds)
+      const localConns = server.connectionManager.getLocalConnections()
+      const localMap = {}
+      for (const conn of localConns) {
+        localMap[conn.id] = conn
+      }
+
+      const connections = allMeta.map(({ id, metadata }) => {
+        const local = localMap[id]
+        return {
+          id,
+          metadata,
+          local: !!local,
+          latency: local?.latency?.ms ?? null,
+          alive: local?.alive ?? null,
+          remoteAddress: local?.remoteAddress ?? null
+        }
+      })
+
+      const roomNames = await server.roomManager.getAllRooms()
+      const rooms = []
+      for (const name of roomNames) {
+        const members = await server.roomManager.getRoomConnectionIds(name)
+        const statesMap = await server.presenceManager.getAllPresenceStates(name)
+        const presence = {}
+        statesMap.forEach((state, connId) => { presence[connId] = state })
+        rooms.push({ name, members, presence })
+      }
+
+      const channels = {}
+      for (const [channel, subscribers] of Object.entries(server.channelManager.channelSubscriptions)) {
+        if (channel.startsWith("mesh:presence:updates:")) continue
+        channels[channel] = [...subscribers].map(c => c.id)
+      }
+
+      const collections = {}
+      server.collectionManager.collectionSubscriptions.forEach((subs, collId) => {
+        const subscribers = {}
+        subs.forEach((info, connId) => { subscribers[connId] = info })
+        collections[collId] = { subscribers }
+      })
+
+      const records = {}
+      server.recordSubscriptionManager.recordSubscriptions.forEach((subs, recordId) => {
+        const subscribers = {}
+        subs.forEach((mode, connId) => { subscribers[connId] = mode })
+        records[recordId] = { subscribers }
+      })
+
+      const exposed = {
+        channels: server.channelManager.exposedChannels.map(patternToString),
+        records: server.recordSubscriptionManager.exposedRecords.map(patternToString),
+        writableRecords: server.recordSubscriptionManager.exposedWritableRecords.map(patternToString),
+        collections: server.collectionManager.exposedCollections.map(e => patternToString(e.pattern)),
+        presence: server.presenceManager.trackedRooms.map(patternToString),
+        commands: server.commandManager.commands
+          ? Object.keys(server.commandManager.commands).filter(c => !c.startsWith("mesh/"))
+          : []
+      }
+
+      res.json({
+        instanceId: server.instanceId,
+        connections,
+        rooms,
+        channels,
+        collections,
+        records,
+        exposed
+      })
+    } catch (err) {
+      res.status(500).json({ error: err.message })
+    }
+  })
+
+  router.get("/api/connection/:id", async (req, res) => {
+    try {
+      const { id } = req.params
+      const metadata = await server.connectionManager.getMetadata(id)
+      const rooms = await server.roomManager.getRoomsForConnection(id)
+
+      const presence = {}
+      for (const room of rooms) {
+        const state = await server.presenceManager.getPresenceState(id, room)
+        if (state) presence[room] = state
+      }
+
+      const channels = []
+      for (const [channel, subscribers] of Object.entries(server.channelManager.channelSubscriptions)) {
+        if (channel.startsWith("mesh:presence:updates:")) continue
+        for (const conn of subscribers) {
+          if (conn.id === id) { channels.push(channel); break }
+        }
+      }
+
+      const collections = []
+      server.collectionManager.collectionSubscriptions.forEach((subs, collId) => {
+        if (subs.has(id)) collections.push({ id: collId, ...subs.get(id) })
+      })
+
+      const records = []
+      server.recordSubscriptionManager.recordSubscriptions.forEach((subs, recordId) => {
+        if (subs.has(id)) records.push({ id: recordId, mode: subs.get(id) })
+      })
+
+      const local = server.connectionManager.getLocalConnection(id)
+
+      res.json({
+        id,
+        metadata,
+        rooms,
+        presence,
+        channels,
+        collections,
+        records,
+        local: !!local,
+        latency: local?.latency?.ms ?? null,
+        alive: local?.alive ?? null,
+        remoteAddress: local?.remoteAddress ?? null
+      })
+    } catch (err) {
+      res.status(500).json({ error: err.message })
+    }
+  })
+
+  router.get("/api/room/:name", async (req, res) => {
+    try {
+      const { name } = req.params
+      const membersWithMeta = await server.getRoomMembersWithMetadata(name)
+      const statesMap = await server.presenceManager.getAllPresenceStates(name)
+      const presence = {}
+      statesMap.forEach((state, connId) => { presence[connId] = state })
+      res.json({ name, members: membersWithMeta, presence })
+    } catch (err) {
+      res.status(500).json({ error: err.message })
+    }
+  })
+
+  router.get("/api/collection/:id/records", async (req, res) => {
+    try {
+      const collId = req.params.id
+      const connId = req.query.connId
+      if (!connId) return res.status(400).json({ error: "connId query param required" })
+
+      const raw = await server.redisManager.redis.get(`mesh:collection:${collId}:${connId}`)
+      if (!raw) return res.json({ recordIds: [], records: [] })
+
+      const recordIds = JSON.parse(raw)
+      const records = []
+      for (const rid of recordIds) {
+        const data = await server.recordManager.getRecord(rid)
+        records.push({ id: rid, data })
+      }
+      res.json({ recordIds, records })
+    } catch (err) {
+      res.status(500).json({ error: err.message })
+    }
+  })
+
+  if (existsSync(distPath)) {
+    router.use(express.static(distPath))
+    router.get("/{*splat}", (_req, res) => {
+      res.sendFile(join(distPath, "index.html"))
+    })
+  }
+
+  return router
+}

package/src/index.js

@@ -0,0 +1,9 @@
+export { RealtimeServer } from "./server/server.js"
+export { Context } from "./server/context.js"
+export { Connection } from "./server/connection.js"
+export { RoomManager } from "./server/managers/rooms.js"
+export { RecordManager } from "./server/managers/records.js"
+export { ConnectionManager } from "./server/managers/connections.js"
+export { PresenceManager } from "./server/managers/presence.js"
+export { PersistenceManager } from "./server/managers/persistence.js"
+export { MessageStream } from "./server/message-stream.js"

package/src/server/connection.js

@@ -0,0 +1,116 @@
+import { EventEmitter } from "node:events"
+import { parseCommand, stringifyCommand, serverLogger, Status } from "../shared/index.js"
+import { generateConnectionId } from "./utils/ids.js"
+
+export class Connection extends EventEmitter {
+  constructor(socket, req, options, server) {
+    super()
+    this.socket = socket
+    this.id = generateConnectionId()
+    this.alive = true
+    this.missedPongs = 0
+    this.remoteAddress = req.socket.remoteAddress
+    this.connectionOptions = options
+    this.server = server
+    this.status = Status.ONLINE
+    this.latency = { start: 0, end: 0, ms: 0, interval: null }
+    this.ping = { interval: null }
+
+    this._applyListeners()
+    this._startIntervals()
+  }
+
+  get isDead() {
+    return !this.socket || this.socket.readyState !== this.socket.constructor.OPEN
+  }
+
+  _startIntervals() {
+    this.latency.interval = setInterval(() => {
+      if (!this.alive) return
+      if (typeof this.latency.ms === "number") {
+        this.send({ command: "latency", payload: this.latency.ms })
+      }
+      this.latency.start = Date.now()
+      this.send({ command: "latency:request", payload: {} })
+    }, this.connectionOptions.latencyInterval)
+
+    this.ping.interval = setInterval(() => {
+      if (!this.alive) {
+        this.missedPongs++
+        const maxMissedPongs = this.connectionOptions.maxMissedPongs ?? 1
+        if (this.missedPongs > maxMissedPongs) {
+          serverLogger.info(`Closing connection (${this.id}) due to missed pongs`)
+          this.close()
+          this.server.cleanupConnection(this)
+          return
+        }
+      } else {
+        this.missedPongs = 0
+      }
+      this.alive = false
+      this.send({ command: "ping", payload: {} })
+    }, this.connectionOptions.pingInterval)
+  }
+
+  stopIntervals() {
+    clearInterval(this.latency.interval)
+    clearInterval(this.ping.interval)
+  }
+
+  _applyListeners() {
+    this.socket.on("close", () => {
+      serverLogger.info("Client's socket closed:", this.id)
+      this.status = Status.OFFLINE
+      this.emit("close")
+    })
+
+    this.socket.on("error", (error) => {
+      this.emit("error", error)
+    })
+
+    this.socket.on("message", (data) => {
+      try {
+        const command = parseCommand(data.toString())
+        if (command.command === "latency:response") {
+          this.latency.end = Date.now()
+          this.latency.ms = this.latency.end - this.latency.start
+          return
+        } else if (command.command === "pong") {
+          this.alive = true
+          this.missedPongs = 0
+          this.emit("pong", this.id)
+          return
+        }
+        this.emit("message", data)
+      } catch (error) {
+        this.emit("error", error)
+      }
+    })
+  }
+
+  send(cmd) {
+    if (this.isDead) return false
+    try {
+      this.socket.send(stringifyCommand(cmd))
+      return true
+    } catch (error) {
+      this.emit("error", error)
+      return false
+    }
+  }
+
+  async close() {
+    if (this.isDead) return false
+    try {
+      await new Promise((resolve, reject) => {
+        this.socket.once("close", resolve)
+        this.socket.once("error", reject)
+        this.socket.close()
+      })
+      return true
+    } catch (error) {
+      this.emit("error", error)
+      return false
+    }
+  }
+}

package/src/server/context.js

@@ -0,0 +1,22 @@
+export class Context {
+  constructor(server, command, connection, payload) {
+    this.server = server
+    this.command = command
+    this.connection = connection
+    this.payload = payload
+  }
+
+  /** @returns {Promise<any>} */
+  getMetadata() {
+    return this.server.connectionManager.getMetadata(this.connection)
+  }
+
+  /**
+   * @param {any} metadata
+   * @param {{strategy?: 'replace' | 'merge' | 'deepMerge'}} [options]
+   * @returns {Promise<void>}
+   */
+  setMetadata(metadata, options) {
+    return this.server.connectionManager.setMetadata(this.connection, metadata, options)
+  }
+}

package/src/server/managers/broadcast.js

@@ -0,0 +1,94 @@
+export class BroadcastManager {
+  constructor({ connectionManager, roomManager, instanceId, pubClient, getPubSubChannel, emitError }) {
+    this.connectionManager = connectionManager
+    this.roomManager = roomManager
+    this.instanceId = instanceId
+    this.pubClient = pubClient
+    this.getPubSubChannel = getPubSubChannel
+    this.emitError = emitError
+  }
+
+  async sendTo(connectionId, command, payload) {
+    try {
+      await this._publishOrSend([connectionId], { command, payload })
+    } catch (err) {
+      this.emitError(new Error(`Failed to send command "${command}" to ${connectionId}: ${err}`))
+    }
+  }
+
+  async broadcast(command, payload, connections) {
+    const cmd = { command, payload }
+    try {
+      if (connections) {
+        const allConnectionIds = connections.map(({ id }) => id)
+        const connectionIds = await this.connectionManager.getAllConnectionIds()
+        const filteredIds = allConnectionIds.filter((id) => connectionIds.includes(id))
+        await this._publishOrSend(filteredIds, cmd)
+      } else {
+        const allConnectionIds = await this.connectionManager.getAllConnectionIds()
+        await this._publishOrSend(allConnectionIds, cmd)
+      }
+    } catch (err) {
+      this.emitError(new Error(`Failed to broadcast command "${command}": ${err}`))
+    }
+  }
+
+  async broadcastRoom(roomName, command, payload) {
+    const connectionIds = await this.roomManager.getRoomConnectionIds(roomName)
+    try {
+      await this._publishOrSend(connectionIds, { command, payload })
+    } catch (err) {
+      this.emitError(new Error(`Failed to broadcast command "${command}": ${err}`))
+    }
+  }
+
+  async broadcastExclude(command, payload, exclude) {
+    const excludedIds = new Set((Array.isArray(exclude) ? exclude : [exclude]).map(({ id }) => id))
+    try {
+      const connectionIds = (await this.connectionManager.getAllConnectionIds()).filter((id) => !excludedIds.has(id))
+      await this._publishOrSend(connectionIds, { command, payload })
+    } catch (err) {
+      this.emitError(new Error(`Failed to broadcast command "${command}": ${err}`))
+    }
+  }
+
+  async broadcastRoomExclude(roomName, command, payload, exclude) {
+    const excludedIds = new Set((Array.isArray(exclude) ? exclude : [exclude]).map(({ id }) => id))
+    try {
+      const connectionIds = (await this.roomManager.getRoomConnectionIds(roomName)).filter((id) => !excludedIds.has(id))
+      await this._publishOrSend(connectionIds, { command, payload })
+    } catch (err) {
+      this.emitError(new Error(`Failed to broadcast command "${command}": ${err}`))
+    }
+  }
+
+  async _publishOrSend(connectionIds, command) {
+    if (connectionIds.length === 0) return
+    const connectionInstanceMapping = await this.connectionManager.getInstanceIdsForConnections(connectionIds)
+    const instanceMap = {}
+    for (const connectionId of connectionIds) {
+      const instanceId = connectionInstanceMapping[connectionId]
+      if (instanceId) {
+        if (!instanceMap[instanceId]) instanceMap[instanceId] = []
+        instanceMap[instanceId].push(connectionId)
+      }
+    }
+    for (const [instanceId, targetConnectionIds] of Object.entries(instanceMap)) {
+      if (targetConnectionIds.length === 0) continue
+      if (instanceId === this.instanceId) {
+        targetConnectionIds.forEach((connectionId) => {
+          const connection = this.connectionManager.getLocalConnection(connectionId)
+          if (connection && !connection.isDead) connection.send(command)
+        })
+      } else {
+        const messagePayload = { targetConnectionIds, command }
+        const message = JSON.stringify(messagePayload)
+        try {
+          await this.pubClient.publish(this.getPubSubChannel(instanceId), message)
+        } catch (err) {
+          this.emitError(new Error(`Failed to publish command "${command.command}": ${err}`))
+        }
+      }
+    }
+  }
+}