@rapidmx/meet-plugin 0.5.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (110) hide show
  1. package/README.md +15 -2
  2. package/apps/meet/[token].tsx +3 -2
  3. package/apps/meet/_CallControls.tsx +28 -2
  4. package/apps/meet/_CallView.tsx +23 -2
  5. package/apps/meet/_EffectsPanel.tsx +134 -0
  6. package/apps/meet/_MeetLobby.tsx +22 -0
  7. package/apps/meet/_ParticipantTile.tsx +62 -9
  8. package/apps/meet/_icons.tsx +4 -0
  9. package/apps/meet/_meetApi.ts +6 -0
  10. package/apps/shared/media/filters/VideoFilterProcessor.ts +383 -0
  11. package/apps/shared/media/filters/accessories.ts +262 -0
  12. package/apps/shared/media/filters/backgroundImage.ts +69 -0
  13. package/apps/shared/media/filters/filterTypes.ts +89 -0
  14. package/apps/shared/media/filters/mlModels.ts +174 -0
  15. package/apps/shared/media/filters/pixelEffects.ts +61 -0
  16. package/apps/shared/media/mediaPreferences.ts +105 -0
  17. package/apps/shared/media/useLocalMedia.ts +198 -14
  18. package/apps/shared/relay/AudioPlayer.ts +111 -0
  19. package/apps/shared/relay/AudioSender.ts +194 -0
  20. package/apps/shared/relay/RelayClient.ts +262 -0
  21. package/apps/shared/relay/RelayReceiver.ts +72 -0
  22. package/apps/shared/relay/RelaySender.ts +74 -0
  23. package/apps/shared/relay/RelayTransport.ts +188 -0
  24. package/apps/shared/relay/VideoPlayer.ts +110 -0
  25. package/apps/shared/relay/VideoSender.ts +216 -0
  26. package/apps/shared/relay/audioResume.ts +55 -0
  27. package/apps/shared/relay/frames.ts +275 -0
  28. package/apps/shared/relay/relayEnv.ts +230 -0
  29. package/apps/shared/relay/senderCommon.ts +30 -0
  30. package/apps/shared/webrtc/MeshConnectionManager.ts +158 -6
  31. package/apps/shared/webrtc/realPeerConnection.ts +22 -0
  32. package/apps/shared/webrtc/types.ts +32 -2
  33. package/dist/apps/meet/[token].js +3 -3
  34. package/dist/apps/meet/_CallControls.d.ts +2 -1
  35. package/dist/apps/meet/_CallControls.js +6 -3
  36. package/dist/apps/meet/_CallView.d.ts +4 -1
  37. package/dist/apps/meet/_CallView.js +17 -3
  38. package/dist/apps/meet/_EffectsPanel.d.ts +17 -0
  39. package/dist/apps/meet/_EffectsPanel.js +60 -0
  40. package/dist/apps/meet/_MeetLobby.d.ts +4 -0
  41. package/dist/apps/meet/_MeetLobby.js +8 -1
  42. package/dist/apps/meet/_ParticipantTile.d.ts +7 -1
  43. package/dist/apps/meet/_ParticipantTile.js +20 -3
  44. package/dist/apps/meet/_icons.d.ts +1 -0
  45. package/dist/apps/meet/_icons.js +1 -0
  46. package/dist/apps/meet/_meetApi.d.ts +6 -0
  47. package/dist/apps/shared/media/filters/VideoFilterProcessor.d.ts +75 -0
  48. package/dist/apps/shared/media/filters/VideoFilterProcessor.js +304 -0
  49. package/dist/apps/shared/media/filters/accessories.d.ts +37 -0
  50. package/dist/apps/shared/media/filters/accessories.js +211 -0
  51. package/dist/apps/shared/media/filters/backgroundImage.d.ts +22 -0
  52. package/dist/apps/shared/media/filters/backgroundImage.js +54 -0
  53. package/dist/apps/shared/media/filters/filterTypes.d.ts +40 -0
  54. package/dist/apps/shared/media/filters/filterTypes.js +63 -0
  55. package/dist/apps/shared/media/filters/mlModels.d.ts +54 -0
  56. package/dist/apps/shared/media/filters/mlModels.js +106 -0
  57. package/dist/apps/shared/media/filters/pixelEffects.d.ts +14 -0
  58. package/dist/apps/shared/media/filters/pixelEffects.js +55 -0
  59. package/dist/apps/shared/media/mediaPreferences.d.ts +34 -0
  60. package/dist/apps/shared/media/mediaPreferences.js +93 -0
  61. package/dist/apps/shared/media/useLocalMedia.d.ts +17 -1
  62. package/dist/apps/shared/media/useLocalMedia.js +160 -14
  63. package/dist/apps/shared/relay/AudioPlayer.d.ts +38 -0
  64. package/dist/apps/shared/relay/AudioPlayer.js +103 -0
  65. package/dist/apps/shared/relay/AudioSender.d.ts +50 -0
  66. package/dist/apps/shared/relay/AudioSender.js +172 -0
  67. package/dist/apps/shared/relay/RelayClient.d.ts +65 -0
  68. package/dist/apps/shared/relay/RelayClient.js +236 -0
  69. package/dist/apps/shared/relay/RelayReceiver.d.ts +21 -0
  70. package/dist/apps/shared/relay/RelayReceiver.js +59 -0
  71. package/dist/apps/shared/relay/RelaySender.d.ts +34 -0
  72. package/dist/apps/shared/relay/RelaySender.js +54 -0
  73. package/dist/apps/shared/relay/RelayTransport.d.ts +38 -0
  74. package/dist/apps/shared/relay/RelayTransport.js +142 -0
  75. package/dist/apps/shared/relay/VideoPlayer.d.ts +31 -0
  76. package/dist/apps/shared/relay/VideoPlayer.js +102 -0
  77. package/dist/apps/shared/relay/VideoSender.d.ts +56 -0
  78. package/dist/apps/shared/relay/VideoSender.js +204 -0
  79. package/dist/apps/shared/relay/audioResume.d.ts +12 -0
  80. package/dist/apps/shared/relay/audioResume.js +45 -0
  81. package/dist/apps/shared/relay/frames.d.ts +101 -0
  82. package/dist/apps/shared/relay/frames.js +221 -0
  83. package/dist/apps/shared/relay/relayEnv.d.ts +181 -0
  84. package/dist/apps/shared/relay/relayEnv.js +69 -0
  85. package/dist/apps/shared/relay/senderCommon.d.ts +19 -0
  86. package/dist/apps/shared/relay/senderCommon.js +10 -0
  87. package/dist/apps/shared/webrtc/MeshConnectionManager.d.ts +51 -1
  88. package/dist/apps/shared/webrtc/MeshConnectionManager.js +137 -6
  89. package/dist/apps/shared/webrtc/realPeerConnection.d.ts +7 -0
  90. package/dist/apps/shared/webrtc/realPeerConnection.js +20 -0
  91. package/dist/apps/shared/webrtc/types.d.ts +29 -2
  92. package/dist/lib/index.js +2 -0
  93. package/dist/lib/index.js.map +1 -1
  94. package/dist/lib/routes/BaseVideoMeetingRoute.js +237 -3
  95. package/dist/lib/routes/BaseVideoMeetingRoute.js.map +1 -1
  96. package/dist/lib/util/EffectsUtils.js +21 -0
  97. package/dist/lib/util/EffectsUtils.js.map +1 -0
  98. package/dist/lib/util/IceServerUtils.js +25 -4
  99. package/dist/lib/util/IceServerUtils.js.map +1 -1
  100. package/dist/lib/util/RedisRelayBus.js +241 -0
  101. package/dist/lib/util/RedisRelayBus.js.map +1 -0
  102. package/dist/lib/util/RelayHub.js +405 -0
  103. package/dist/lib/util/RelayHub.js.map +1 -0
  104. package/dist/types/index.d.ts +2 -0
  105. package/dist/types/routes/BaseVideoMeetingRoute.d.ts +79 -0
  106. package/dist/types/util/EffectsUtils.d.ts +10 -0
  107. package/dist/types/util/IceServerUtils.d.ts +12 -1
  108. package/dist/types/util/RedisRelayBus.d.ts +131 -0
  109. package/dist/types/util/RelayHub.d.ts +188 -0
  110. package/package.json +24 -7
@@ -0,0 +1,131 @@
1
+ import type { RelayBus, RelayBusListener } from "./RelayHub.js";
2
+ /** The prefix of the Redis pub/sub channel a room's frames travel on: `videoconf:relay:<roomId>`. */
3
+ export declare const REDIS_RELAY_CHANNEL_PREFIX = "videoconf:relay:";
4
+ /** The length, in bytes, of the random id every `RedisRelayBus` stamps on what it publishes. */
5
+ export declare const REDIS_RELAY_ORIGIN_BYTES = 16;
6
+ /** The least time between two `warn` messages about failing Redis commands; the failures in between are logged at
7
+ * `debug`, so a Redis outage cannot flood the log at the rate media frames arrive. */
8
+ export declare const REDIS_RELAY_WARN_INTERVAL_MS = 30000;
9
+ /** The channel a room's frames are published to. */
10
+ export declare function redisRelayChannel(roomId: string): string;
11
+ /** The logger a `RedisRelayBus` writes to - the framework's logger satisfies it. */
12
+ export interface RedisRelayBusLogger {
13
+ debug(message: string): void;
14
+ info(message: string): void;
15
+ warn(message: string): void;
16
+ error(message: string): void;
17
+ }
18
+ export interface RedisRelayBusOptions {
19
+ /** The `datastores:events` Redis URL. */
20
+ url: string;
21
+ logger: RedisRelayBusLogger;
22
+ }
23
+ /** What the bus counts, for tests and diagnostics. */
24
+ export interface RedisRelayBusStats {
25
+ /** Frames published to Redis. */
26
+ published: number;
27
+ /** Frames not published because the publisher was not connected (or the sender id cannot be encoded). */
28
+ publishSkipped: number;
29
+ /** Redis commands (publish, subscribe, unsubscribe, connect) that failed. */
30
+ failures: number;
31
+ /** Messages received from Redis and handed to local listeners. */
32
+ received: number;
33
+ /** Messages received from Redis and ignored: this bus's own, garbled, or for a room with no local listener. */
34
+ receivedDropped: number;
35
+ }
36
+ /**
37
+ * A `RelayBus` that also carries frames between server replicas over Redis pub/sub, so participants whose relay
38
+ * sockets landed on different replicas still reach each other. The Redis is the deployment's `datastores:events` one,
39
+ * the same the framework's push system (`BasePushRoute`) uses across replicas.
40
+ *
41
+ * ## How
42
+ *
43
+ * The bus owns exactly two Redis clients for the whole process, however many sockets and rooms there are: a
44
+ * publisher, and one shared subscriber (a client in subscriber mode cannot publish). Both come from
45
+ * `importRedis()` and `attachRedisErrorHandler()` of `@rapidrest/service-core`, exactly as `BasePushRoute` obtains its
46
+ * clients - see `connect()`.
47
+ *
48
+ * Each room has a channel, `videoconf:relay:<roomId>`. The bus counts the local listeners of every room: the first
49
+ * one to arrive makes it SUBSCRIBE and the last one to leave makes it UNSUBSCRIBE, so a replica receives only the
50
+ * rooms it has sockets in. Subscriptions use the client's BUFFER mode, so media bytes are never decoded as text.
51
+ *
52
+ * `publish()` delivers to this process's listeners synchronously, exactly as `InProcessRelayBus` does - local
53
+ * delivery never goes through Redis - and, when the publisher is connected, also publishes the frame to the room's
54
+ * channel without waiting for it. Every message carries the id of the bus that published it and a bus ignores its own,
55
+ * so nothing is delivered twice on the replica of origin, while every other replica delivers it to its own listeners.
56
+ *
57
+ * ## Wire format of a Redis message
58
+ *
59
+ * `[16 bytes: origin bus id][1 byte: N][N bytes: the sender's peer id, UTF-8][the frame]`, where `frame` is what the
60
+ * `RelayHub` gave `publish()` (its own `[sender length][sender][payload]` header included). The origin id is random
61
+ * per bus instance. A message that is shorter than that, whose length byte is 0 or overruns it, or that carries no
62
+ * frame, is dropped; parsing never throws.
63
+ *
64
+ * ## Failure behaviour
65
+ *
66
+ * Redis being down or slow never affects local delivery and never throws into the hub: `publish()` is fire-and-forget,
67
+ * is skipped outright while the publisher is not connected (its offline queue is disabled, so an outage cannot pile
68
+ * frames up in memory), and every failed command is caught and counted. Failures are logged at `warn` at most once per
69
+ * `REDIS_RELAY_WARN_INTERVAL_MS` and at `debug` otherwise; connection errors themselves are logged by
70
+ * `attachRedisErrorHandler()` (one error per outage). The clients keep node-redis's default reconnect strategy
71
+ * (retry forever with backoff), and on reconnect node-redis re-issues SUBSCRIBE for every channel the subscriber held
72
+ * (`resubscribe()` in the client's socket initiator), so the bus needs no re-subscribe logic of its own - relay
73
+ * between replicas resumes by itself when Redis comes back, and frames sent during the outage are simply lost, which
74
+ * media tolerates. A bus used before `connect()`, after `close()`, or whose Redis is unavailable behaves as an
75
+ * `InProcessRelayBus`.
76
+ *
77
+ * ## Bandwidth
78
+ *
79
+ * Every relayed frame is published once, whatever the number of receivers, and Redis delivers it to every replica
80
+ * subscribed to that room - including replicas whose sockets there do not `want` the sender at all (the hub filters
81
+ * after the bus). A room's cross-replica traffic is therefore roughly its total relayed upload times the number of
82
+ * OTHER replicas holding sockets of it, and each of those replicas' subscribers reads all of it. A single-replica
83
+ * room costs a publish per frame and no receiving. Redis pub/sub buffers nothing for a slow subscriber beyond the
84
+ * server's client-output-buffer limit for pub/sub, after which Redis drops that connection (the client then
85
+ * reconnects and resubscribes).
86
+ *
87
+ * @author Jean-Philippe Steinmetz
88
+ */
89
+ export declare class RedisRelayBus implements RelayBus {
90
+ private readonly options;
91
+ /** What the bus has done so far. */
92
+ readonly stats: RedisRelayBusStats;
93
+ private readonly id;
94
+ private readonly rooms;
95
+ private publisher?;
96
+ private subscriber?;
97
+ private lastWarn;
98
+ constructor(options: RedisRelayBusOptions);
99
+ /** Whether frames are currently being published to Redis: `connect()` was called, the bus is not closed and the
100
+ * publisher's connection is up. */
101
+ get connected(): boolean;
102
+ /**
103
+ * Creates the publisher and the shared subscriber and starts connecting them, then returns without waiting for
104
+ * either connection - a Redis that is down at start-up delays cross-replica relay, not the server, and the
105
+ * clients keep retrying. Rooms subscribed to before this call are not caught up (the route calls it before it
106
+ * exposes the bus). Does nothing when already connected. Throws when the `redis` package is not installed or the
107
+ * URL is unusable; nothing is left open then.
108
+ */
109
+ connect(): Promise<void>;
110
+ /**
111
+ * Shuts the bus down: destroys both clients (which ends their connections, and with them every subscription) and
112
+ * stops publishing. Local delivery keeps working, as after a failed `connect()`. Safe to call repeatedly.
113
+ * Nothing is unsubscribed one by one: the connection going away does that server-side, and an UNSUBSCRIBE could
114
+ * only stall behind a Redis that is down.
115
+ */
116
+ close(): void;
117
+ subscribe(roomId: string, listener: RelayBusListener): () => void;
118
+ publish(roomId: string, sender: string, frame: Buffer): void;
119
+ private publishToRedis;
120
+ private redisSubscribe;
121
+ private redisUnsubscribe;
122
+ /** Handles one message of a room's channel: parses it defensively and hands its frame to the local listeners,
123
+ * unless this very bus published it. Never throws - it runs inside the Redis client's own message dispatch. */
124
+ private receive;
125
+ /** Splits a Redis message into its sender and frame; `undefined` for a garbled one or this bus's own. */
126
+ private parse;
127
+ /** Runs a Redis command without waiting for it: a failure, thrown or rejected, is only counted and logged. */
128
+ private run;
129
+ /** Counts a failure and logs it - at `warn` once per `REDIS_RELAY_WARN_INTERVAL_MS`, at `debug` in between. */
130
+ private report;
131
+ }
@@ -0,0 +1,188 @@
1
+ /** The version of the relay wire protocol this module implements - the `v` of `hello`/`ready`. */
2
+ export declare const RELAY_PROTOCOL_VERSION = 1;
3
+ /** The largest binary payload (in bytes, excluding the sender header the server adds) a socket may send; anything
4
+ * larger or empty is dropped. Matches `@rapidrest/service-core`'s uWS `DEFAULT_WS_OPTIONS.maxPayloadLength` (16 KiB),
5
+ * which this framework's route registration cannot raise - a larger message closes the socket at the transport. */
6
+ export declare const RELAY_MAX_PAYLOAD_BYTES: number;
7
+ /** The larger message limit the relay route asks the framework for when the installed `@rapidrest/service-core` lets a
8
+ * `@WebSocket()` route set one (its 2.4.0 and later): a video key frame then fits in one message instead of several,
9
+ * with far less fragmenting and reassembly for the client to get wrong. The hub advertises whichever limit applies to
10
+ * the client in `ready` (`maxMessageBytes`). */
11
+ export declare const RELAY_LARGE_PAYLOAD_BYTES: number;
12
+ /** How many bytes the framework may hold unsent for one relay socket before it drops further messages to it, when the
13
+ * installed `@rapidrest/service-core` lets a route set that (its default, 64 KiB, is smaller than one key frame's worth
14
+ * of headroom and made a briefly slow receiver lose frames constantly). One MiB is roughly two seconds of a 350 kbps
15
+ * video stream plus audio. */
16
+ export declare const RELAY_WS_MAX_BACKPRESSURE_BYTES: number;
17
+ /** The most sockets (registered or still awaiting their `hello`) one room holds at once. */
18
+ export declare const RELAY_MAX_SOCKETS_PER_ROOM = 16;
19
+ /** The most sockets one authenticated uid may hold in one room at once (several tabs or devices of one account). */
20
+ export declare const RELAY_MAX_SOCKETS_PER_UID = 4;
21
+ /** Sustained inbound binary budget per socket, in bytes per second (a token bucket refill rate). */
22
+ export declare const RELAY_RATE_BYTES_PER_SECOND: number;
23
+ /** Burst allowance of a socket's token bucket, in bytes - also the bucket's capacity. */
24
+ export declare const RELAY_BURST_BYTES: number;
25
+ /** How long a socket may stay over budget continuously before it is closed with 1008 "Too much data.". */
26
+ export declare const RELAY_OVER_BUDGET_CLOSE_MS = 10000;
27
+ /** Two over-budget drops further apart than this are separate episodes: a socket is "continuously" over budget only
28
+ * while it keeps being dropped at least this often. A sender that exceeds its budget by a hair (one drop every few
29
+ * seconds) is throttled but never closed; one that keeps sending well past it is closed after
30
+ * `RELAY_OVER_BUDGET_CLOSE_MS`. */
31
+ export declare const RELAY_OVER_BUDGET_GRACE_MS = 1000;
32
+ /** How long a new socket has to send its `hello` before it is closed with 1008 "Hello timeout.". */
33
+ export declare const RELAY_HELLO_TIMEOUT_MS = 10000;
34
+ /** The longest peer id, in characters. Its UTF-8 encoding must also fit in the one length byte of a forwarded frame
35
+ * (255), which `isValidRelayPeer()` enforces too. */
36
+ export declare const RELAY_MAX_PEER_LENGTH = 128;
37
+ /** The most peers one `want` message may name; further valid entries are ignored. */
38
+ export declare const RELAY_MAX_WANT_PEERS = 32;
39
+ /** The minimal socket surface the hub needs - satisfied by the framework's uWS/Bun/`ws` sockets and by a trivial
40
+ * fake in a test. */
41
+ export interface RelaySocket {
42
+ /** Sends a text or binary message. `cb` is called with an error when the message could not be sent - for the
43
+ * framework's uWS shim that includes "dropped because the receiver's backpressure limit was reached". */
44
+ send(data: string | Uint8Array, cb?: (err?: Error | null) => void): void;
45
+ close(code?: number, reason?: string): void;
46
+ }
47
+ /** Receives every frame published to a room, whichever replica published it. */
48
+ export type RelayBusListener = (sender: string, frame: Buffer) => void;
49
+ /**
50
+ * **The seam between sockets.** How a frame gets from the socket that sent it to the sockets that want it. The hub
51
+ * never delivers directly: it `publish()`es the stamped frame and delivers whatever its own `subscribe()`d listener
52
+ * receives, so a bus that also carries frames between server replicas (a Redis pub/sub channel per room, say) makes
53
+ * the whole relay multi-replica without touching the hub. `InProcessRelayBus` delivers within this process only;
54
+ * `RedisRelayBus` (`util/RedisRelayBus.ts`) also crosses server replicas over Redis pub/sub.
55
+ */
56
+ export interface RelayBus {
57
+ /** Registers `listener` for `roomId`'s frames - including frames this very process publishes - and returns a
58
+ * function that removes it. */
59
+ subscribe(roomId: string, listener: RelayBusListener): () => void;
60
+ /** Publishes `frame` (the complete forwarded message, sender header included) that `sender` (a peer id) sent
61
+ * to `roomId`. */
62
+ publish(roomId: string, sender: string, frame: Buffer): void;
63
+ }
64
+ /** A `RelayBus` that delivers synchronously to the listeners of this one process - see `RelayHub`'s doc comment for
65
+ * what that means for a deployment with several server replicas. */
66
+ export declare class InProcessRelayBus implements RelayBus {
67
+ private readonly listeners;
68
+ subscribe(roomId: string, listener: RelayBusListener): () => void;
69
+ publish(roomId: string, sender: string, frame: Buffer): void;
70
+ }
71
+ /** What the hub tells a caller about one socket it accepted; the caller feeds it the socket's events. */
72
+ export interface RelayConnection {
73
+ /** Handles one message received from the socket. Never throws, whatever the payload. */
74
+ message(data: string | Uint8Array, isBinary: boolean): void;
75
+ /** Handles the socket having closed. Idempotent, and safe after the hub itself closed the socket. */
76
+ close(): void;
77
+ }
78
+ /** Counters of what the hub did with frames - for tests and diagnostics. */
79
+ export interface RelayStats {
80
+ /** Frames handed to a receiver's `send()`, including any of them it then reported as dropped. */
81
+ framesForwarded: number;
82
+ /** Frames a receiver's `send()` reported as dropped (uWS backpressure) or threw on - a subset of the above,
83
+ * apart from a throw, which is not counted as forwarded. */
84
+ framesDroppedBackpressure: number;
85
+ /** Frames dropped because their sender was over its token-bucket budget. */
86
+ framesDroppedBudget: number;
87
+ /** Frames dropped for being empty, oversized, or sent before `hello`. */
88
+ framesDroppedInvalid: number;
89
+ }
90
+ export interface RelayHubOptions {
91
+ /** Where frames travel between sockets. Defaults to a new `InProcessRelayBus`. */
92
+ bus?: RelayBus;
93
+ logger?: {
94
+ debug(message: string): void;
95
+ };
96
+ /** The largest binary message (and text message) a socket may send, and what `ready` tells the client. Defaults to
97
+ * `RELAY_MAX_PAYLOAD_BYTES`; the route sets `RELAY_LARGE_PAYLOAD_BYTES` when the framework was asked for that much. */
98
+ maxPayloadBytes?: number;
99
+ }
100
+ /**
101
+ * Returns whether `peer` is a well-formed relay peer id: a string of 1 to `RELAY_MAX_PEER_LENGTH` characters with no
102
+ * whitespace, separator, control or other invisible (Unicode category C or Z) character, whose UTF-8 encoding fits in
103
+ * the single length byte of a forwarded frame.
104
+ */
105
+ export declare function isValidRelayPeer(peer: unknown): peer is string;
106
+ /**
107
+ * Reads the `mail:videoconf:relay:enabled` setting. Environment-variable config arrives as a string ("true", "false",
108
+ * "0", "1") and nconf may parse it into a boolean or a number, so all of those are accepted; `false`, `0`, `"false"`
109
+ * and `"0"` (case-insensitive, trimmed) disable the relay and anything else - including an unset or unrecognizable
110
+ * value - leaves it enabled, the default.
111
+ */
112
+ export declare function parseRelayEnabled(value: unknown): boolean;
113
+ /**
114
+ * A fan-out relay of opaque binary frames between the participants of a room (one meeting) - the last-resort tier
115
+ * for participants whose network allows neither a direct WebRTC connection nor a TURN relay, only a WebSocket to this
116
+ * server. It knows nothing about meetings, users, HTTP or any framework: a caller authorizes a socket however it
117
+ * likes, hands it to `attach()` and feeds the returned `RelayConnection` the socket's `message` and `close` events.
118
+ *
119
+ * ## Wire protocol (version 1)
120
+ *
121
+ * Client to server, the first message must be a text message
122
+ * `{"op":"hello","v":1,"peer":"<peerId>"}`, sent within `RELAY_HELLO_TIMEOUT_MS`. `peer` is the caller's uid (the `uid`
123
+ * given to `attach()`) or starts with `<uid>~` - a per-tab or per-device id - and passes `isValidRelayPeer()`. Anything
124
+ * else closes the socket with 1008 "Invalid peer."; a wrong `v` closes it with 1008 "Unsupported version.". The server
125
+ * answers with the text message `{"op":"ready","v":1,"maxMessageBytes":<the limit below>}`. A second socket registering an already registered peer in the
126
+ * same room replaces the older one, which is closed with 1008 "Replaced.".
127
+ *
128
+ * After that, `{"op":"want","peers":["<peerId>", ...]}` REPLACES the socket's interest set with the valid entries, at
129
+ * most `RELAY_MAX_WANT_PEERS` of them; every other or malformed text message is ignored. Binary messages of 1 to
130
+ * `RelayHubOptions.maxPayloadBytes` bytes (`RELAY_MAX_PAYLOAD_BYTES` unless the route asked for more; opaque to the hub) are forwarded to every other socket of the room whose interest
131
+ * set contains the sender's peer id; binary before `hello`, empty and oversized messages are dropped.
132
+ *
133
+ * Server to client: `{"op":"ready","v":1,"maxMessageBytes":N}` once, then binary messages `[1 byte: UTF-8 length N of the sender's peer
134
+ * id][N bytes: the sender's peer id][the original payload]`. The sender is stamped by the hub from the `hello`,
135
+ * never trusted from the payload.
136
+ *
137
+ * ## Limits
138
+ *
139
+ * At most `RELAY_MAX_SOCKETS_PER_ROOM` sockets per room and `RELAY_MAX_SOCKETS_PER_UID` per uid per room, counting
140
+ * a socket from `attach()` on (a socket refused for either is closed with 1008 "Room full." / "Too many
141
+ * connections."). Each socket's inbound binary traffic is metered by a token bucket of `RELAY_RATE_BYTES_PER_SECOND`
142
+ * sustained with a `RELAY_BURST_BYTES` burst: an over-budget message is dropped silently, and a socket kept over
143
+ * budget for more than `RELAY_OVER_BUDGET_CLOSE_MS` (see `RELAY_OVER_BUDGET_GRACE_MS`) is closed with 1008 "Too
144
+ * much data.". A receiver whose `send()` reports the message as dropped (uWS backpressure) simply misses that
145
+ * frame - counted in `stats`, never queued or retried. Sockets and rooms are released when their socket closes; an
146
+ * empty room is deleted.
147
+ *
148
+ * ## Replicas
149
+ *
150
+ * The transport between sockets is the `RelayBus` given to the constructor, `InProcessRelayBus` by default, which
151
+ * delivers only within this process: **with several server replicas, participants whose sockets landed on different
152
+ * replicas cannot relay to each other** - a room exists independently on each replica and its members see only the
153
+ * members connected to the same one. Give the hub a `RedisRelayBus` and frames cross replicas over Redis pub/sub (one
154
+ * channel per room; `BaseVideoMeetingRoute` does that whenever `datastores:events` is configured); nothing else in the
155
+ * hub changes for it.
156
+ *
157
+ * Every limit above is enforced **per replica**, because a hub only knows its own sockets. A room's capacity across
158
+ * replicas is therefore up to `replicas x RELAY_MAX_SOCKETS_PER_ROOM` sockets (and `replicas x
159
+ * RELAY_MAX_SOCKETS_PER_UID` for one uid) in the worst case, where the load balancer spreads a meeting's sockets
160
+ * evenly. A peer id that registers on two replicas is not detected as a duplicate either (the "Replaced." rule is
161
+ * per replica), and each frame reaches every socket that wants its sender on every replica.
162
+ *
163
+ * @author Jean-Philippe Steinmetz
164
+ */
165
+ export declare class RelayHub {
166
+ /** What has happened to frames so far. */
167
+ readonly stats: RelayStats;
168
+ private readonly bus;
169
+ private readonly logger?;
170
+ private readonly rooms;
171
+ private readonly env;
172
+ constructor(options?: RelayHubOptions);
173
+ /** The number of rooms that currently hold at least one socket. */
174
+ get roomCount(): number;
175
+ /** The number of sockets in `roomId` (0 for an unknown room). */
176
+ socketCount(roomId: string): number;
177
+ /**
178
+ * Accepts an already-authorized `socket` of `uid` into room `roomId`, or refuses it (closing it with 1008) when
179
+ * the room or the uid is at its limit. Returns the connection to feed the socket's events to, or `undefined`
180
+ * when refused.
181
+ */
182
+ attach(roomId: string, uid: string, socket: RelaySocket): RelayConnection | undefined;
183
+ /** Forgets `conn`, and its room once that is empty. */
184
+ private remove;
185
+ /** Hands `frame` from `sender` to every other socket of room `roomId` that wants it. A bus that delivers
186
+ * asynchronously (across replicas) may deliver a frame for a room that has just emptied; that is ignored. */
187
+ private deliver;
188
+ }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@rapidmx/meet-plugin",
3
- "version": "0.5.0",
3
+ "version": "0.8.0",
4
4
  "description": "RapidMX plugin that provides WebRTC video conferencing, joinable from a calendar invite or a shared link.",
5
5
  "repository": "https://github.com/rapidmx/meet-plugin.git",
6
6
  "author": "RapidREST <rapidrests@gmail.com>",
@@ -21,6 +21,9 @@
21
21
  "release": "npx @rapidrest/cli release",
22
22
  "prepublish": "yarn build"
23
23
  },
24
+ "dependencies": {
25
+ "@mediapipe/tasks-vision": "^1.0.1"
26
+ },
24
27
  "peerDependencies": {
25
28
  "@rapidmx/react-shared": ">=0.19.0 <1",
26
29
  "@rapidmx/restapi": ">=0.23.0 <1",
@@ -32,12 +35,12 @@
32
35
  "react-dom": "19.x"
33
36
  },
34
37
  "devDependencies": {
35
- "@rapidmx/react-shared": "^0.19.0",
36
- "@rapidmx/restapi": "^0.23.0",
37
- "@rapidmx/web-client": "^0.17.0",
38
+ "@rapidmx/react-shared": "^0.22.0",
39
+ "@rapidmx/restapi": "^0.24.0",
40
+ "@rapidmx/web-client": "^0.21.0",
38
41
  "@rapidrest/core": "^6.0.0",
39
42
  "@rapidrest/react": "^2.1.0",
40
- "@rapidrest/service-core": "^2.3.0",
43
+ "@rapidrest/service-core": "^2.4.0",
41
44
  "@swc/core": "^1.16.2",
42
45
  "@testing-library/dom": "^10.4.1",
43
46
  "@testing-library/jest-dom": "^7.0.1",
@@ -114,7 +117,7 @@
114
117
  "rapidmx": {
115
118
  "plugin": {
116
119
  "apiVersion": 1,
117
- "displayName": "Video Conferencing",
120
+ "displayName": "Meet",
118
121
  "description": "Private and public WebRTC video meetings. A private meeting mints one link per invitee for a calendar invite's location and body; a public meeting shares a single link. Joining needs no account - a guest picks a name, checks their camera and microphone, and enters.",
119
122
  "mailboxScopedData": true,
120
123
  "settings": [
@@ -130,7 +133,7 @@
130
133
  "label": "TURN server URL",
131
134
  "type": "string",
132
135
  "default": "",
133
- "help": "A turn: or turns: URL (e.g. turns:turn.example.com:5349), or several separated by commas (turn:turn.example.com:3478,turns:turn.example.com:5349), relaying media for a participant behind a restrictive NAT or firewall, for whom a direct peer-to-peer connection fails. Public STUN servers are used for everyone else and need no configuration. Leave empty to serve calls with STUN only - most calls will still connect, but some participants may not."
136
+ "help": "A turn: or turns: URL (e.g. turns:turn.example.com:5349), or several separated by commas (turn:turn.example.com:3478,turns:turn.example.com:5349), relaying media for a participant behind a restrictive NAT or firewall, for whom a direct peer-to-peer connection fails. A plain turn: address is offered over both UDP and TCP (a network that blocks UDP can still use the TCP one); add ?transport=udp or ?transport=tcp to an address to offer only that. Public STUN servers are used for everyone else and need no configuration. Leave empty to serve calls with STUN only - most calls will still connect, but some participants may not."
134
137
  },
135
138
  {
136
139
  "key": "mail:videoconf:turn:username",
@@ -152,6 +155,20 @@
152
155
  "type": "string",
153
156
  "default": "",
154
157
  "help": "coturn's static-auth-secret, if configured. When set, every join mints its own short-lived username/credential pair (an HMAC-SHA1 of a timestamp with this secret) instead of handing out one permanent password to every participant - the recommended setup. Leave empty to use the static username/credential above instead."
158
+ },
159
+ {
160
+ "key": "mail:videoconf:relay:enabled",
161
+ "label": "WebSocket media relay",
162
+ "type": "boolean",
163
+ "default": true,
164
+ "help": "The last-resort tier for a participant whose network blocks both a direct connection and TURN (e.g. a corporate firewall that only allows web traffic): their media travels over a WebSocket this server relays between the meeting's participants. Turn off to serve calls with direct and TURN connections only. With several server replicas it connects participants across them through the same Redis the push system uses; without that Redis it works only between participants on the same replica."
165
+ },
166
+ {
167
+ "key": "mail:videoconf:effects:assets_url",
168
+ "label": "Video effects assets URL",
169
+ "type": "string",
170
+ "default": "",
171
+ "help": "Video effects (background blur, custom backgrounds, face accessories) run a small machine-learning model in the participant's browser. By default the model files and their WebAssembly runtime are downloaded, only when a participant turns such an effect on, from cdn.jsdelivr.net and storage.googleapis.com. To serve them yourself instead, set this to the base URL (https://files.example.com/meet-effects, or a path on this server) of a directory holding wasm/ (the contents of the @mediapipe/tasks-vision package's wasm directory), selfie_segmenter.tflite and face_landmarker.task. Leave empty to use the public CDNs."
155
172
  }
156
173
  ],
157
174
  "ui": {