@fluidframework/presence 2.70.0-361788 → 2.71.0

package/lib/presenceDatastoreManager.js

@@ -53,6 +53,27 @@ function mergeGeneralDatastoreMessageContent(base, newData) {
     }
     return queueDatastore;
 }
+/**
+ * Delays used for broadcasting join responses to clients.
+ *
+ * @remarks
+ * Exported for test coordination.
+ * These could be made customizable in the future to accommodate different
+ * session configurations.
+ */
+export const broadcastJoinResponseDelaysMs = {
+    /**
+     * The delay in milliseconds before a join response is sent to any client.
+     * This is used to accumulate other join response requests and reduce
+     * network traffic.
+     */
+    namedResponder: 200,
+    /**
+     * The additional delay in milliseconds a backup responder waits before sending
+     * a join response to allow others to respond first.
+     */
+    backupResponderIncrement: 40,
+};
 /**
  * Manages singleton datastore for all Presence.
  */
@@ -65,22 +86,123 @@ export class PresenceDatastoreManagerImpl {
         this.presence = presence;
         this.averageLatency = 0;
         this.returnedMessages = 0;
-        this.refreshBroadcastRequested = false;
-        this.timer = new TimerManager();
+        this.sendMessageTimer = new TimerManager();
         this.workspaces = new Map();
+        /**
+         * Map of outstanding broadcast (join response) requests.
+         */
+        this.broadcastRequests = new Map();
+        /**
+         * Timer for managing broadcast (join response) request timing.
+         */
+        this.broadcastRequestsTimer = new TimerManager();
+        /**
+         * Broadcasts a join response (complete datastore update message)
+         * if there is an outstanding join response request.
+         */
+        this.sendJoinResponseIfStillNeeded = () => {
+            // Make sure we are currently connected and a broadcast is still needed.
+            // If not connected, nothing we can do.
+            if (this.runtime.getJoinedStatus() !== "disconnected" && this.broadcastRequests.size > 0) {
+                // Confirm that of remaining requests, now is the time to respond.
+                const now = Date.now();
+                let minResponseTime = Number.POSITIVE_INFINITY;
+                for (const { deadlineTime } of this.broadcastRequests.values()) {
+                    minResponseTime = Math.min(minResponseTime, deadlineTime);
+                }
+                if (minResponseTime <= now) {
+                    if (this.reasonForCompleteSnapshot) {
+                        this.broadcastAllKnownState();
+                    }
+                }
+                else {
+                    // No response needed yet - schedule a later attempt
+                    this.broadcastRequestsTimer.setTimeout(this.sendJoinResponseIfStillNeeded, minResponseTime - now);
+                }
+            }
+        };
         // eslint-disable-next-line @typescript-eslint/consistent-type-assertions
         this.datastore = { "system:presence": systemWorkspaceDatastore };
         this.workspaces.set("system:presence", systemWorkspace);
         this.targetedSignalSupport = this.runtime.supportedFeatures.has("submit_signals_v2");
+        // If audience member is removed, they won't need a broadcast response.
+        this.runtime.getAudience().on("removeMember", (clientId) => {
+            this.broadcastRequests.delete(clientId);
+        });
+    }
+    getAudienceInformation(selfClientId) {
+        const audience = this.runtime.getAudience();
+        const members = audience.getMembers();
+        const all = new Set();
+        const writers = new Set();
+        const selfPresent = members.has(selfClientId);
+        if (selfPresent) {
+            // Remove self
+            members.delete(selfClientId);
+        }
+        // Gather interactive client IDs
+        for (const [id, client] of members) {
+            if (client.details.capabilities.interactive) {
+                all.add(id);
+                if (client.mode === "write") {
+                    writers.add(id);
+                }
+            }
+        }
+        return {
+            audience,
+            selfPresent,
+            interactiveMembersExcludingSelf: {
+                all,
+                writers,
+            },
+        };
     }
-    joinSession(clientId) {
+    joinSession(selfClientId, alternateProvider = undefined) {
+        // Before broadcasting the join message, check that there is at least
+        // one audience member present (self or another). This is useful to
+        // optimize join messages while not using targeted join responses.
+        // (We need at least one other to be able to elect them as update
+        // provider.)
+        // Lack of anyone likely means that this client is very freshly joined
+        // and has not received any Join Signals (type="join") from the service
+        // yet.
+        const { audience, selfPresent, interactiveMembersExcludingSelf } = this.getAudienceInformation(selfClientId);
+        if (interactiveMembersExcludingSelf.all.size === 0 && alternateProvider !== undefined) {
+            if (selfPresent) {
+                // If there aren't any members connected except self, then this client
+                // must have complete information.
+                this.reasonForCompleteSnapshot = "alone";
+                // It would be possible to return at this time and skip ClientJoin
+                // signal. Instead continue in case audience information is
+                // inaccurate. This client might temporarily erroneously believe it
+                // has complete information, but the other(s) should respond to
+                // ClientJoin soon rectifying that and covering for bad incomplete
+                // responses this client sent in the meantime.
+            }
+            else {
+                // No one is known. Not even self. Defer judgement on
+                // complete snapshot until at least self is known to be present.
+                this.listenForSelfInAudience(selfClientId, audience);
+                return;
+            }
+        }
         // Broadcast join message to all clients
-        const updateProviders = [...this.runtime.getQuorum().getMembers().keys()].filter((quorumClientId) => quorumClientId !== clientId);
+        // Select primary update providers
+        // Use write members if any, then fallback to read-only members.
+        const updateProviders = [
+            ...(interactiveMembersExcludingSelf.writers.size > 0
+                ? interactiveMembersExcludingSelf.writers
+                : interactiveMembersExcludingSelf.all),
+        ];
         // Limit to three providers to prevent flooding the network.
         // If none respond, others present will (should) after a delay.
         if (updateProviders.length > 3) {
             updateProviders.length = 3;
         }
+        else if (updateProviders.length === 0 && alternateProvider !== undefined) {
+            updateProviders.push(alternateProvider);
+        }
         this.runtime.submitSignal({
             type: joinMessageType,
             content: {
@@ -90,6 +212,58 @@ export class PresenceDatastoreManagerImpl {
                 updateProviders,
             },
         });
+        this.logger?.sendTelemetryEvent({
+            eventName: "JoinRequested",
+            details: {
+                attendeeId: this.attendeeId,
+                connectionId: selfClientId,
+                // Empty updateProviders is indicative of join when alone.
+                updateProviders: JSON.stringify(updateProviders),
+                // If false and providers is single entry, then join was probably forced.
+                selfPresent,
+            },
+        });
+    }
+    listenForSelfInAudience(selfClientId, audience) {
+        this.logger?.sendTelemetryEvent({
+            eventName: "JoinDeferred",
+            details: {
+                attendeeId: this.attendeeId,
+                connectionId: selfClientId,
+            },
+        });
+        // Prepare to join once self audience member joins.
+        // Alternatively, processSignal may force a join when a presence
+        // signal is received even without audience members (assumes
+        // audience signals were lost).
+        const joinWhenSelfAudienceMemberAdded = (addedClientId) => {
+            if (addedClientId !== selfClientId) {
+                // Keep listening
+                return;
+            }
+            // No need to force here by providing alternate provider as self is
+            // now present.
+            // Do avoid forcing so that reasonForCompleteSnapshot is set correctly
+            // if no others have been added.
+            this.stopWaitingAndJoin(selfClientId, /* alternateProvider */ undefined);
+        };
+        audience.on("addMember", joinWhenSelfAudienceMemberAdded);
+        this.stopWaitingForSelfInAudience = () => {
+            audience.off("addMember", joinWhenSelfAudienceMemberAdded);
+        };
+    }
+    stopWaitingAndJoin(selfClientId, alternateProvider) {
+        this.stopWaitingForSelfInAudience?.();
+        this.stopWaitingForSelfInAudience = undefined;
+        // Confirm not currently disconnected
+        if (this.runtime.getJoinedStatus() !== "disconnected") {
+            this.joinSession(selfClientId, alternateProvider);
+        }
+    }
+    onDisconnected() {
+        delete this.reasonForCompleteSnapshot;
+        this.stopWaitingForSelfInAudience?.();
+        this.stopWaitingForSelfInAudience = undefined;
     }
     getWorkspace(internalWorkspaceAddress, requestedContent, controls) {
         const existing = this.workspaces.get(internalWorkspaceAddress);
@@ -126,9 +300,14 @@ export class PresenceDatastoreManagerImpl {
      * the send timer, other messages in the queue, the configured allowed latency, etc.
      */
     enqueueMessage(data, options) {
-        // Merging the message with any queued messages effectively queues the message.
-        // It is OK to queue all incoming messages as long as when we send, we send the queued data.
-        this.queuedData = mergeGeneralDatastoreMessageContent(this.queuedData, data);
+        if (this.queuedData !== "sendAll") {
+            this.queuedData =
+                data === "sendAll"
+                    ? "sendAll"
+                    : // Merging the message with any queued messages effectively queues the message.
+                        // It is OK to queue all incoming messages as long as when we send, we send the queued data.
+                        mergeGeneralDatastoreMessageContent(this.queuedData, data);
+        }
         const { allowableUpdateLatencyMs } = options;
         const now = Date.now();
         const thisMessageDeadline = now + allowableUpdateLatencyMs;
@@ -136,20 +315,20 @@ export class PresenceDatastoreManagerImpl {
         // If the timer has not expired, we can short-circuit because the timer will fire
         // and cover this update. In other words, queuing this will be fast enough to
         // meet its deadline, because a timer is already scheduled to fire before its deadline.
-        !this.timer.hasExpired() &&
+        !this.sendMessageTimer.hasExpired() &&
             // If the deadline for this message is later than the overall send deadline, then
             // we can exit early since a timer will take care of sending it.
-            thisMessageDeadline >= this.timer.expireTime) {
+            thisMessageDeadline >= this.sendMessageTimer.expireTime) {
             return;
         }
         // Either we need to send this message immediately, or we need to schedule a timer
         // to fire at the send deadline that will take care of it.
-        // Note that timeoutInMs === allowableUpdateLatency, but the calculation is done this way for clarity.
+        // Note that timeoutInMs === allowableUpdateLatencyMs, but the calculation is done this way for clarity.
         const timeoutInMs = thisMessageDeadline - now;
         const scheduleForLater = timeoutInMs > 0;
         if (scheduleForLater) {
             // Schedule the queued messages to be sent at the updateDeadline
-            this.timer.setTimeout(this.sendQueuedMessage.bind(this), timeoutInMs);
+            this.sendMessageTimer.setTimeout(this.sendQueuedMessage.bind(this), timeoutInMs);
         }
         else {
             this.sendQueuedMessage();
@@ -159,7 +338,7 @@ export class PresenceDatastoreManagerImpl {
      * Send any queued signal immediately. Does nothing if no message is queued.
      */
     sendQueuedMessage() {
-        this.timer.clearTimeout();
+        this.sendMessageTimer.clearTimeout();
         if (this.queuedData === undefined) {
             return;
         }
@@ -170,12 +349,16 @@ export class PresenceDatastoreManagerImpl {
             this.queuedData = undefined;
             return;
         }
+        if (this.queuedData === "sendAll") {
+            this.broadcastAllKnownState();
+            return;
+        }
         const clientConnectionId = this.runtime.getClientId();
         assert(clientConnectionId !== undefined, 0xa59 /* Client connected without clientId */);
         const currentClientToSessionValueState = 
         // When connected, `clientToSessionId` must always have current connection entry.
-        // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
         this.datastore["system:presence"].clientToSessionId[clientConnectionId];
+        assert(currentClientToSessionValueState !== undefined, "Client connection update missing");
         const newMessage = {
             sendTimestamp: Date.now(),
             avgLatency: this.averageLatency,
@@ -248,16 +431,50 @@ export class PresenceDatastoreManagerImpl {
         return valueData;
     }
     broadcastAllKnownState() {
+        const content = {
+            sendTimestamp: Date.now(),
+            avgLatency: this.averageLatency,
+            isComplete: true,
+            data: this.stripValidationMetadata(this.datastore),
+        };
+        const primaryRequestors = [];
+        const secondaryRequestors = [];
+        if (this.broadcastRequests.size > 0) {
+            content.joinResponseFor = [...this.broadcastRequests.keys()];
+            if (this.logger) {
+                // Build telemetry data
+                for (const [requestor, { responseOrder }] of this.broadcastRequests.entries()) {
+                    if (responseOrder === undefined) {
+                        primaryRequestors.push(requestor);
+                    }
+                    else {
+                        secondaryRequestors.push([requestor, responseOrder]);
+                    }
+                }
+            }
+            this.broadcastRequests.clear();
+        }
+        // This broadcast will satisfy all requests; clear any remaining timer.
+        this.broadcastRequestsTimer.clearTimeout();
+        this.sendMessageTimer.clearTimeout();
         this.runtime.submitSignal({
             type: datastoreUpdateMessageType,
-            content: {
-                sendTimestamp: Date.now(),
-                avgLatency: this.averageLatency,
-                isComplete: true,
-                data: this.stripValidationMetadata(this.datastore),
-            },
+            content,
         });
-        this.refreshBroadcastRequested = false;
+        if (content.joinResponseFor) {
+            this.logger?.sendTelemetryEvent({
+                eventName: "JoinResponse",
+                details: {
+                    type: "broadcastAll",
+                    attendeeId: this.attendeeId,
+                    connectionId: this.runtime.getClientId(),
+                    primaryResponses: JSON.stringify(primaryRequestors),
+                    secondaryResponses: JSON.stringify(secondaryRequestors),
+                },
+            });
+        }
+        // Sending all must account for anything queued before.
+        this.queuedData = undefined;
     }
     processSignal(message, local, optional) {
         const received = Date.now();
@@ -278,8 +495,22 @@ export class PresenceDatastoreManagerImpl {
                     this.returnedMessages;
             return;
         }
+        const selfClientId = this.runtime.getClientId();
+        assert(selfClientId !== undefined, "Received signal without clientId");
+        // Check for undesired case of receiving a remote presence signal
+        // without having been alerted to self audience join. (Perhaps join
+        // signal was dropped.)
+        // In practice it is commonly observed that local signals can be
+        // returned ahead of audience join notification. So, it is reasonable
+        // to expect that audience join notification may be delayed until after
+        // other presence signals are received. One is enough to get things
+        // rolling.
+        if (this.stopWaitingForSelfInAudience !== undefined) {
+            this.stopWaitingAndJoin(selfClientId, /* alternateProvider */ message.clientId);
+        }
         const timeModifier = received -
             (this.averageLatency + message.content.avgLatency + message.content.sendTimestamp);
+        const postUpdateActions = [];
         if (message.type === joinMessageType) {
             // It is possible for some signals to come in while client is not connected due
             // to how work is scheduled. If we are not connected, we can't respond to the
@@ -291,8 +522,48 @@ export class PresenceDatastoreManagerImpl {
             // connected.
         }
         else {
-            if (message.content.isComplete) {
-                this.refreshBroadcastRequested = false;
+            // Update join response requests that are now satisfied.
+            const joinResponseFor = message.content.joinResponseFor;
+            if (joinResponseFor) {
+                let justGainedCompleteSnapshot = false;
+                if (joinResponseFor.includes(selfClientId)) {
+                    if (this.reasonForCompleteSnapshot) {
+                        if (this.reasonForCompleteSnapshot === "alone") {
+                            // No response was expected. This might happen when
+                            // either cautionary ClientJoin signal is received
+                            // by audience member that was unknown.
+                            this.logger?.sendTelemetryEvent({
+                                eventName: "JoinResponseWhenAlone",
+                                details: {
+                                    attendeeId: this.attendeeId,
+                                    connectionId: this.runtime.getClientId(),
+                                },
+                            });
+                        }
+                    }
+                    else {
+                        // If we are the intended recipient of the join response,
+                        // we can consider our knowledge complete and can respond
+                        // to others join requests.
+                        justGainedCompleteSnapshot = true;
+                    }
+                    this.reasonForCompleteSnapshot = "join response";
+                }
+                if (this.broadcastRequests.size > 0) {
+                    for (const responseFor of joinResponseFor) {
+                        this.broadcastRequests.delete(responseFor);
+                    }
+                    if (this.broadcastRequests.size === 0) {
+                        // If no more requests are pending, clear any timer.
+                        this.broadcastRequestsTimer.clearTimeout();
+                    }
+                    else if (justGainedCompleteSnapshot) {
+                        // May or may not be time to respond to remaining requests.
+                        // Clear the timer and recheck after processing.
+                        this.broadcastRequestsTimer.clearTimeout();
+                        postUpdateActions.push(this.sendJoinResponseIfStillNeeded);
+                    }
+                }
             }
             // If the message requests an acknowledgement, we will send a targeted acknowledgement message back to just the requestor.
             if (message.content.acknowledgementId !== undefined) {
@@ -322,7 +593,6 @@ export class PresenceDatastoreManagerImpl {
             const internalWorkspaceType = internalWorkspaceTypes[prefix] ?? "Unknown";
             this.events.emit("workspaceActivated", publicWorkspaceAddress, internalWorkspaceType);
         }
-        const postUpdateActions = [];
         // While the system workspace is processed here too, it is declared as
         // conforming to the general schema. So drop its override.
         const data = message.content.data;
@@ -358,67 +628,91 @@ export class PresenceDatastoreManagerImpl {
      * correlation with other telemetry where it is often called just `clientId`.
      */
     prepareJoinResponse(updateProviders, requestor) {
-        this.refreshBroadcastRequested = true;
         // We must be connected to receive this message, so clientId should be defined.
-        // If it isn't then, not really a problem; just won't be in provider or quorum list.
+        // If it isn't then, not really a problem; just won't be in provider or audience list.
         // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-        const clientId = this.runtime.getClientId();
-        // const requestor = message.clientId;
-        if (updateProviders.includes(clientId)) {
-            // Send all current state to the new client
-            this.broadcastAllKnownState();
-            this.logger?.sendTelemetryEvent({
-                eventName: "JoinResponse",
-                details: {
-                    type: "broadcastAll",
-                    requestor,
-                    role: "primary",
-                },
-            });
-        }
-        else {
+        const selfClientId = this.runtime.getClientId();
+        let joinResponseDelayMs = broadcastJoinResponseDelaysMs.namedResponder;
+        let relativeResponseOrder;
+        if (!updateProviders.includes(selfClientId)) {
             // Schedule a broadcast to the new client after a delay only to send if
-            // another broadcast hasn't been seen in the meantime. The delay is based
-            // on the position in the quorum list. It doesn't have to be a stable
-            // list across all clients. We need something to provide suggested order
-            // to prevent a flood of broadcasts.
-            let relativeResponseOrder;
+            // another broadcast satisfying the request hasn't been seen in the
+            // meantime. The delay is based on the position in the quorum list. It
+            // doesn't have to be a stable list across all clients. We need
+            // something to provide suggested order to prevent a flood of broadcasts.
             const quorumMembers = this.runtime.getQuorum().getMembers();
-            const self = quorumMembers.get(clientId);
+            const self = quorumMembers.get(selfClientId);
             if (self) {
                 // Compute order quorum join order (indicated by sequenceNumber)
                 relativeResponseOrder = 0;
-                for (const { sequenceNumber } of quorumMembers.values()) {
-                    if (sequenceNumber < self.sequenceNumber) {
+                for (const { client, sequenceNumber } of quorumMembers.values()) {
+                    if (sequenceNumber < self.sequenceNumber &&
+                        client.details.capabilities.interactive) {
                         relativeResponseOrder++;
                     }
                 }
             }
             else {
                 // Order past quorum members + arbitrary additional offset up to 10
-                relativeResponseOrder = quorumMembers.size + Math.random() * 10;
-            }
-            // These numbers have been chosen arbitrarily to start with.
-            // 20 is minimum wait time, 20 is the additional wait time per provider
-            // given an chance before us with named providers given more time.
-            const waitTime = 20 + 20 * (3 * updateProviders.length + relativeResponseOrder);
-            setTimeout(() => {
-                // Make sure a broadcast is still needed and we are currently connected.
-                // If not connected, nothing we can do.
-                if (this.refreshBroadcastRequested &&
-                    this.runtime.getJoinedStatus() !== "disconnected") {
-                    this.broadcastAllKnownState();
-                    this.logger?.sendTelemetryEvent({
-                        eventName: "JoinResponse",
-                        details: {
-                            type: "broadcastAll",
-                            requestor,
-                            role: "secondary",
-                            order: relativeResponseOrder,
-                        },
-                    });
+                let possibleQuorumRespondents = 0;
+                for (const { client } of quorumMembers.values()) {
+                    if (client.details.capabilities.interactive) {
+                        possibleQuorumRespondents++;
+                    }
                 }
-            }, waitTime);
+                relativeResponseOrder = possibleQuorumRespondents + Math.random() * 10;
+            }
+            // When not named to provide update, wait an additional amount
+            // of time for those named or others to respond.
+            joinResponseDelayMs +=
+                broadcastJoinResponseDelaysMs.backupResponderIncrement *
+                    (3 * updateProviders.length + relativeResponseOrder);
+        }
+        // Add the requestor to the list of clients that will receive the broadcast.
+        const deadlineTime = Date.now() + joinResponseDelayMs;
+        this.broadcastRequests.set(requestor, {
+            deadlineTime,
+            responseOrder: relativeResponseOrder,
+        });
+        if (!this.reasonForCompleteSnapshot) {
+            // Check if requestor count meets or exceeds count of other audience
+            // members indicating that we effectively have a complete snapshot
+            // (once the current message being processed is processed).
+            const { selfPresent, interactiveMembersExcludingSelf } = this.getAudienceInformation(selfClientId);
+            if (
+            // Self-present check is done to help ensure that audience
+            // information is accurate. If self is not present, audience
+            // information might be incomplete.
+            selfPresent &&
+                this.broadcastRequests.size >= interactiveMembersExcludingSelf.all.size) {
+                // Note that no action is taken here specifically.
+                // We want action to be queued so that it takes place after
+                // current message is completely processed. All of the actions
+                // below should be delayed (not immediate).
+                this.reasonForCompleteSnapshot = "full requests";
+            }
+        }
+        // Check if capable of full primary response. If requested to provide
+        // primary response, but do not yet have complete snapshot, we need to
+        // delay a full response, until we think we have complete snapshot. In
+        // the meantime we will send partial updates as usual.
+        if (this.reasonForCompleteSnapshot && updateProviders.includes(selfClientId)) {
+            // Use regular message queue to handle timing of the broadcast.
+            // Any more immediate broadcasts will accelerate the response time.
+            // As a primary responder, it is expected that broadcast will happen and
+            // using the regular queue allows other updates to avoid merge work.
+            this.enqueueMessage("sendAll", {
+                allowableUpdateLatencyMs: joinResponseDelayMs,
+            });
+        }
+        else {
+            // Check if there isn't already a timer scheduled to send a join
+            // response with in this request's deadline.
+            if (this.broadcastRequestsTimer.hasExpired() ||
+                deadlineTime < this.broadcastRequestsTimer.expireTime) {
+                // Set or update the timer.
+                this.broadcastRequestsTimer.setTimeout(this.sendJoinResponseIfStillNeeded, joinResponseDelayMs);
+            }
         }
     }
 }