@fluidframework/container-loader 2.0.0-rc.3.0.2 → 2.0.0-rc.4.0.0

package/src/connectionStateHandler.ts

@@ -3,13 +3,14 @@
  * Licensed under the MIT License.
  */
 
-import { IDeltaManager } from "@fluidframework/container-definitions";
+import { IDeltaManager } from "@fluidframework/container-definitions/internal";
 import { ITelemetryBaseProperties } from "@fluidframework/core-interfaces";
 import { assert, Timer } from "@fluidframework/core-utils/internal";
-import { IAnyDriverError } from "@fluidframework/driver-definitions";
+import { IAnyDriverError } from "@fluidframework/driver-definitions/internal";
 import { IClient, ISequencedClient } from "@fluidframework/protocol-definitions";
-import { ITelemetryLoggerExt, type TelemetryEventCategory } from "@fluidframework/telemetry-utils";
 import {
+	type TelemetryEventCategory,
+	ITelemetryLoggerExt,
 	PerformanceEvent,
 	loggerToMonitoringContext,
 } from "@fluidframework/telemetry-utils/internal";
@@ -25,7 +26,7 @@ import { IProtocolHandler } from "./protocol.js";
 const JoinOpTimeoutMs = 45000;
 
 // Timeout waiting for "self" join signal, before giving up
-const JoinSignalTimeoutMs = 5000;
+const JoinSignalTimeoutMs = 10000;
 
 /** Constructor parameter type for passing in dependencies needed by the ConnectionStateHandler */
 export interface IConnectionStateHandlerInputs {
@@ -48,6 +49,9 @@ export interface IConnectionStateHandlerInputs {
 	) => void;
 	/** Callback to note that an old local client ID is still present in the Quorum that should have left and should now be considered invalid */
 	clientShouldHaveLeft: (clientId: string) => void;
+
+	/** Some critical error was hit. Container should be closed and error logged. */
+	onCriticalError: (error: unknown) => void;
 }
 
 /**
@@ -55,7 +59,19 @@ export interface IConnectionStateHandlerInputs {
  */
 export interface IConnectionStateHandler {
 	readonly connectionState: ConnectionState;
+	/**
+	 * Pending clientID.
+	 * Changes whenever socket connection is established.
+	 * Resets to undefined when connection is lost
+	 */
 	readonly pendingClientId: string | undefined;
+	/**
+	 * clientId of a last established connection.
+	 * Does not reset on disconnect.
+	 * Changes only when new connection is established, client is fully caught up, and
+	 * there is no chance to ops from previous connection (i.e. if needed, we have waited and observed leave op from previous connection)
+	 */
+	readonly clientId: string | undefined;
 
 	containerSaved(): void;
 	dispose(): void;
@@ -78,8 +94,8 @@ export function createConnectionStateHandler(
 ) {
 	const mc = loggerToMonitoringContext(inputs.logger);
 	return createConnectionStateHandlerCore(
-		mc.config.getBoolean("Fluid.Container.CatchUpBeforeDeclaringConnected") === true, // connectedRaisedWhenCaughtUp
-		mc.config.getBoolean("Fluid.Container.EnableJoinSignalWait") === true, // readClientsWaitForJoinSignal
+		mc.config.getBoolean("Fluid.Container.DisableCatchUpBeforeDeclaringConnected") !== true, // connectedRaisedWhenCaughtUp
+		mc.config.getBoolean("Fluid.Container.DisableJoinSignalWait") !== true, // readClientsWaitForJoinSignal
 		inputs,
 		deltaManager,
 		clientId,
@@ -93,15 +109,12 @@ export function createConnectionStateHandlerCore(
 	deltaManager: IDeltaManager<any, any>,
 	clientId?: string,
 ) {
-	if (!connectedRaisedWhenCaughtUp) {
-		return new ConnectionStateHandler(inputs, readClientsWaitForJoinSignal, clientId);
-	}
-	return new ConnectionStateCatchup(
-		inputs,
-		(handler: IConnectionStateHandlerInputs) =>
-			new ConnectionStateHandler(handler, readClientsWaitForJoinSignal, clientId),
-		deltaManager,
-	);
+	const factory = (handler: IConnectionStateHandlerInputs) =>
+		new ConnectionStateHandler(handler, readClientsWaitForJoinSignal, clientId);
+
+	return connectedRaisedWhenCaughtUp
+		? new ConnectionStateCatchup(inputs, factory, deltaManager)
+		: factory(inputs);
 }
 
 /**
@@ -140,6 +153,9 @@ class ConnectionStateHandlerPassThrough
 	public get pendingClientId() {
 		return this.pimpl.pendingClientId;
 	}
+	public get clientId() {
+		return this.pimpl.clientId;
+	}
 
 	public containerSaved() {
 		return this.pimpl.containerSaved();
@@ -196,6 +212,10 @@ class ConnectionStateHandlerPassThrough
 	public clientShouldHaveLeft(clientId: string) {
 		return this.inputs.clientShouldHaveLeft(clientId);
 	}
+
+	public onCriticalError(error: unknown) {
+		return this.inputs.onCriticalError(error);
+	}
 }
 
 /**
@@ -325,7 +345,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 	/**
 	 * Tracks that we observe our own "join" op within the timeout after receiving a "connected" event from the DeltaManager
 	 */
-	private readonly joinOpTimer: Timer;
+	private readonly joinTimer: Timer;
 
 	private protocol?: IProtocolHandler;
 	private connection?: IConnectionDetailsInternal;
@@ -338,7 +358,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		return this._connectionState;
 	}
 
-	private get clientId(): string | undefined {
+	public get clientId(): string | undefined {
 		return this._clientId;
 	}
 
@@ -352,6 +372,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		clientIdFromPausedSession?: string,
 	) {
 		this._clientId = clientIdFromPausedSession;
+		const errorHandler = (error) => this.handler.onCriticalError(error);
 		this.prevClientLeftTimer = new Timer(
 			// Default is 5 min for which we are going to wait for its own "leave" message. This is same as
 			// the max time on server after which leave op is sent.
@@ -363,10 +384,11 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 				);
 				this.applyForConnectedState("timeout");
 			},
+			errorHandler,
 		);
 
-		this.joinOpTimer = new Timer(
-			0, // default value is not used - startJoinOpTimer() explicitly provides timeout
+		this.joinTimer = new Timer(
+			0, // default value is not used - startjoinTimer() explicitly provides timeout
 			() => {
 				// I've observed timer firing within couple ms from disconnect event, looks like
 				// queued timer callback is not cancelled if timer is cancelled while callback sits in the queue.
@@ -379,22 +401,27 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 					clientJoined: this.hasMember(this.pendingClientId),
 					waitingForLeaveOp: this.waitingForLeaveOp,
 				};
-				this.handler.logConnectionIssue("NoJoinOp", "error", details);
+				this.handler.logConnectionIssue(
+					this.connection?.mode === "read" ? "NoJoinSignal" : "NoJoinOp", // eventName
+					"error", // category
+					details,
+				);
 			},
+			errorHandler,
 		);
 	}
 
-	private startJoinOpTimer() {
-		assert(!this.joinOpTimer.hasTimer, 0x234 /* "has joinOpTimer" */);
+	private startjoinTimer() {
+		assert(!this.joinTimer.hasTimer, 0x234 /* "has joinTimer" */);
 		assert(this.connection !== undefined, 0x4b3 /* have connection */);
-		this.joinOpTimer.start(
+		this.joinTimer.start(
 			this.connection.mode === "write" ? JoinOpTimeoutMs : JoinSignalTimeoutMs,
 		);
 	}
 
-	private stopJoinOpTimer() {
-		assert(this.joinOpTimer.hasTimer, 0x235 /* "no joinOpTimer" */);
-		this.joinOpTimer.clear();
+	private stopjoinTimer() {
+		assert(this.joinTimer.hasTimer, 0x235 /* "no joinTimer" */);
+		this.joinTimer.clear();
 	}
 
 	private get waitingForLeaveOp() {
@@ -402,7 +429,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 	}
 
 	public dispose() {
-		assert(!this.joinOpTimer.hasTimer, 0x2a5 /* "join timer" */);
+		assert(!this.joinTimer.hasTimer, 0x2a5 /* "join timer" */);
 		this.prevClientLeftTimer.clear();
 	}
 
@@ -418,15 +445,18 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 	private receivedAddMemberEvent(clientId: string) {
 		// This is the only one that requires the pending client ID
 		if (clientId === this.pendingClientId) {
-			if (this.joinOpTimer.hasTimer) {
-				this.stopJoinOpTimer();
-			} else if (this.shouldWaitForJoinSignal()) {
+			if (this.joinTimer.hasTimer) {
+				this.stopjoinTimer();
+			} else if (this.shouldWaitForSelf()) {
 				// timer has already fired, meaning it took too long to get join op/signal.
 				// Record how long it actually took to recover.
 				// This is generic event, as it by itself is not an error.
 				// We also have a case where NoJoinOp happens during container boot (we do not report it as error in such case),
 				// if this log statement happens after boot - we do not want to consider it error case.
-				this.handler.logConnectionIssue("ReceivedJoinOp", "generic");
+				this.handler.logConnectionIssue(
+					this.connection?.mode === "read" ? "ReceivedJoinSignal" : "ReceivedJoinOp", // eventName
+					"generic", // category
+				);
 			}
 			// Start the event in case we are waiting for leave or timeout.
 			if (this.waitingForLeaveOp) {
@@ -514,7 +544,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 			this._connectionState === ConnectionState.EstablishingConnection,
 			0x6d3 /* Connection state should be EstablishingConnection */,
 		);
-		assert(this.connection === undefined, 0x6d4 /* No connetion should be present */);
+		assert(this.connection === undefined, 0x6d4 /* No connection should be present */);
 		const oldState = this._connectionState;
 		this._connectionState = ConnectionState.Disconnected;
 		this.handler.connectionStateChanged(ConnectionState.Disconnected, oldState, reason);
@@ -529,7 +559,12 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		});
 	}
 
-	private shouldWaitForJoinSignal() {
+	/**
+	 * Tells if need to wait for "self" to show up in audience.
+	 * @returns - true if we should wait for "self" to appear in audience.
+	 * false is returned only for "read" connections, and only if this.readClientsWaitForJoinSignal is false.
+	 */
+	private shouldWaitForSelf() {
 		assert(
 			this.connection !== undefined,
 			0x4b4 /* all callers call here with active connection */,
@@ -576,10 +611,10 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		//    We are fetching ops from storage in parallel to connecting to Relay Service,
 		//    and given async processes, it's possible that we have already processed our own join message before
 		//    connection was fully established.
-		if (!this.hasMember(this._pendingClientId) && this.shouldWaitForJoinSignal()) {
+		if (!this.hasMember(this._pendingClientId) && this.shouldWaitForSelf()) {
 			// We are waiting for our own join op / signal. When it is processed
 			// we'll attempt to transition to Connected state via receivedAddMemberEvent() flow.
-			this.startJoinOpTimer();
+			this.startjoinTimer();
 		} else if (!this.waitingForLeaveOp) {
 			// We're not waiting for Join or Leave op (if read-only connection those don't even apply),
 			// go ahead and declare the state to be Connected!
@@ -628,8 +663,8 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 			// Clear pending state immediately to prepare for reconnect
 			this._pendingClientId = undefined;
 
-			if (this.joinOpTimer.hasTimer) {
-				this.stopJoinOpTimer();
+			if (this.joinTimer.hasTimer) {
+				this.stopjoinTimer();
 			}
 
 			// Only wait for "leave" message if the connected client exists in the quorum and had some non-acked ops
@@ -660,15 +695,10 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		this.handler.connectionStateChanged(this._connectionState, oldState, reason);
 	}
 
-	// Helper method to switch between quorum and audience.
-	// Old design was checking only quorum for "write" clients.
-	// Latest change checks audience for all types of connections.
 	protected get membership(): IMembership | undefined {
-		// We could always use audience here, and in practice it will probably be correct.
-		// (including case when this.readClientsWaitForJoinSignal === false).
-		// But only if it's superset of quorum, i.e. when filtered to "write" clients, they are always identical!
-		// It's safer to assume that we have bugs and engaging kill-bit switch should bring us back to well-known
-		// and tested state!
+		// We could always use audience here.
+		// This is true because Audience is a superset of quorum, i.e. when we filter Audience to "write" clients,
+		// it is exactly the same as quorum! Please see asserts in Audience callbacks setup by initProtocol() enforcing that.
 		return this.readClientsWaitForJoinSignal ? this.protocol?.audience : this.protocol?.quorum;
 	}
 
@@ -676,6 +706,9 @@ class ConnectionStateHandler implements IConnectionStateHandler {
 		this.protocol = protocol;
 
 		this.membership?.on("addMember", (clientId, details) => {
+			// This is very important constrain. We rely on it when testing presence of "self" in Audience.
+			// We do not want to move to "connected" state for "write" connections when JoinSignal shows up
+			// for "self" - we want to move to "connected" state only when "join" op is received.
 			assert(
 				(details as IClient).mode === "read" ||
 					protocol.quorum.getMember(clientId) !== undefined,