@fluidframework/task-manager 2.53.1 → 2.61.0-355054

package/src/taskManager.ts

@@ -4,8 +4,11 @@
  */
 
 import { EventEmitter } from "@fluid-internal/client-utils";
-import type { ReadOnlyInfo } from "@fluidframework/container-definitions/internal";
-import { assert, unreachableCase } from "@fluidframework/core-utils/internal";
+import {
+	AttachState,
+	type ReadOnlyInfo,
+} from "@fluidframework/container-definitions/internal";
+import { assert } from "@fluidframework/core-utils/internal";
 import type {
 	IChannelAttributes,
 	IFluidDataStoreRuntime,
@@ -51,6 +54,18 @@ interface IPendingOp {
 	messageId: number;
 }
 
+function assertIsTaskManagerOperation(op: unknown): asserts op is ITaskManagerOperation {
+	assert(
+		typeof op === "object" &&
+			op !== null &&
+			"taskId" in op &&
+			typeof op.taskId === "string" &&
+			"type" in op &&
+			(op.type === "volunteer" || op.type === "abandon" || op.type === "complete"),
+		"Not a TaskManager operation",
+	);
+}
+
 const snapshotFileName = "header";
 
 /**
@@ -62,8 +77,7 @@ const placeholderClientId = "placeholder";
  * {@inheritDoc ITaskManager}
  *
  * @sealed
- * @legacy
- * @alpha
+ * @legacy @beta
  */
 export class TaskManagerClass
 	extends SharedObject<ITaskManagerEvents>
@@ -85,23 +99,20 @@ export class TaskManagerClass
 	private readonly connectionWatcher: EventEmitter = new EventEmitter();
 	// completedWatcher emits an event whenever the local client receives a completed op.
 	private readonly completedWatcher: EventEmitter = new EventEmitter();
+	// rollbackWatcher emits an event whenever a pending op is rolled back.
+	private readonly rollbackWatcher: EventEmitter = new EventEmitter();
 
-	private messageId: number = -1;
+	private nextPendingMessageId: number = 0;
 	/**
 	 * Tracks the most recent pending op for a given task
 	 */
-	private readonly latestPendingOps = new Map<string, IPendingOp>();
+	private readonly latestPendingOps = new Map<string, IPendingOp[]>();
 
 	/**
 	 * Tracks tasks that are this client is currently subscribed to.
 	 */
 	private readonly subscribedTasks = new Set<string>();
 
-	/**
-	 * Map to track tasks that have pending complete ops.
-	 */
-	private readonly pendingCompletedTasks = new Map<string, number[]>();
-
 	/**
 	 * Returns the clientId. Will return a placeholder if the runtime is detached and not yet assigned a clientId.
 	 */
@@ -132,16 +143,17 @@ export class TaskManagerClass
 
 		this.opWatcher.on(
 			"volunteer",
-			(taskId: string, clientId: string, local: boolean, messageId: number) => {
-				// We're tracking local ops from this connection. Filter out local ops during "connecting"
-				// state since these were sent on the prior connection and were already cleared from the latestPendingOps.
-				if (runtime.connected && local) {
-					const pendingOp = this.latestPendingOps.get(taskId);
-					assert(pendingOp !== undefined, 0x07b /* "Unexpected op" */);
-					// Need to check the id, since it's possible to volunteer and abandon multiple times before the acks
-					if (messageId === pendingOp.messageId) {
-						assert(pendingOp.type === "volunteer", 0x07c /* "Unexpected op type" */);
-						// Delete the pending, because we no longer have an outstanding op
+			(taskId: string, clientId: string, local: boolean, messageId: number | undefined) => {
+				if (local) {
+					const latestPendingOps = this.latestPendingOps.get(taskId);
+					assert(latestPendingOps !== undefined, "No pending ops for task");
+					const pendingOp = latestPendingOps.shift();
+					assert(
+						pendingOp !== undefined && pendingOp.messageId === messageId,
+						"Unexpected op",
+					);
+					assert(pendingOp.type === "volunteer", 0x07c /* "Unexpected op type" */);
+					if (latestPendingOps.length === 0) {
 						this.latestPendingOps.delete(taskId);
 					}
 				}
@@ -152,16 +164,20 @@ export class TaskManagerClass
 
 		this.opWatcher.on(
 			"abandon",
-			(taskId: string, clientId: string, local: boolean, messageId: number) => {
-				if (runtime.connected && local) {
-					const pendingOp = this.latestPendingOps.get(taskId);
-					assert(pendingOp !== undefined, 0x07d /* "Unexpected op" */);
-					// Need to check the id, since it's possible to abandon and volunteer multiple times before the acks
-					if (messageId === pendingOp.messageId) {
-						assert(pendingOp.type === "abandon", 0x07e /* "Unexpected op type" */);
-						// Delete the pending, because we no longer have an outstanding op
+			(taskId: string, clientId: string, local: boolean, messageId: number | undefined) => {
+				if (local) {
+					const latestPendingOps = this.latestPendingOps.get(taskId);
+					assert(latestPendingOps !== undefined, "No pending ops for task");
+					const pendingOp = latestPendingOps.shift();
+					assert(
+						pendingOp !== undefined && pendingOp.messageId === messageId,
+						"Unexpected op",
+					);
+					assert(pendingOp.type === "abandon", 0x07e /* "Unexpected op type" */);
+					if (latestPendingOps.length === 0) {
 						this.latestPendingOps.delete(taskId);
 					}
+					this.abandonWatcher.emit("abandon", taskId, messageId);
 				}
 
 				this.removeClientFromQueue(taskId, clientId);
@@ -170,33 +186,24 @@ export class TaskManagerClass
 
 		this.opWatcher.on(
 			"complete",
-			(taskId: string, clientId: string, local: boolean, messageId: number) => {
-				if (runtime.connected && local) {
-					const pendingOp = this.latestPendingOps.get(taskId);
-					assert(pendingOp !== undefined, 0x400 /* Unexpected op */);
-					// Need to check the id, since it's possible to complete multiple times before the acks
-					if (messageId === pendingOp.messageId) {
-						assert(pendingOp.type === "complete", 0x401 /* Unexpected op type */);
-						// Delete the pending, because we no longer have an outstanding op
-						this.latestPendingOps.delete(taskId);
-					}
-
-					// Remove complete op from this.pendingCompletedTasks
-					const pendingIds = this.pendingCompletedTasks.get(taskId);
+			(taskId: string, clientId: string, local: boolean, messageId: number | undefined) => {
+				if (local) {
+					const latestPendingOps = this.latestPendingOps.get(taskId);
+					assert(latestPendingOps !== undefined, "No pending ops for task");
+					const pendingOp = latestPendingOps.shift();
 					assert(
-						pendingIds !== undefined && pendingIds.length > 0,
-						0x402 /* pendingIds is empty */,
+						pendingOp !== undefined && pendingOp.messageId === messageId,
+						"Unexpected op",
 					);
-					const removed = pendingIds.shift();
-					assert(removed === messageId, 0x403 /* Removed complete op id does not match */);
+					assert(pendingOp.type === "complete", 0x401 /* Unexpected op type */);
+					if (latestPendingOps.length === 0) {
+						this.latestPendingOps.delete(taskId);
+					}
 				}
 
-				// For clients in queue, we need to remove them from the queue and raise the proper events.
-				if (!local) {
-					this.taskQueues.delete(taskId);
-					this.completedWatcher.emit("completed", taskId);
-					this.emit("completed", taskId);
-				}
+				this.taskQueues.delete(taskId);
+				this.completedWatcher.emit("completed", taskId, messageId);
+				this.emit("completed", taskId);
 			},
 		);
 
@@ -229,18 +236,16 @@ export class TaskManagerClass
 		this.connectionWatcher.on("disconnect", () => {
 			assert(this.clientId !== undefined, 0x1d3 /* "Missing client id on disconnect" */);
 
-			// We don't modify the taskQueues on disconnect (they still reflect the latest known consensus state).
-			// After reconnect these will get cleaned up by observing the clientLeaves.
-			// However we do need to recognize that we lost the lock if we had it.  Calls to .queued() and
-			// .assigned() are also connection-state-aware to be consistent.
+			// Emit "lost" for any tasks we were assigned to.
 			for (const [taskId, clientQueue] of this.taskQueues.entries()) {
 				if (this.isAttached() && clientQueue[0] === this.clientId) {
 					this.emit("lost", taskId);
 				}
 			}
 
-			// All of our outstanding ops will be for the old clientId even if they get ack'd
-			this.latestPendingOps.clear();
+			// Remove this client from all queues to reflect the new state, since being disconnected automatically removes
+			// this client from all queues.
+			this.removeClientFromAllQueues(this.clientId);
 		});
 	}
 
@@ -251,10 +256,15 @@ export class TaskManagerClass
 		};
 		const pendingOp: IPendingOp = {
 			type: "volunteer",
-			messageId: ++this.messageId,
+			messageId: this.nextPendingMessageId++,
 		};
 		this.submitLocalMessage(op, pendingOp.messageId);
-		this.latestPendingOps.set(taskId, pendingOp);
+		const latestPendingOps = this.latestPendingOps.get(taskId);
+		if (latestPendingOps === undefined) {
+			this.latestPendingOps.set(taskId, [pendingOp]);
+		} else {
+			latestPendingOps.push(pendingOp);
+		}
 	}
 
 	private submitAbandonOp(taskId: string): void {
@@ -264,10 +274,15 @@ export class TaskManagerClass
 		};
 		const pendingOp: IPendingOp = {
 			type: "abandon",
-			messageId: ++this.messageId,
+			messageId: this.nextPendingMessageId++,
 		};
 		this.submitLocalMessage(op, pendingOp.messageId);
-		this.latestPendingOps.set(taskId, pendingOp);
+		const latestPendingOps = this.latestPendingOps.get(taskId);
+		if (latestPendingOps === undefined) {
+			this.latestPendingOps.set(taskId, [pendingOp]);
+		} else {
+			latestPendingOps.push(pendingOp);
+		}
 	}
 
 	private submitCompleteOp(taskId: string): void {
@@ -277,25 +292,26 @@ export class TaskManagerClass
 		};
 		const pendingOp: IPendingOp = {
 			type: "complete",
-			messageId: ++this.messageId,
+			messageId: this.nextPendingMessageId++,
 		};
 
-		if (this.pendingCompletedTasks.has(taskId)) {
-			this.pendingCompletedTasks.get(taskId)?.push(pendingOp.messageId);
+		this.submitLocalMessage(op, pendingOp.messageId);
+		const latestPendingOps = this.latestPendingOps.get(taskId);
+		if (latestPendingOps === undefined) {
+			this.latestPendingOps.set(taskId, [pendingOp]);
 		} else {
-			this.pendingCompletedTasks.set(taskId, [pendingOp.messageId]);
+			latestPendingOps.push(pendingOp);
 		}
-
-		this.submitLocalMessage(op, pendingOp.messageId);
-		this.latestPendingOps.set(taskId, pendingOp);
 	}
 
 	/**
 	 * {@inheritDoc ITaskManager.volunteerForTask}
 	 */
 	public async volunteerForTask(taskId: string): Promise<boolean> {
-		// If we have the lock, resolve immediately
-		if (this.assigned(taskId)) {
+		// If we are both queued and assigned, then we have the lock and do not
+		// have any pending abandon/complete ops. In this case we can resolve
+		// true immediately.
+		if (this.queuedOptimistically(taskId) && this.assigned(taskId)) {
 			return true;
 		}
 
@@ -307,7 +323,7 @@ export class TaskManagerClass
 			throw error;
 		}
 
-		if (!this.isAttached()) {
+		if (this.isDetached()) {
 			// Simulate auto-ack in detached scenario
 			assert(this.clientId !== undefined, 0x472 /* clientId should not be undefined */);
 			this.addClientToQueue(taskId, this.clientId);
@@ -320,62 +336,82 @@ export class TaskManagerClass
 
 		// This promise works even if we already have an outstanding volunteer op.
 		const lockAcquireP = new Promise<boolean>((resolve, reject) => {
+			// If we don't send an op (meaning the latest pending op is "volunteer"), nextPendingMessageId
+			// will be greater than that prior "volunteer" op's messageId.  This is OK because
+			// we only use it to filter stale abandon/complete, and not when determining if we
+			// acquired the lock.
+			const nextPendingMessageId = this.nextPendingMessageId;
+			const setupListeners = (): void => {
+				this.queueWatcher.on("queueChange", checkIfAcquiredLock);
+				this.abandonWatcher.on("abandon", checkIfAbandoned);
+				this.connectionWatcher.on("disconnect", rejectOnDisconnect);
+				this.completedWatcher.on("completed", checkIfCompleted);
+				this.rollbackWatcher.on("rollback", checkIfRolledBack);
+			};
+			const removeListeners = (): void => {
+				this.queueWatcher.off("queueChange", checkIfAcquiredLock);
+				this.abandonWatcher.off("abandon", checkIfAbandoned);
+				this.connectionWatcher.off("disconnect", rejectOnDisconnect);
+				this.completedWatcher.off("completed", checkIfCompleted);
+				this.rollbackWatcher.off("rollback", checkIfRolledBack);
+			};
+
 			const checkIfAcquiredLock = (eventTaskId: string): void => {
 				if (eventTaskId !== taskId) {
 					return;
 				}
-
 				// Also check pending ops here because it's possible we are currently in the queue from a previous
 				// lock attempt, but have an outstanding abandon AND the outstanding volunteer for this lock attempt.
 				// If we reach the head of the queue based on the previous lock attempt, we don't want to resolve.
-				if (this.assigned(taskId) && !this.latestPendingOps.has(taskId)) {
-					this.queueWatcher.off("queueChange", checkIfAcquiredLock);
-					this.abandonWatcher.off("abandon", checkIfAbandoned);
-					this.connectionWatcher.off("disconnect", rejectOnDisconnect);
-					this.completedWatcher.off("completed", checkIfCompleted);
+				if (this.assigned(taskId)) {
+					removeListeners();
 					resolve(true);
 				}
 			};
 
-			const checkIfAbandoned = (eventTaskId: string): void => {
+			const checkIfAbandoned = (eventTaskId: string, messageId: number | undefined): void => {
 				if (eventTaskId !== taskId) {
 					return;
 				}
-
-				this.queueWatcher.off("queueChange", checkIfAcquiredLock);
-				this.abandonWatcher.off("abandon", checkIfAbandoned);
-				this.connectionWatcher.off("disconnect", rejectOnDisconnect);
-				this.completedWatcher.off("completed", checkIfCompleted);
+				if (messageId !== undefined && messageId <= nextPendingMessageId) {
+					// Ignore abandon events that were for abandon ops that were sent prior to our current volunteer attempt.
+					return;
+				}
+				removeListeners();
 				reject(new Error("Abandoned before acquiring task assignment"));
 			};
 
 			const rejectOnDisconnect = (): void => {
-				this.queueWatcher.off("queueChange", checkIfAcquiredLock);
-				this.abandonWatcher.off("abandon", checkIfAbandoned);
-				this.connectionWatcher.off("disconnect", rejectOnDisconnect);
-				this.completedWatcher.off("completed", checkIfCompleted);
+				removeListeners();
 				reject(new Error("Disconnected before acquiring task assignment"));
 			};
 
-			const checkIfCompleted = (eventTaskId: string): void => {
+			const checkIfCompleted = (eventTaskId: string, messageId: number | undefined): void => {
 				if (eventTaskId !== taskId) {
 					return;
 				}
+				if (messageId !== undefined && messageId <= nextPendingMessageId) {
+					// Ignore abandon events that were for abandon ops that were sent prior to our current volunteer attempt.
+					return;
+				}
+				removeListeners();
+				resolve(false);
+			};
 
-				this.queueWatcher.off("queueChange", checkIfAcquiredLock);
-				this.abandonWatcher.off("abandon", checkIfAbandoned);
-				this.connectionWatcher.off("disconnect", rejectOnDisconnect);
-				this.completedWatcher.off("completed", checkIfCompleted);
+			const checkIfRolledBack = (eventTaskId: string): void => {
+				if (eventTaskId !== taskId) {
+					return;
+				}
+
+				removeListeners();
 				resolve(false);
 			};
 
-			this.queueWatcher.on("queueChange", checkIfAcquiredLock);
-			this.abandonWatcher.on("abandon", checkIfAbandoned);
-			this.connectionWatcher.on("disconnect", rejectOnDisconnect);
-			this.completedWatcher.on("completed", checkIfCompleted);
+			setupListeners();
 		});
 
-		if (!this.queued(taskId)) {
+		if (!this.queuedOptimistically(taskId)) {
+			// Only send the volunteer op if we are not already queued.
 			this.submitVolunteerOp(taskId);
 		}
 		return lockAcquireP;
@@ -393,46 +429,82 @@ export class TaskManagerClass
 			throw new Error("Attempted to subscribe with read-only permissions");
 		}
 
+		let volunteerOpMessageId: number | undefined;
+
 		const submitVolunteerOp = (): void => {
+			volunteerOpMessageId = this.nextPendingMessageId;
 			this.submitVolunteerOp(taskId);
 		};
 
+		const setupListeners = (): void => {
+			this.abandonWatcher.on("abandon", checkIfAbandoned);
+			this.connectionWatcher.on("disconnect", disconnectHandler);
+			this.completedWatcher.on("completed", checkIfCompleted);
+			this.rollbackWatcher.on("rollback", checkIfRolledBack);
+		};
+		const removeListeners = (): void => {
+			this.abandonWatcher.off("abandon", checkIfAbandoned);
+			this.connectionWatcher.off("disconnect", disconnectHandler);
+			this.connectionWatcher.off("connect", submitVolunteerOp);
+			this.completedWatcher.off("completed", checkIfCompleted);
+			this.rollbackWatcher.off("rollback", checkIfRolledBack);
+		};
+
 		const disconnectHandler = (): void => {
 			// Wait to be connected again and then re-submit volunteer op
 			this.connectionWatcher.once("connect", submitVolunteerOp);
 		};
 
-		const checkIfAbandoned = (eventTaskId: string): void => {
+		const checkIfAbandoned = (eventTaskId: string, messageId: number | undefined): void => {
 			if (eventTaskId !== taskId) {
 				return;
 			}
-
-			this.abandonWatcher.off("abandon", checkIfAbandoned);
-			this.connectionWatcher.off("disconnect", disconnectHandler);
-			this.connectionWatcher.off("connect", submitVolunteerOp);
-			this.completedWatcher.off("completed", checkIfCompleted);
-
+			// abandonWatcher emits twice for a local abandon() call. When initially called it
+			// will emit with undefined messageId. It will emit a second time when the op is
+			// ack'd and processed, this time with the messageId for the ack.
+			// This condition accounts ensures we don't ignore the initial abandon() emit and
+			// only ignore emits associated with ack'd abandon ops that were sent prior to the
+			// current volunteer attempt.
+			if (
+				messageId !== undefined &&
+				volunteerOpMessageId !== undefined &&
+				messageId <= volunteerOpMessageId
+			) {
+				// Ignore abandon events that were for abandon ops that were sent prior to our current volunteer attempt.
+				return;
+			}
+			removeListeners();
 			this.subscribedTasks.delete(taskId);
 		};
 
-		const checkIfCompleted = (eventTaskId: string): void => {
+		const checkIfCompleted = (eventTaskId: string, messageId: number | undefined): void => {
 			if (eventTaskId !== taskId) {
 				return;
 			}
+			if (
+				messageId !== undefined &&
+				volunteerOpMessageId !== undefined &&
+				messageId <= volunteerOpMessageId
+			) {
+				// Ignore abandon events that were for abandon ops that were sent prior to our current volunteer attempt.
+				return;
+			}
+			removeListeners();
+			this.subscribedTasks.delete(taskId);
+		};
 
-			this.abandonWatcher.off("abandon", checkIfAbandoned);
-			this.connectionWatcher.off("disconnect", disconnectHandler);
-			this.connectionWatcher.off("connect", submitVolunteerOp);
-			this.completedWatcher.off("completed", checkIfCompleted);
+		const checkIfRolledBack = (eventTaskId: string): void => {
+			if (eventTaskId !== taskId) {
+				return;
+			}
 
+			removeListeners();
 			this.subscribedTasks.delete(taskId);
 		};
 
-		this.abandonWatcher.on("abandon", checkIfAbandoned);
-		this.connectionWatcher.on("disconnect", disconnectHandler);
-		this.completedWatcher.on("completed", checkIfCompleted);
+		setupListeners();
 
-		if (!this.isAttached()) {
+		if (this.isDetached()) {
 			// Simulate auto-ack in detached scenario
 			assert(this.clientId !== undefined, 0x473 /* clientId should not be undefined */);
 			this.addClientToQueue(taskId, this.clientId);
@@ -440,22 +512,19 @@ export class TaskManagerClass
 			// a real clientId. At that point we should re-enter the queue with a real volunteer op (assuming we are
 			// connected).
 			this.runtime.once("attached", () => {
-				if (this.queued(taskId)) {
-					// If we are already queued, then we were able to replace the placeholderClientId with our real
-					// clientId and no action is required.
-					return;
-				} else if (this.connected) {
+				// We call scrubClientsNotInQuorum() in case our clientId changed during the attach process.
+				this.scrubClientsNotInQuorum();
+				if (this.connected) {
 					submitVolunteerOp();
 				} else {
-					this.connectionWatcher.once("connect", () => {
-						submitVolunteerOp();
-					});
+					this.connectionWatcher.once("connect", submitVolunteerOp);
 				}
 			});
 		} else if (!this.connected) {
 			// If we are disconnected (and attached), wait to be connected and submit volunteer op
 			disconnectHandler();
-		} else if (!this.assigned(taskId) && !this.queued(taskId)) {
+		} else if (!this.queuedOptimistically(taskId)) {
+			// We don't need to send a second volunteer op if we just sent one.
 			submitVolunteerOp();
 		}
 		this.subscribedTasks.add(taskId);
@@ -466,14 +535,13 @@ export class TaskManagerClass
 	 */
 	public abandon(taskId: string): void {
 		// Always allow abandon if the client is subscribed to allow clients to unsubscribe while disconnected.
-		// Otherwise, we should check to make sure the client is both connected queued for the task before sending an
-		// abandon op.
-		if (!this.subscribed(taskId) && !this.queued(taskId)) {
+		// Otherwise, we should check to make sure the client is optimistically queued for the task before trying to abandon.
+		if (!this.queuedOptimistically(taskId) && !this.subscribed(taskId)) {
 			// Nothing to do
 			return;
 		}
 
-		if (!this.isAttached()) {
+		if (this.isDetached()) {
 			// Simulate auto-ack in detached scenario
 			assert(this.clientId !== undefined, 0x474 /* clientId is undefined */);
 			this.removeClientFromQueue(taskId, this.clientId);
@@ -481,10 +549,7 @@ export class TaskManagerClass
 			return;
 		}
 
-		// If we're subscribed but not queued, we don't need to submit an abandon op (probably offline)
-		if (this.queued(taskId)) {
-			this.submitAbandonOp(taskId);
-		}
+		this.submitAbandonOp(taskId);
 		this.abandonWatcher.emit("abandon", taskId);
 	}
 
@@ -497,11 +562,7 @@ export class TaskManagerClass
 		}
 
 		const currentAssignee = this.taskQueues.get(taskId)?.[0];
-		return (
-			currentAssignee !== undefined &&
-			currentAssignee === this.clientId &&
-			!this.latestPendingOps.has(taskId)
-		);
+		return currentAssignee !== undefined && currentAssignee === this.clientId;
 	}
 
 	/**
@@ -513,14 +574,7 @@ export class TaskManagerClass
 		}
 
 		assert(this.clientId !== undefined, 0x07f /* "clientId undefined" */);
-
-		const clientQueue = this.taskQueues.get(taskId);
-		// If we have no queue for the taskId, then no one has signed up for it.
-		return (
-			((clientQueue?.includes(this.clientId) ?? false) &&
-				!this.latestPendingOps.has(taskId)) ||
-			this.latestPendingOps.get(taskId)?.type === "volunteer"
-		);
+		return this.taskQueues.get(taskId)?.includes(this.clientId) ?? false;
 	}
 
 	/**
@@ -540,16 +594,17 @@ export class TaskManagerClass
 
 		// If we are detached we will simulate auto-ack for the complete op. Therefore we only need to send the op if
 		// we are attached. Additionally, we don't need to check if we are connected while detached.
-		if (this.isAttached()) {
-			if (!this.connected) {
-				throw new Error("Attempted to complete task in disconnected state");
-			}
-			this.submitCompleteOp(taskId);
+		if (this.isDetached()) {
+			this.taskQueues.delete(taskId);
+			this.completedWatcher.emit("completed", taskId);
+			this.emit("completed", taskId);
+			return;
 		}
 
-		this.taskQueues.delete(taskId);
-		this.completedWatcher.emit("completed", taskId);
-		this.emit("completed", taskId);
+		if (!this.connected) {
+			throw new Error("Attempted to complete task in disconnected state");
+		}
+		this.submitCompleteOp(taskId);
 	}
 
 	/**
@@ -619,12 +674,24 @@ export class TaskManagerClass
 		this.connectionWatcher.emit("connect");
 	}
 
-	//
 	/**
 	 * Override resubmit core to avoid resubmission on reconnect.  On disconnect we accept our removal from the
 	 * queues, and leave it up to the user to decide whether they want to attempt to re-enter a queue on reconnect.
+	 * However, we do need to update latestPendingOps to account for the ops we will no longer be processing.
 	 */
-	protected reSubmitCore(): void {}
+	protected reSubmitCore(content: unknown, localOpMetadata: number): void {
+		assertIsTaskManagerOperation(content);
+		const pendingOps = this.latestPendingOps.get(content.taskId);
+		assert(pendingOps !== undefined, "No pending ops for task on resubmit attempt");
+		const pendingOpIndex = pendingOps.findIndex(
+			(op) => op.messageId === localOpMetadata && op.type === content.type,
+		);
+		assert(pendingOpIndex !== -1, "Could not match pending op on resubmit attempt");
+		pendingOps.splice(pendingOpIndex, 1);
+		if (pendingOps.length === 0) {
+			this.latestPendingOps.delete(content.taskId);
+		}
+	}
 
 	/**
 	 * Process a task manager operation
@@ -637,12 +704,12 @@ export class TaskManagerClass
 	protected processCore(
 		message: ISequencedDocumentMessage,
 		local: boolean,
-		localOpMetadata: unknown,
+		localOpMetadata: number | undefined,
 	): void {
 		// eslint-disable-next-line @typescript-eslint/no-unsafe-enum-comparison
 		if (message.type === MessageType.Operation) {
 			const op = message.contents as ITaskManagerOperation;
-			const messageId = localOpMetadata as number;
+			const messageId = localOpMetadata;
 
 			switch (op.type) {
 				case "volunteer": {
@@ -668,12 +735,6 @@ export class TaskManagerClass
 	}
 
 	private addClientToQueue(taskId: string, clientId: string): void {
-		const pendingIds = this.pendingCompletedTasks.get(taskId);
-		if (pendingIds !== undefined && pendingIds.length > 0) {
-			// Ignore the volunteer op if we know this task is about to be completed
-			return;
-		}
-
 		// Ensure that the clientId exists in the quorum, or it is placeholderClientId (detached scenario)
 		if (
 			this.runtime.getQuorum().getMembers().has(clientId) ||
@@ -686,6 +747,13 @@ export class TaskManagerClass
 				this.taskQueues.set(taskId, clientQueue);
 			}
 
+			if (clientQueue.includes(clientId)) {
+				// We shouldn't re-add the client if it's already in the queue.
+				// This may be possible in scenarios where a client was added in
+				// while detached.
+				return;
+			}
+
 			const oldLockHolder = clientQueue[0];
 			clientQueue.push(clientId);
 			const newLockHolder = clientQueue[0];
@@ -735,7 +803,12 @@ export class TaskManagerClass
 		for (const clientQueue of this.taskQueues.values()) {
 			const clientIdIndex = clientQueue.indexOf(placeholderClientId);
 			if (clientIdIndex !== -1) {
-				clientQueue[clientIdIndex] = this.runtime.clientId;
+				if (clientQueue.includes(this.runtime.clientId)) {
+					// If the real clientId is already in the queue, just remove the placeholder.
+					clientQueue.splice(clientIdIndex, 1);
+				} else {
+					clientQueue[clientIdIndex] = this.runtime.clientId;
+				}
 			}
 		}
 	}
@@ -759,24 +832,65 @@ export class TaskManagerClass
 		}
 	}
 
+	/**
+	 * Checks whether this client is currently assigned or in queue to become assigned, while also accounting
+	 * for the latest pending ops.
+	 */
+	private queuedOptimistically(taskId: string): boolean {
+		if (this.isAttached() && !this.connected) {
+			return false;
+		}
+
+		assert(this.clientId !== undefined, 0x07f /* "clientId undefined" */);
+
+		const inQueue = this.taskQueues.get(taskId)?.includes(this.clientId) ?? false;
+		const latestPendingOps = this.latestPendingOps.get(taskId);
+
+		const latestPendingOp =
+			latestPendingOps !== undefined && latestPendingOps.length > 0
+				? latestPendingOps[latestPendingOps.length - 1]
+				: undefined;
+		const isPendingVolunteer = latestPendingOp?.type === "volunteer";
+		const isPendingAbandonOrComplete =
+			latestPendingOp?.type === "abandon" || latestPendingOp?.type === "complete";
+		// We return true if the client is either in queue already or the latest pending op for this task is a volunteer op.
+		// But we should always return false if the latest pending op is an abandon or complete op.
+		return (inQueue && !isPendingAbandonOrComplete) || isPendingVolunteer;
+	}
+
+	/**
+	 * Returns true if the client is detached.
+	 * This is distinct from !this.isAttached() because `isAttached()` also checks if `this._isBoundToContext`
+	 * is true. We use `isDetached()` to determine if we should simulate auto-ack behavior for ops, which is
+	 * mainly concerned with if we have been assigned a real clientId yet.
+	 */
+	private isDetached(): boolean {
+		return this.runtime.attachState === AttachState.Detached;
+	}
+
 	protected applyStashedOp(content: unknown): void {
-		const taskOp: ITaskManagerOperation = content as ITaskManagerOperation;
-		switch (taskOp.type) {
-			case "abandon": {
-				this.abandon(taskOp.taskId);
-				break;
-			}
-			case "complete": {
-				this.complete(taskOp.taskId);
-				break;
-			}
-			case "volunteer": {
-				this.subscribeToTask(taskOp.taskId);
-				break;
-			}
-			default: {
-				unreachableCase(taskOp);
-			}
+		// We don't apply any stashed ops since during the rehydration process. Since we lose any assigned tasks
+		// during rehydration we cannot be assigned any tasks. Additionally, without the in-memory state of the
+		// previous dds, we also cannot re-volunteer based on a previous subscribeToTask() call. Since we are
+		// unable to be assigned to any tasks, there is no reason to process abandon/complete ops either.
+	}
+
+	/**
+	 * {@inheritDoc @fluidframework/shared-object-base#SharedObject.rollback}
+	 */
+	protected rollback(content: unknown, localOpMetadata: unknown): void {
+		assert(typeof localOpMetadata === "number", "Expect localOpMetadata to be a number");
+		assertIsTaskManagerOperation(content);
+		const latestPendingOps = this.latestPendingOps.get(content.taskId);
+		assert(latestPendingOps !== undefined, "No pending ops when trying to rollback");
+		const pendingOpToRollback = latestPendingOps.pop();
+		assert(
+			pendingOpToRollback !== undefined && pendingOpToRollback.messageId === localOpMetadata,
+			"pending op mismatch",
+		);
+		if (latestPendingOps.length === 0) {
+			this.latestPendingOps.delete(content.taskId);
 		}
+		this.rollbackWatcher.emit("rollback", content.taskId);
 	}
 }