@fluidframework/id-compressor 2.0.0-rc.3.0.3 → 2.0.0-rc.3.0.4

package/src/idCompressor.ts

@@ -6,8 +6,11 @@
 import { bufferToString, stringToBuffer } from "@fluid-internal/client-utils";
 import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
 import { assert } from "@fluidframework/core-utils/internal";
-import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
-import { createChildLogger } from "@fluidframework/telemetry-utils/internal";
+import {
+	ITelemetryLoggerExt,
+	LoggingError,
+	createChildLogger,
+} from "@fluidframework/telemetry-utils/internal";
 
 import { FinalSpace } from "./finalSpace.js";
 import { FinalCompressedId, LocalCompressedId, NumericUuid, isFinalId } from "./identifiers.js";
@@ -58,6 +61,13 @@ import {
  */
 const currentWrittenVersion = 2.0;
 
+function rangeFinalizationError(expectedStart: number, actualStart: number): LoggingError {
+	return new LoggingError("Ranges finalized out of order", {
+		expectedStart,
+		actualStart,
+	});
+}
+
 /**
  * See {@link IIdCompressor} and {@link IIdCompressorCore}
  */
@@ -223,14 +233,49 @@ export class IdCompressor implements IIdCompressor, IIdCompressorCore {
 				),
 			},
 		};
+		return this.updateToRange(range);
+	}
+
+	public takeUnfinalizedCreationRange(): IdCreationRange {
+		const lastLocalCluster = this.localSession.getLastCluster();
+		let count: number;
+		let firstGenCount: number;
+		if (lastLocalCluster === undefined) {
+			firstGenCount = 1;
+			count = this.localGenCount;
+		} else {
+			firstGenCount = genCountFromLocalId(
+				(lastLocalCluster.baseLocalId - lastLocalCluster.count) as LocalCompressedId,
+			);
+			count = this.localGenCount - firstGenCount + 1;
+		}
+
+		if (count === 0) {
+			return {
+				sessionId: this.localSessionId,
+			};
+		}
+
+		const range: IdCreationRange = {
+			ids: {
+				count,
+				firstGenCount,
+				localIdRanges: this.normalizer.getRangesBetween(firstGenCount, this.localGenCount),
+				requestedClusterSize: this.nextRequestedClusterSize,
+			},
+			sessionId: this.localSessionId,
+		};
+		return this.updateToRange(range);
+	}
+
+	private updateToRange(range: IdCreationRange): IdCreationRange {
 		this.nextRangeBaseGenCount = this.localGenCount + 1;
-		IdCompressor.assertValidRange(range);
-		return range;
+		return IdCompressor.assertValidRange(range);
 	}
 
-	private static assertValidRange(range: IdCreationRange): void {
+	private static assertValidRange(range: IdCreationRange): IdCreationRange {
 		if (range.ids === undefined) {
-			return;
+			return range;
 		}
 		const { count, requestedClusterSize } = range.ids;
 		assert(count > 0, 0x755 /* Malformed ID Range. */);
@@ -239,6 +284,7 @@ export class IdCompressor implements IIdCompressor, IIdCompressorCore {
 			requestedClusterSize <= IdCompressor.maxClusterSize,
 			0x877 /* Clusters must not exceed max cluster size. */,
 		);
+		return range;
 	}
 
 	public finalizeCreationRange(range: IdCreationRange): void {
@@ -261,7 +307,7 @@ export class IdCompressor implements IIdCompressor, IIdCompressorCore {
 		if (lastCluster === undefined) {
 			// This is the first cluster in the session space
 			if (rangeBaseLocal !== -1) {
-				throw new Error("Ranges finalized out of order.");
+				throw rangeFinalizationError(-1, rangeBaseLocal);
 			}
 			lastCluster = this.addEmptyCluster(session, requestedClusterSize + count);
 			if (isLocal) {
@@ -274,7 +320,10 @@ export class IdCompressor implements IIdCompressor, IIdCompressorCore {
 
 		const remainingCapacity = lastCluster.capacity - lastCluster.count;
 		if (lastCluster.baseLocalId - lastCluster.count !== rangeBaseLocal) {
-			throw new Error("Ranges finalized out of order.");
+			throw rangeFinalizationError(
+				lastCluster.baseLocalId - lastCluster.count,
+				rangeBaseLocal,
+			);
 		}
 
 		if (remainingCapacity >= count) {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/id-compressor";
-export const pkgVersion = "2.0.0-rc.3.0.3";
+export const pkgVersion = "2.0.0-rc.3.0.4";

package/src/test/idCompressorTestUtilities.ts

@@ -33,6 +33,7 @@ import {
 } from "../index.js";
 import { assertIsSessionId, createSessionId, localIdFromGenCount } from "../utilities.js";
 
+import { SessionSpaceNormalizer } from "../sessionSpaceNormalizer.js";
 import {
 	FinalCompressedId,
 	ReadonlyIdCompressor,
@@ -286,8 +287,7 @@ export class IdCompressorTestNetwork {
 	 * Changes the capacity request amount for a client. It will take effect immediately.
 	 */
 	public changeCapacity(client: Client, newClusterCapacity: number): void {
-		// eslint-disable-next-line @typescript-eslint/dot-notation
-		this.compressors.get(client)["nextRequestedClusterSize"] = newClusterCapacity;
+		changeCapacity(this.compressors.get(client), newClusterCapacity);
 	}
 
 	private addNewId(
@@ -427,26 +427,89 @@ export class IdCompressorTestNetwork {
 			(client) => [this.compressors.get(client), this.getSequencedIdLog(client)] as const,
 		);
 
-		// Ensure creation ranges for clients we track contain the correct local ID ranges
-		this.serverOperations.forEach(([range, opSpaceIds, clientFrom]) => {
+		const getLocalIdsInRange = (
+			range: IdCreationRange,
+			opSpaceIds?: OpSpaceCompressedId[],
+		): Set<SessionSpaceCompressedId> => {
 			const localIdsInCreationRange = new Set<SessionSpaceCompressedId>();
-			if (clientFrom !== OriginatingClient.Remote) {
-				const ids = range.ids;
-				if (ids !== undefined) {
-					const { firstGenCount, localIdRanges } = ids;
-					for (const [genCount, count] of localIdRanges) {
-						for (let g = genCount; g < genCount + count; g++) {
-							const local = localIdFromGenCount(g);
+			const ids = range.ids;
+			if (ids !== undefined) {
+				const { firstGenCount, localIdRanges } = ids;
+				for (const [genCount, count] of localIdRanges) {
+					for (let g = genCount; g < genCount + count; g++) {
+						const local = localIdFromGenCount(g);
+						if (opSpaceIds) {
 							assert.strictEqual(opSpaceIds[g - firstGenCount], local);
-							localIdsInCreationRange.add(local);
 						}
+						localIdsInCreationRange.add(local);
 					}
 				}
+			}
+			return localIdsInCreationRange;
+		};
+
+		// Ensure creation ranges for clients we track contain the correct local ID ranges
+		this.serverOperations.forEach(([range, opSpaceIds, clientFrom]) => {
+			if (clientFrom !== OriginatingClient.Remote) {
+				const localIdsInCreationRange = getLocalIdsInRange(range, opSpaceIds);
+				let localCount = 0;
 				opSpaceIds.forEach((id) => {
 					if (isLocalId(id)) {
+						localCount++;
 						assert(localIdsInCreationRange.has(id), "Local ID not in creation range");
 					}
 				});
+				assert.strictEqual(
+					localCount,
+					localIdsInCreationRange.size,
+					"Local ID count mismatch",
+				);
+			}
+		});
+
+		const undeliveredRanges = new Map<Client, IdCreationRange[]>();
+		this.clientProgress.forEach((progress, client) => {
+			const ranges = this.serverOperations
+				.slice(progress)
+				.filter((op) => op[2] === client)
+				.map(([range]) => range);
+			undeliveredRanges.set(client, ranges);
+		});
+		undeliveredRanges.forEach((ranges, client) => {
+			const compressor = this.compressors.get(client);
+			let firstGenCount: number | undefined;
+			let totalCount = 0;
+			const unionedLocalRanges = new SessionSpaceNormalizer();
+			ranges.forEach((range) => {
+				assert(range.sessionId === compressor.localSessionId);
+				if (range.ids !== undefined) {
+					// initialize firstGenCount if not set
+					if (firstGenCount === undefined) {
+						firstGenCount = range.ids.firstGenCount;
+					}
+					totalCount += range.ids.count;
+					range.ids.localIdRanges.forEach(([genCount, count]) => {
+						unionedLocalRanges.addLocalRange(genCount, count);
+					});
+				}
+			});
+
+			const retakenRange = compressor.takeUnfinalizedCreationRange();
+			if (retakenRange.ids !== undefined) {
+				const retakenLocalIds = new SessionSpaceNormalizer();
+				retakenRange.ids.localIdRanges.forEach(([genCount, count]) => {
+					retakenLocalIds.addLocalRange(genCount, count);
+				});
+				assert.strictEqual(
+					retakenLocalIds.equals(unionedLocalRanges),
+					true,
+					"Local ID ranges mismatch",
+				);
+				assert.strictEqual(retakenRange.ids.count, totalCount, "Count mismatch");
+				assert.strictEqual(retakenRange.ids.firstGenCount, firstGenCount, "Count mismatch");
+			} else {
+				assert.strictEqual(totalCount, 0);
+				assert.strictEqual(unionedLocalRanges.idRanges.size, 0);
 			}
 		});
 
@@ -588,6 +651,11 @@ export class IdCompressorTestNetwork {
 	}
 }
 
+function changeCapacity(compressor: IdCompressor, newClusterCapacity: number): void {
+	// eslint-disable-next-line @typescript-eslint/dot-notation
+	compressor["nextRequestedClusterSize"] = newClusterCapacity;
+}
+
 /**
  * Roundtrips the supplied compressor through serialization and deserialization.
  */
@@ -608,13 +676,21 @@ export function roundtrip(
 	compressor: ReadonlyIdCompressor,
 	withSession: boolean,
 ): [SerializedIdCompressorWithOngoingSession | SerializedIdCompressorWithNoSession, IdCompressor] {
+	// preserve the capacity request as this property is normally private and resets
+	// to a default on construction (deserialization)
+	// eslint-disable-next-line @typescript-eslint/dot-notation
+	const capacity = compressor["nextRequestedClusterSize"];
 	if (withSession) {
 		const serialized = compressor.serialize(withSession);
-		return [serialized, IdCompressor.deserialize(serialized)];
+		const roundtripped = IdCompressor.deserialize(serialized);
+		changeCapacity(roundtripped, capacity);
+		return [serialized, roundtripped];
+	} else {
+		const nonLocalSerialized = compressor.serialize(withSession);
+		const roundtripped = IdCompressor.deserialize(nonLocalSerialized, createSessionId());
+		changeCapacity(roundtripped, capacity);
+		return [nonLocalSerialized, roundtripped];
 	}
-
-	const nonLocalSerialized = compressor.serialize(withSession);
-	return [nonLocalSerialized, IdCompressor.deserialize(nonLocalSerialized, createSessionId())];
 }
 
 /**
@@ -811,13 +887,13 @@ export function makeOpGenerator(
 		return { type: "reconnect", client: random.pick(activeClients) };
 	}
 
-	const allocationWeight = 16;
+	const allocationWeight = 20;
 	return interleave(
 		createWeightedGenerator<Operation, FuzzTestState>([
 			[changeCapacityGenerator, 1],
 			[allocateIdsGenerator, Math.round(allocationWeight * (1 - outsideAllocationFraction))],
 			[allocateOutsideIdsGenerator, Math.round(allocationWeight * outsideAllocationFraction)],
-			[deliverAllOperationsGenerator, 2],
+			[deliverAllOperationsGenerator, 1],
 			[deliverSomeOperationsGenerator, 6],
 			[reconnectGenerator, 1],
 		]),
@@ -889,7 +965,6 @@ export function performFuzzActions(
 				return state;
 			},
 			validate: (state) => {
-				network.deliverOperations(DestinationClient.All);
 				validator?.(network);
 				return state;
 			},

package/src/types/idCompressor.ts

@@ -80,12 +80,23 @@ import {
 export interface IIdCompressorCore {
 	/**
 	 * Returns a range of IDs created by this session in a format for sending to the server for finalizing.
-	 * The range will include all IDs generated via calls to `generateCompressedId` since the last time this method was called.
+	 * The range will include all IDs generated via calls to `generateCompressedId` since the last time a
+	 * range was taken (via this method or `takeUnfinalizedCreationRange`).
 	 * @returns the range of IDs, which may be empty. This range must be sent to the server for ordering before
 	 * it is finalized. Ranges must be sent to the server in the order that they are taken via calls to this method.
 	 */
 	takeNextCreationRange(): IdCreationRange;
 
+	/**
+	 * Returns a range of IDs created by this session in a format for sending to the server for finalizing.
+	 * The range will include all unfinalized IDs generated via calls to `generateCompressedId`.
+	 * @returns the range of IDs, which may be empty. This range must be sent to the server for ordering before
+	 * it is finalized. Ranges must be sent to the server in the order that they are taken via calls to this method.
+	 * Note: after finalizing the range returned by this method, finalizing any ranges that had been previously taken
+	 * will result in an error.
+	 */
+	takeUnfinalizedCreationRange(): IdCreationRange;
+
 	/**
 	 * Finalizes the supplied range of IDs (which may be from either a remote or local session).
 	 * @param range - the range of session-local IDs to finalize.