@fluidframework/id-compressor 2.0.0-dev-rc.1.0.0.224419

package/src/types/idCompressor.ts

@@ -0,0 +1,155 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { SessionSpaceCompressedId, OpSpaceCompressedId, SessionId, StableId } from "./identifiers";
+import {
+	IdCreationRange,
+	SerializedIdCompressorWithNoSession,
+	SerializedIdCompressorWithOngoingSession,
+} from "./persisted-types";
+
+/**
+ * @alpha
+ */
+export interface IIdCompressorCore {
+	/**
+	 * Returns a range of IDs created by this session in a format for sending to the server for finalizing.
+	 * The range will include all IDs generated via calls to `generateCompressedId` since the last time this method was called.
+	 * @returns the range of IDs, which may be empty. This range must be sent to the server for ordering before
+	 * it is finalized. Ranges must be sent to the server in the order that they are taken via calls to this method.
+	 */
+	takeNextCreationRange(): IdCreationRange;
+
+	/**
+	 * Finalizes the supplied range of IDs (which may be from either a remote or local session).
+	 * @param range - the range of session-local IDs to finalize.
+	 */
+	finalizeCreationRange(range: IdCreationRange): void;
+
+	/**
+	 * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
+	 * This includes finalized state as well as un-finalized state and is therefore suitable for use in offline scenarios.
+	 */
+	serialize(withSession: true): SerializedIdCompressorWithOngoingSession;
+
+	/**
+	 * Returns a persistable form of the current state of this `IdCompressor` which can be rehydrated via `IdCompressor.deserialize()`.
+	 * This only includes finalized state and is therefore suitable for use in summaries.
+	 */
+	serialize(withSession: false): SerializedIdCompressorWithNoSession;
+}
+
+/**
+ * A distributed UUID generator and compressor.
+ *
+ * Generates arbitrary non-colliding v4 UUIDs, called stable IDs, for multiple "sessions" (which can be distributed across the network),
+ * providing each session with the ability to map these UUIDs to `numbers`.
+ *
+ * A session is a unique identifier that denotes a single compressor. New IDs are created through a single compressor API
+ * which should then sent in ranges to the server for total ordering (and are subsequently relayed to other clients). When a new ID is
+ * created it is said to be created by the compressor's "local" session.
+ *
+ * For each stable ID created, two numeric IDs are provided by the compressor:
+ *
+ * 1. A session-local ID, which is stable for the lifetime of the session (which could be longer than that of the compressor object, as it may
+ * be serialized for offline usage). Available as soon as the stable ID is allocated. These IDs are session-unique and are thus only
+ * safely usable within the scope of the compressor that created it.
+ *
+ * 2. A final ID, which is stable across serialization and deserialization of an IdCompressor. Available as soon as the range containing
+ * the corresponding session-local ID is totally ordered (via consensus) with respect to other sessions' allocations.
+ * Final IDs are known to and publicly usable by any compressor that has received them.
+ *
+ * Compressors will allocate UUIDs in non-random ways to reduce entropy allowing for optimized storage of the data needed
+ * to map the UUIDs to the numbers.
+ *
+ * The following invariants are upheld by IdCompressor:
+ *
+ * 1. Session-local IDs will always decompress to the same UUIDs for the lifetime of the session.
+ *
+ * 2. Final IDs will always decompress to the same UUIDs.
+ *
+ * 3. After a server-processed range of session-local IDs (from any session) is received by a compressor, any of those session-local IDs may be
+ * translated by the compressor into the corresponding final ID. For any given session-local ID, this translation will always yield the
+ * same final ID.
+ *
+ * 4. A UUID will always compress into the same session-local ID for the lifetime of the session.
+ *
+ * Session-local IDs are sent across the wire in efficiently-represented ranges. These ranges are created by querying the compressor, and *must*
+ * be ordered (i.e. sent to the server) in the order they are created in order to preserve the above invariants.
+ *
+ * Session-local IDs can be used immediately after creation, but will eventually (after being sequenced) have a corresponding final ID. This
+ * could make reasoning about equality of those two forms difficult. For example, if a cache is keyed off of a
+ * session-local ID but is later queried using the final ID (which is semantically equal, as it decompresses to the same UUID/string) it will
+ * produce a cache miss. In order to make using collections of both remotely created and locally created IDs easy, regardless of whether the
+ * session-local IDs have been finalized, the compressor defines two "spaces" of IDs:
+ *
+ * 1. Session space: in this space, all IDs are normalized to their "most local form". This means that all IDs created by the local session
+ * will be in local form, regardless of if they have been finalized. Remotely created IDs, which could only have been received after
+ * finalizing and will never have a local form for the compressor, will of course be final IDs. This space should be used with consumer APIs
+ * and data structures, as the lifetime of the IDs is guaranteed to be the same as the compressor object. Care must be taken to not use
+ * these IDs across compressor objects, as the local IDs are specific to the compressor that created them.
+ *
+ * 2. Op space: in this space, all IDs are normalized to their "most final form". This means that all IDs except session-local IDs that
+ * have not yet been finalized will be in final ID form. This space is useful for serialization in ops (e.g. references), as other clients
+ * that receive them need not do any work to normalize them to *their* session-space in the common case. Note that IDs in op space may move
+ * out of Op space over time, namely, when a session-local ID in this space becomes finalized, and thereafter has a "more final form".
+ * Consequentially, it may be useful to restrict parameters of a persisted type to this space (to optimize perf), but it is potentially
+ * incorrect to use this type for a runtime variable. This is an asymmetry that does not affect session space, as local IDs are always as
+ * "local as possible".
+ *
+ * These two spaces naturally define a rule: consumers of compressed IDs should use session-space IDs, but serialized forms such as ops
+ * should use op-space IDs.
+ * @public
+ */
+export interface IIdCompressor {
+	localSessionId: SessionId;
+
+	/**
+	 * Generates a new compressed ID or returns an existing one.
+	 * This should ONLY be called to generate IDs for local operations.
+	 * @returns A new local ID in session space.
+	 */
+	generateCompressedId(): SessionSpaceCompressedId;
+	/**
+	 * Normalizes a session space ID into op space.
+	 * @param id - the local ID to normalize.
+	 * @returns the ID in op space.
+	 */
+	normalizeToOpSpace(id: SessionSpaceCompressedId): OpSpaceCompressedId;
+	/**
+	 * Normalizes an ID into session space.
+	 * @param id - the ID to normalize. If it is a local ID, it is assumed to have been created by the session corresponding
+	 * to `sessionId`.
+	 * @param originSessionId - the session from which `id` originated
+	 * @returns the session-space ID corresponding to `id`, which might not have been a final ID if the client that created it had not yet
+	 * finalized it. This can occur when a client references an ID during the window of time in which it is waiting to receive the ordered
+	 * range that contained it from the server.
+	 */
+	normalizeToSessionSpace(
+		id: OpSpaceCompressedId,
+		originSessionId: SessionId,
+	): SessionSpaceCompressedId;
+
+	/**
+	 * Decompresses a previously compressed ID into a UUID.
+	 * @param id - the compressed ID to be decompressed.
+	 * @returns the UUID associated with the compressed ID. Fails if the ID was not generated by this compressor.
+	 */
+	decompress(id: SessionSpaceCompressedId): StableId;
+
+	/**
+	 * Recompresses a UUID.
+	 * @param uncompressed - the UUID to recompress.
+	 * @returns the `CompressedId` associated with `uncompressed`. Fails if it has not been previously compressed by this compressor.
+	 */
+	recompress(uncompressed: StableId): SessionSpaceCompressedId;
+
+	/**
+	 * Attempts to recompresses a UUID.
+	 * @param uncompressed - the UUID to recompress,
+	 * @returns the `CompressedId` associated with `uncompressed` or undefined if it has not been previously compressed by this compressor.
+	 */
+	tryRecompress(uncompressed: StableId): SessionSpaceCompressedId | undefined;
+}

package/src/types/identifiers.ts

@@ -0,0 +1,39 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/**
+ * A compressed ID that has been normalized into "session space" (see `IdCompressor` for more).
+ * Consumer-facing APIs and data structures should use session-space IDs as their lifetime and equality is stable and tied to
+ * the scope of the session (i.e. compressor) that produced them.
+ * @public
+ */
+export type SessionSpaceCompressedId = number & {
+	readonly SessionUnique: "cea55054-6b82-4cbf-ad19-1fa645ea3b3e";
+};
+
+/**
+ * A compressed ID that has been normalized into "op space".
+ * Serialized/persisted structures (e.g. ops) should use op-space IDs as a performance optimization, as they require less normalizing when
+ * received by a remote client due to the fact that op space for a given compressor is session space for all other compressors.
+ * @public
+ */
+export type OpSpaceCompressedId = number & {
+	readonly OpNormalized: "9209432d-a959-4df7-b2ad-767ead4dbcae";
+};
+
+/**
+ * A version 4, variant 1 uuid (https://datatracker.ietf.org/doc/html/rfc4122).
+ * A 128-bit Universally Unique IDentifier. Represented here
+ * with a string of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx,
+ * where x is a lowercase hex digit.
+ * @public
+ */
+export type StableId = string & { readonly StableId: "53172b0d-a3d5-41ea-bd75-b43839c97f5a" };
+
+/**
+ * A StableId which is suitable for use as a session identifier
+ * @public
+ */
+export type SessionId = StableId & { readonly SessionId: "4498f850-e14e-4be9-8db0-89ec00997e58" };

package/src/types/index.ts

@@ -0,0 +1,15 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	IdCreationRange,
+	SerializedIdCompressor,
+	SerializedIdCompressorWithNoSession,
+	SerializedIdCompressorWithOngoingSession,
+} from "./persisted-types";
+
+export { IIdCompressorCore, IIdCompressor } from "./idCompressor";
+
+export { SessionSpaceCompressedId, OpSpaceCompressedId, SessionId, StableId } from "./identifiers";

package/src/types/persisted-types/0.0.1.ts

@@ -0,0 +1,58 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import type { SessionId } from "../identifiers";
+
+/**
+ * The serialized contents of an IdCompressor, suitable for persistence in a summary.
+ * @alpha
+ */
+export type SerializedIdCompressor = string & {
+	readonly _serializedIdCompressor: "8c73c57c-1cf4-4278-8915-6444cb4f6af5";
+};
+
+/**
+ * The serialized contents of an IdCompressor, suitable for persistence in a summary.
+ * @alpha
+ */
+export type SerializedIdCompressorWithNoSession = SerializedIdCompressor & {
+	readonly _noLocalState: "3aa2e1e8-cc28-4ea7-bc1a-a11dc3f26dfb";
+};
+
+/**
+ * The serialized contents of an IdCompressor, suitable for persistence in a summary.
+ * @alpha
+ */
+export type SerializedIdCompressorWithOngoingSession = SerializedIdCompressor & {
+	readonly _hasLocalState: "1281acae-6d14-47e7-bc92-71c8ee0819cb";
+};
+
+/**
+ * Data describing a range of session-local IDs (from a remote or local session).
+ *
+ * A range is composed of local IDs that were generated.
+ * @alpha
+ */
+export interface IdCreationRange {
+	readonly sessionId: SessionId;
+	readonly ids?: {
+		/**
+		 * The gen count of the first ID in the range created by `sessionId.`
+		 */
+		readonly firstGenCount: number;
+
+		/**
+		 * The number of IDs created in the range created by `sessionId.`
+		 */
+		readonly count: number;
+
+		/**
+		 * The size of the ID cluster to create if `count` overflows the existing cluster for
+		 * `sessionId`, if one exists. This request will be respected, and the size of the cluster
+		 * will be equal to overflow + `requestedClusterSize`.
+		 */
+		readonly requestedClusterSize: number;
+	};
+}

package/src/types/persisted-types/README.md

@@ -0,0 +1,3 @@
+# Persisted Types
+
+The types in this folder inherit all requirements outlined in [the experimental SharedTree persisted types README](../../../../../../experimental/dds/tree/src/persisted-types/README.md).

package/src/types/persisted-types/index.ts

@@ -0,0 +1,11 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	IdCreationRange,
+	SerializedIdCompressor,
+	SerializedIdCompressorWithNoSession,
+	SerializedIdCompressorWithOngoingSession,
+} from "./0.0.1";

package/src/utilities.ts

@@ -0,0 +1,191 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+/* eslint-disable no-bitwise */
+import { assert } from "@fluidframework/core-utils";
+import { v4 } from "uuid";
+import { SessionId, StableId } from "./types";
+import { LocalCompressedId, NumericUuid } from "./identifiers";
+
+const hexadecimalCharCodes = Array.from("09afAF").map((c) => c.charCodeAt(0)) as [
+	zero: number,
+	nine: number,
+	a: number,
+	f: number,
+	A: number,
+	F: number,
+];
+
+function isHexadecimalCharacter(charCode: number): boolean {
+	return (
+		(charCode >= hexadecimalCharCodes[0] && charCode <= hexadecimalCharCodes[1]) ||
+		(charCode >= hexadecimalCharCodes[2] && charCode <= hexadecimalCharCodes[3]) ||
+		(charCode >= hexadecimalCharCodes[4] && charCode <= hexadecimalCharCodes[5])
+	);
+}
+
+/**
+ * Generate a random session ID
+ * @alpha
+ */
+export function createSessionId(): SessionId {
+	return assertIsStableId(v4()) as SessionId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ * @internal
+ */
+export function assertIsStableId(stableId: string): StableId {
+	assert(isStableId(stableId), 0x4a3 /* Expected a StableId */);
+	return stableId;
+}
+
+/**
+ * Asserts that the given string is a stable ID.
+ */
+export function assertIsSessionId(stableId: string): SessionId {
+	assertIsStableId(stableId);
+	return stableId as SessionId;
+}
+
+/**
+ * Generate a random stable ID
+ * @internal
+ */
+export function generateStableId(): StableId {
+	return assertIsStableId(v4());
+}
+
+/**
+ * Returns true iff the given string is a valid Version 4, variant 2 UUID
+ * 'xxxxxxxx-xxxx-4xxx-vxxx-xxxxxxxxxxxx'
+ * @internal
+ */
+export function isStableId(str: string): str is StableId {
+	if (str.length !== 36) {
+		return false;
+	}
+
+	for (let i = 0; i < str.length; i++) {
+		switch (i) {
+			case 8:
+			case 13:
+			case 18:
+			case 23:
+				if (str.charAt(i) !== "-") {
+					return false;
+				}
+				break;
+
+			case 14:
+				if (str.charAt(i) !== "4") {
+					return false;
+				}
+				break;
+
+			case 19: {
+				const char = str.charAt(i);
+				if (char !== "8" && char !== "9" && char !== "a" && char !== "b") {
+					return false;
+				}
+				break;
+			}
+
+			default:
+				if (!isHexadecimalCharacter(str.charCodeAt(i))) {
+					return false;
+				}
+				break;
+		}
+	}
+
+	return true;
+}
+
+/**
+ * A numeric comparator used for sorting in ascending order.
+ *
+ * Handles +/-0 like Map: -0 is equal to +0.
+ */
+export function compareFiniteNumbers<T extends number>(a: T, b: T): number {
+	return a - b;
+}
+
+/**
+ * Compares strings lexically to form a strict partial ordering.
+ */
+export function compareStrings<T extends string>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+/**
+ * Compares bigints to form a strict partial ordering.
+ */
+export function compareBigints<T extends bigint>(a: T, b: T): number {
+	return a > b ? 1 : a === b ? 0 : -1;
+}
+
+export function genCountFromLocalId(localId: LocalCompressedId): number {
+	return -localId;
+}
+
+export function localIdFromGenCount(genCount: number): LocalCompressedId {
+	return -genCount as LocalCompressedId;
+}
+
+// xxxxxxxx-xxxx-Mxxx-Nxxx-xxxxxxxxxxxx
+const versionMask = 0x4n << (19n * 4n); // Version 4
+const variantMask = 0x8n << (15n * 4n); // Variant RFC4122 (1 0 x x)
+const upperMask = 0xffffffffffffn << (20n * 4n);
+// Upper mask when version/variant bits are removed
+const strippedUpperMask = upperMask >> 6n;
+const middieBittiesMask = 0xfffn << (16n * 4n);
+// Middie mask when version/variant bits are removed
+const strippedMiddieBittiesMask = middieBittiesMask >> 2n;
+// Note: leading character should be 3 to mask at 0011
+// The more-significant half of the N nibble is used to denote the variant (10xx)
+const lowerMask = 0x3fffffffffffffffn;
+
+export function numericUuidFromStableId(stableId: StableId): NumericUuid {
+	const uuidU128 = BigInt(`0x${stableId.replace(/-/g, "")}`);
+	const upperMasked = uuidU128 & upperMask;
+	const middieBittiesMasked = uuidU128 & middieBittiesMask;
+	const lowerMasked = uuidU128 & lowerMask;
+
+	const upperMaskedPlaced = upperMasked >> 6n;
+	const middieBittiesMaskedPlaced = middieBittiesMasked >> 2n;
+
+	const id = upperMaskedPlaced | middieBittiesMaskedPlaced | lowerMasked;
+	return id as NumericUuid;
+}
+
+export function stableIdFromNumericUuid(numericUuid: NumericUuid): StableId {
+	// bitwise reverse transform
+	const upperMasked = (numericUuid & strippedUpperMask) << 6n;
+	const middieBittiesMasked = (numericUuid & strippedMiddieBittiesMask) << 2n;
+	const lowerMasked = numericUuid & lowerMask;
+	const uuidU128 = upperMasked | versionMask | middieBittiesMasked | variantMask | lowerMasked;
+	// Pad to 32 characters, inserting leading zeroes if needed
+	const uuidString = uuidU128.toString(16).padStart(32, "0");
+	return `${uuidString.substring(0, 8)}-${uuidString.substring(8, 12)}-${uuidString.substring(
+		12,
+		16,
+	)}-${uuidString.substring(16, 20)}-${uuidString.substring(20, 32)}` as StableId;
+}
+
+export function offsetNumericUuid(numericUuid: NumericUuid, offset: number): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (numericUuid + BigInt(offset)) as NumericUuid;
+}
+
+export function subtractNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	return (a - b) as NumericUuid;
+}
+
+export function addNumericUuids(a: NumericUuid, b: NumericUuid): NumericUuid {
+	// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+	return (a + b) as NumericUuid;
+}

package/tsc-multi.test.json

@@ -0,0 +1,4 @@
+{
+	"targets": [{ "extname": ".cjs", "module": "CommonJS", "moduleResolution": "Node16" }],
+	"projects": ["./tsconfig.json", "./src/test/tsconfig.json"]
+}

package/tsconfig.json

@@ -0,0 +1,12 @@
+{
+	"extends": [
+		"../../../common/build/build-common/tsconfig.base.json",
+		"../../../common/build/build-common/tsconfig.cjs.json",
+	],
+	"compilerOptions": {
+		"rootDir": "./src",
+		"outDir": "./dist",
+	},
+	"include": ["src/**/*"],
+	"exclude": ["src/test/**/*"],
+}