@fluidframework/id-compressor 2.0.0-dev-rc.1.0.0.228517 → 2.0.0-dev-rc.2.0.0.245554

package/lib/test/idCompressorTestUtilities.js

@@ -0,0 +1,576 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { strict as assert } from "assert";
+import { createWeightedGenerator, interleave, makeRandom, performFuzzActions as performFuzzActionsBase, repeat, take, } from "@fluid-private/stochastic-test-utils";
+import { createIdCompressor, } from "..//index.js";
+import { IdCompressor } from "../idCompressor.js";
+import { assertIsSessionId, createSessionId } from "../utilities.js";
+import { getOrCreate, incrementStableId, isFinalId, isLocalId, fail, } from "./testCommon.js";
+/** Identifies a compressor in a network */
+export var Client;
+(function (Client) {
+    Client["Client1"] = "Client1";
+    Client["Client2"] = "Client2";
+    Client["Client3"] = "Client3";
+})(Client || (Client = {}));
+/** Identifies categories of compressors */
+export var MetaClient;
+(function (MetaClient) {
+    MetaClient["All"] = "All";
+})(MetaClient || (MetaClient = {}));
+/** Identifies a compressor inside the network but outside the three specially tracked clients. */
+export var OutsideClient;
+(function (OutsideClient) {
+    OutsideClient["Remote"] = "Remote";
+})(OutsideClient || (OutsideClient = {}));
+export const OriginatingClient = { ...Client, ...OutsideClient };
+export const DestinationClient = { ...Client, ...MetaClient };
+// eslint-disable-next-line @typescript-eslint/no-extraneous-class
+export class CompressorFactory {
+    /**
+     * Creates a new compressor with the supplied cluster capacity.
+     */
+    static createCompressor(client, clusterCapacity = 5, logger) {
+        return CompressorFactory.createCompressorWithSession(sessionIds.get(client), clusterCapacity, logger);
+    }
+    /**
+     * Creates a new compressor with the supplied cluster capacity.
+     */
+    static createCompressorWithSession(sessionId, clusterCapacity = 5, logger) {
+        const compressor = createIdCompressor(sessionId, logger);
+        // eslint-disable-next-line @typescript-eslint/dot-notation
+        compressor["nextRequestedClusterSize"] = clusterCapacity;
+        return compressor;
+    }
+}
+/**
+ * Utility for building a huge compressor.
+ * Build via the compressor factory.
+ */
+export function buildHugeCompressor(numSessions = 10000, capacity = 10, numClustersPerSession = 3) {
+    const compressor = CompressorFactory.createCompressorWithSession(createSessionId(), capacity);
+    const sessions = [];
+    for (let i = 0; i < numSessions; i++) {
+        sessions.push(createSessionId());
+    }
+    for (let i = 0; i < numSessions * numClustersPerSession; i++) {
+        const sessionId = sessions[i % numSessions];
+        if (Math.random() > 0.1) {
+            for (let j = 0; j < Math.round(capacity / 2); j++) {
+                compressor.generateCompressedId();
+            }
+            compressor.finalizeCreationRange(compressor.takeNextCreationRange());
+        }
+        compressor.finalizeCreationRange({
+            sessionId,
+            ids: {
+                firstGenCount: Math.floor(i / numSessions) * capacity + 1,
+                count: capacity,
+                requestedClusterSize: capacity,
+            },
+        });
+    }
+    return compressor;
+}
+function makeSessionIds() {
+    const stableIds = new Map();
+    const clients = Object.values(Client);
+    for (let i = 0; i < clients.length; i++) {
+        // Place session uuids roughly in the middle of uuid space to increase odds of encountering interesting
+        // orderings in sorted collections
+        const sessionId = assertIsSessionId(`88888888-8888-4888-b${i}88-888888888888`);
+        stableIds.set(clients[i], sessionId);
+    }
+    return stableIds;
+}
+/**
+ * An array of session ID strings corresponding to all non-local `Client` entries.
+ */
+export const sessionIds = makeSessionIds();
+/**
+ * Simulates a network of ID compressors.
+ * Not suitable for performance testing.
+ */
+export class IdCompressorTestNetwork {
+    constructor(initialClusterSize = 5) {
+        this.initialClusterSize = initialClusterSize;
+        /** The log of operations seen by the server so far. Append-only. */
+        this.serverOperations = [];
+        const compressors = new Map();
+        const clientProgress = new Map();
+        const clientIds = new Map();
+        const clientSequencedIds = new Map();
+        for (const client of Object.values(Client)) {
+            const compressor = CompressorFactory.createCompressor(client, initialClusterSize);
+            compressors.set(client, compressor);
+            clientProgress.set(client, 0);
+            clientIds.set(client, []);
+            clientSequencedIds.set(client, []);
+        }
+        this.compressors = compressors;
+        this.clientProgress = clientProgress;
+        this.idLogs = clientIds;
+        this.sequencedIdLogs = clientSequencedIds;
+    }
+    /**
+     * Returns the number of undelivered operations for the given client that are in flight in the network.
+     */
+    getPendingOperations(destination) {
+        return this.serverOperations.length - this.clientProgress.get(destination);
+    }
+    /**
+     * Returns an immutable handle to a compressor in the network.
+     */
+    getCompressor(client) {
+        const compressors = this.compressors;
+        const handler = {
+            get(_, property) {
+                return compressors.get(client)[property];
+            },
+            set(_, property, value) {
+                compressors.get(client)[property] = value;
+                return true;
+            },
+        };
+        return new Proxy({}, handler);
+    }
+    /**
+     * Returns a mutable handle to a compressor in the network. Use of mutation methods will break the network invariants and
+     * should only be used if the network will not be used again.
+     */
+    getCompressorUnsafe(client) {
+        return this.getCompressor(client);
+    }
+    /**
+     * Returns a mutable handle to a compressor in the network. Use of mutation methods will break the network invariants and
+     * should only be used if the network will not be used again. Additionally, the returned compressor will be invalidated/unusable
+     * if any network operations cause it to be regenerated (serialization/deserialization, etc.).
+     */
+    getCompressorUnsafeNoProxy(client) {
+        return this.compressors.get(client);
+    }
+    /**
+     * Returns data for all IDs created and received by this client, including ack's of their own (i.e. their own IDs will appear twice)
+     */
+    getIdLog(client) {
+        return this.idLogs.get(client);
+    }
+    /**
+     * Returns data for all IDs received by this client, including ack's of their own.
+     */
+    getSequencedIdLog(client) {
+        return this.sequencedIdLogs.get(client);
+    }
+    /**
+     * Get all compressors for the given destination
+     */
+    getTargetCompressors(clientTo) {
+        return clientTo === MetaClient.All
+            ? [...this.compressors.entries()]
+            : [[clientTo, this.getCompressor(clientTo)]];
+    }
+    /**
+     * Changes the capacity request amount for a client. It will take effect immediately.
+     */
+    changeCapacity(client, newClusterCapacity) {
+        // eslint-disable-next-line @typescript-eslint/dot-notation
+        this.compressors.get(client)["nextRequestedClusterSize"] = newClusterCapacity;
+    }
+    addNewId(client, id, originatingClient, sessionIdFrom, isSequenced) {
+        const idData = {
+            id,
+            originatingClient,
+            sessionId: sessionIdFrom,
+            isSequenced,
+        };
+        const clientIds = this.idLogs.get(client);
+        clientIds.push(idData);
+        if (isSequenced) {
+            const sequencedIds = this.sequencedIdLogs.get(client);
+            sequencedIds.push(idData);
+        }
+    }
+    /**
+     * Allocates a new range of local IDs and enqueues them for future delivery via a `testIdDelivery` action.
+     * Calls to this method determine the total order of delivery, regardless of when `deliverOperations` is called.
+     */
+    allocateAndSendIds(clientFrom, numIds) {
+        return this.allocateAndSendIdsFromRemoteClient(clientFrom, sessionIds.get(clientFrom), numIds);
+    }
+    /**
+     * Same contract as `allocateAndSendIds`, but the originating client will be a client with the supplied sessionId.
+     */
+    allocateAndSendIdsFromRemoteClient(clientFrom, sessionIdFrom, numIds) {
+        assert(numIds > 0, "Must allocate a non-zero number of IDs");
+        if (clientFrom === OriginatingClient.Remote) {
+            const range = {
+                sessionId: sessionIdFrom,
+                ids: {
+                    firstGenCount: 1,
+                    count: numIds,
+                    requestedClusterSize: this.getCompressor(Client.Client1)[
+                    // eslint-disable-next-line @typescript-eslint/dot-notation
+                    "nextRequestedClusterSize"],
+                },
+            };
+            const opSpaceIds = [];
+            for (let i = 0; i < numIds; i++) {
+                opSpaceIds.push(-(i + 1));
+            }
+            this.serverOperations.push([range, opSpaceIds, clientFrom, sessionIdFrom]);
+            return opSpaceIds;
+        }
+        else {
+            assert(sessionIdFrom === sessionIds.get(clientFrom));
+            const compressor = this.compressors.get(clientFrom);
+            const sessionSpaceIds = generateCompressedIds(compressor, numIds);
+            for (let i = 0; i < numIds; i++) {
+                this.addNewId(clientFrom, sessionSpaceIds[i], clientFrom, sessionIdFrom, false);
+            }
+            const opSpaceIds = sessionSpaceIds.map((id) => compressor.normalizeToOpSpace(id));
+            const creationRange = compressor.takeNextCreationRange();
+            this.serverOperations.push([creationRange, opSpaceIds, clientFrom, sessionIdFrom]);
+            return opSpaceIds;
+        }
+    }
+    /**
+     * Delivers all undelivered ID ranges from the server to the target clients.
+     */
+    deliverOperations(clientTakingDelivery, opsToDeliver) {
+        let opIndexBound;
+        if (clientTakingDelivery === DestinationClient.All) {
+            assert(opsToDeliver === undefined);
+            opIndexBound = this.serverOperations.length;
+        }
+        else {
+            opIndexBound =
+                opsToDeliver !== undefined
+                    ? this.clientProgress.get(clientTakingDelivery) + opsToDeliver
+                    : this.serverOperations.length;
+        }
+        for (const [clientTo, compressorTo] of this.getTargetCompressors(clientTakingDelivery)) {
+            for (let i = this.clientProgress.get(clientTo); i < opIndexBound; i++) {
+                const [range, opSpaceIds, clientFrom, sessionIdFrom] = this.serverOperations[i];
+                compressorTo.finalizeCreationRange(range);
+                const ids = range.ids;
+                if (ids !== undefined) {
+                    for (const id of opSpaceIds) {
+                        const sessionSpaceId = compressorTo.normalizeToSessionSpace(id, range.sessionId);
+                        this.addNewId(clientTo, sessionSpaceId, clientFrom, sessionIdFrom, true);
+                    }
+                }
+            }
+            this.clientProgress.set(clientTo, opIndexBound);
+        }
+    }
+    /**
+     * Simulate a client disconnecting (and serializing), then reconnecting (and deserializing)
+     */
+    goOfflineThenResume(client) {
+        const compressor = this.compressors.get(client);
+        const [_, resumedCompressor] = roundtrip(compressor, true);
+        this.compressors.set(client, resumedCompressor);
+    }
+    /**
+     * Ensure general validity of the network state. Useful for calling periodically or at the end of test scenarios.
+     */
+    assertNetworkState() {
+        const sequencedLogs = Object.values(Client).map((client) => [this.compressors.get(client), this.getSequencedIdLog(client)]);
+        // First, ensure all clients each generated a unique ID for each of their own calls to generate.
+        for (const [compressor, ids] of sequencedLogs) {
+            const allUuids = new Set();
+            for (const idData of ids) {
+                const uuid = compressor.decompress(idData.id);
+                assert.strictEqual(!allUuids.has(uuid), true, "Duplicate UUID generated.");
+                allUuids.add(uuid);
+            }
+        }
+        const maxLogLength = sequencedLogs
+            .map(([_, data]) => data.length)
+            .reduce((p, n) => Math.max(p, n));
+        function getNextLogWithEntryAt(logsIndex, entryIndex) {
+            for (let i = logsIndex; i < sequencedLogs.length; i++) {
+                const log = sequencedLogs[i];
+                if (log[1].length > entryIndex) {
+                    return i;
+                }
+            }
+            return undefined;
+        }
+        const uuids = new Set();
+        const finalIds = new Set();
+        const idIndicesAggregator = new Map();
+        function* getIdLogEntries(columnIndex) {
+            let current = getNextLogWithEntryAt(0, columnIndex);
+            while (current !== undefined) {
+                const next = getNextLogWithEntryAt(current + 1, columnIndex);
+                const [compressor, log] = sequencedLogs[current];
+                if (next === undefined) {
+                    yield [[compressor, log[columnIndex]]];
+                }
+                else {
+                    const [compressorNext, logNext] = sequencedLogs[next];
+                    yield [
+                        [compressor, log[columnIndex]],
+                        [compressorNext, logNext[columnIndex]],
+                    ];
+                }
+                current = next;
+            }
+        }
+        for (let i = 0; i < maxLogLength; i++) {
+            let idCreatorCount = 0;
+            let originatingSession;
+            for (const [current, next] of getIdLogEntries(i)) {
+                const [compressorA, idDataA] = current;
+                const sessionSpaceIdA = idDataA.id;
+                const idIndex = getOrCreate(idIndicesAggregator, idDataA.sessionId, () => 0);
+                originatingSession ?? (originatingSession = idDataA.sessionId);
+                assert(idDataA.sessionId === originatingSession, "Test infra gave wrong originating client to TestIdData");
+                // Only one client should have this ID as local in its session space, as only one client could have created this ID
+                if (isLocalId(sessionSpaceIdA)) {
+                    if (originatingSession !== OriginatingClient.Remote) {
+                        assert.strictEqual(idDataA.sessionId, this.compressors.get(idDataA.originatingClient)
+                            .localSessionId);
+                    }
+                    idCreatorCount++;
+                }
+                const uuidASessionSpace = compressorA.decompress(sessionSpaceIdA);
+                assert.strictEqual(uuidASessionSpace, incrementStableId(idDataA.sessionId, idIndex));
+                assert.strictEqual(compressorA.recompress(uuidASessionSpace), sessionSpaceIdA);
+                uuids.add(uuidASessionSpace);
+                const opSpaceIdA = compressorA.normalizeToOpSpace(sessionSpaceIdA);
+                if (!isFinalId(opSpaceIdA)) {
+                    fail("IDs should have been finalized.");
+                }
+                const reNormalizedIdA = compressorA.normalizeToSessionSpace(opSpaceIdA, compressorA.localSessionId);
+                assert.strictEqual(reNormalizedIdA, sessionSpaceIdA);
+                finalIds.add(opSpaceIdA);
+                const uuidAOpSpace = compressorA.decompress(reNormalizedIdA);
+                assert.strictEqual(uuidASessionSpace, uuidAOpSpace);
+                if (next !== undefined) {
+                    const [compressorB, idDataB] = next;
+                    const sessionSpaceIdB = idDataB.id;
+                    const uuidBSessionSpace = compressorB.decompress(sessionSpaceIdB);
+                    assert.strictEqual(uuidASessionSpace, uuidBSessionSpace);
+                    const opSpaceIdB = compressorB.normalizeToOpSpace(sessionSpaceIdB);
+                    if (opSpaceIdA !== opSpaceIdB) {
+                        compressorB.normalizeToOpSpace(sessionSpaceIdB);
+                        compressorA.normalizeToOpSpace(sessionSpaceIdA);
+                    }
+                    assert.strictEqual(opSpaceIdA, opSpaceIdB);
+                    if (!isFinalId(opSpaceIdB)) {
+                        fail("IDs should have been finalized.");
+                    }
+                    const uuidBOpSpace = compressorB.decompress(opSpaceIdB);
+                    assert.strictEqual(uuidAOpSpace, uuidBOpSpace);
+                }
+            }
+            assert(idCreatorCount <= 1, "Only one client can create an ID.");
+            assert.strictEqual(uuids.size, finalIds.size);
+            assert(originatingSession !== undefined, "Expected originating client to be defined");
+            idIndicesAggregator.set(originatingSession, 
+            // eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+            (idIndicesAggregator.get(originatingSession) ??
+                fail("Expected pre-existing index for originating client")) + 1);
+        }
+        for (const [compressor] of sequencedLogs) {
+            expectSerializes(compressor);
+        }
+    }
+}
+export function roundtrip(compressor, withSession) {
+    if (withSession) {
+        const serialized = compressor.serialize(withSession);
+        return [serialized, IdCompressor.deserialize(serialized)];
+    }
+    const nonLocalSerialized = compressor.serialize(withSession);
+    return [nonLocalSerialized, IdCompressor.deserialize(nonLocalSerialized, createSessionId())];
+}
+/**
+ * Asserts that the supplied compressor correctly roundtrips through serialization/deserialization.
+ */
+export function expectSerializes(compressor) {
+    function expectSerializesWithSession(withSession) {
+        let serialized;
+        let deserialized;
+        if (withSession) {
+            [serialized, deserialized] = roundtrip(compressor, true);
+        }
+        else {
+            [serialized, deserialized] = roundtrip(compressor, false);
+        }
+        assert.strictEqual(compressor.equals(deserialized, withSession), true);
+        return serialized;
+    }
+    return [
+        expectSerializesWithSession(false),
+        expectSerializesWithSession(true),
+    ];
+}
+/**
+ * Merges 'from' into 'to', and returns 'to'.
+ */
+export function mergeArrayMaps(to, from) {
+    for (const [key, value] of from.entries()) {
+        const entry = to.get(key);
+        if (entry !== undefined) {
+            entry.push(...value);
+        }
+        else {
+            to.set(key, [...value]);
+        }
+    }
+    return to;
+}
+const defaultOptions = {
+    maxClusterSize: 25,
+    validateInterval: 200,
+    outsideAllocationFraction: 0.1,
+};
+export function makeOpGenerator(options) {
+    const { maxClusterSize, validateInterval, outsideAllocationFraction } = {
+        ...defaultOptions,
+        ...options,
+    };
+    assert(outsideAllocationFraction >= 0 && outsideAllocationFraction <= 1);
+    function allocateIdsGenerator({ activeClients, clusterSize, random, }) {
+        const client = random.pick(activeClients);
+        const maxIdsPerUsage = clusterSize * 2;
+        const numIds = Math.floor(random.real(0, 1) ** 3 * maxIdsPerUsage) + 1;
+        return {
+            type: "allocateIds",
+            client,
+            numIds,
+        };
+    }
+    function allocateOutsideIdsGenerator({ clusterSize, random, }) {
+        const maxIdsPerUsage = clusterSize * 2;
+        const numIds = Math.floor(random.real(0, 1) ** 3 * maxIdsPerUsage) + 1;
+        return {
+            type: "allocateOutsideIds",
+            sessionId: createSessionId(),
+            numIds,
+        };
+    }
+    function changeCapacityGenerator({ random, activeClients }) {
+        return {
+            type: "changeCapacity",
+            client: random.pick(activeClients),
+            newSize: Math.min(Math.floor(random.real(0, 1) ** 2 * maxClusterSize) + 1, maxClusterSize),
+        };
+    }
+    function deliverAllOperationsGenerator() {
+        return {
+            type: "deliverAllOperations",
+        };
+    }
+    function deliverSomeOperationsGenerator({ random, selectableClients, network, }) {
+        const pendingClients = selectableClients.filter((c) => network.getPendingOperations(c) > 0);
+        if (pendingClients.length === 0) {
+            return {
+                type: "deliverSomeOperations",
+                client: random.pick(selectableClients),
+                count: 0,
+            };
+        }
+        const client = random.pick(pendingClients);
+        return {
+            type: "deliverSomeOperations",
+            client,
+            count: random.integer(1, network.getPendingOperations(client)),
+        };
+    }
+    function reconnectGenerator({ activeClients, random }) {
+        return { type: "reconnect", client: random.pick(activeClients) };
+    }
+    const allocationWeight = 16;
+    return interleave(createWeightedGenerator([
+        [changeCapacityGenerator, 1],
+        [allocateIdsGenerator, Math.round(allocationWeight * (1 - outsideAllocationFraction))],
+        [allocateOutsideIdsGenerator, Math.round(allocationWeight * outsideAllocationFraction)],
+        [deliverAllOperationsGenerator, 2],
+        [deliverSomeOperationsGenerator, 6],
+        [reconnectGenerator, 1],
+    ]), take(1, repeat({ type: "validate" })), validateInterval);
+}
+/**
+ * Performs random actions on a test network.
+ * @param generator - the generator used to provide operations
+ * @param network - the test network to test
+ * @param seed - the seed for the random generation of the fuzz actions
+ * @param observerClient - if provided, this client will never generate local ids
+ * @param synchronizeAtEnd - if provided, all client will have all operations delivered from the server at the end of the test
+ * @param validator - if provided, this callback will be invoked periodically during the fuzz test.
+ */
+export function performFuzzActions(generator, network, seed, observerClient, synchronizeAtEnd = true, validator, saveInfo) {
+    const random = makeRandom(seed);
+    const selectableClients = network
+        .getTargetCompressors(MetaClient.All)
+        .map(([client]) => client);
+    const initialState = {
+        random,
+        network,
+        activeClients: selectableClients.filter((c) => c !== observerClient),
+        selectableClients,
+        clusterSize: network.initialClusterSize,
+    };
+    performFuzzActionsBase(generator, {
+        allocateIds: (state, { client, numIds }) => {
+            network.allocateAndSendIdsFromRemoteClient(client, sessionIds.get(client), numIds);
+            return state;
+        },
+        allocateOutsideIds: (state, { sessionId, numIds }) => {
+            network.allocateAndSendIdsFromRemoteClient(OriginatingClient.Remote, sessionId, numIds);
+            return state;
+        },
+        changeCapacity: (state, op) => {
+            network.changeCapacity(op.client, op.newSize);
+            return { ...state, clusterSize: op.newSize };
+        },
+        deliverSomeOperations: (state, op) => {
+            network.deliverOperations(op.client, op.count);
+            return state;
+        },
+        deliverAllOperations: (state) => {
+            network.deliverOperations(DestinationClient.All);
+            return state;
+        },
+        reconnect: (state, { client }) => {
+            network.goOfflineThenResume(client);
+            return state;
+        },
+        validate: (state) => {
+            network.deliverOperations(DestinationClient.All);
+            validator?.(network);
+            return state;
+        },
+    }, initialState, saveInfo);
+    if (synchronizeAtEnd) {
+        network.deliverOperations(DestinationClient.All);
+        validator?.(network);
+    }
+}
+/**
+ * Helper to generate a fixed number of IDs.
+ */
+export function generateCompressedIds(compressor, count) {
+    const ids = [];
+    for (let i = 0; i < count; i++) {
+        ids.push(compressor.generateCompressedId());
+    }
+    return ids;
+}
+export function createAlwaysFinalizedIdCompressor(sessionIdOrLogger, loggerOrUndefined) {
+    const compressor = sessionIdOrLogger === undefined
+        ? createIdCompressor()
+        : typeof sessionIdOrLogger === "string"
+            ? createIdCompressor(sessionIdOrLogger, loggerOrUndefined)
+            : createIdCompressor(sessionIdOrLogger);
+    // Permanently put the compressor in a ghost session
+    compressor.startGhostSession(createSessionId());
+    return compressor;
+}
+//# sourceMappingURL=idCompressorTestUtilities.js.map