@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258

package/src/gc/gcSummaryStateTracker.ts

@@ -12,12 +12,11 @@ import {
 	ISummarizeResult,
 	ISummaryTreeWithStats,
 } from "@fluidframework/runtime-definitions";
-import { mergeStats, ReadAndParseBlob, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
-import { IContainerRuntimeMetadata, metadataBlobName, RefreshSummaryResult } from "../summary";
-import { GCVersion, IGCStats } from "./gcDefinitions";
-import { getGCDataFromSnapshot, generateSortedGCState, getGCVersion } from "./gcHelpers";
+import { mergeStats, SummaryTreeBuilder } from "@fluidframework/runtime-utils";
+import { IRefreshSummaryResult } from "../summary";
+import { GCVersion, IGarbageCollectorConfigs, IGCStats } from "./gcDefinitions";
+import { generateSortedGCState } from "./gcHelpers";
 import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions";
-import { IGarbageCollectorConfigs } from ".";
 
 export const gcStateBlobKey = `${gcBlobPrefix}_root`;
 
@@ -264,72 +263,27 @@ export class GCSummaryStateTracker {
 	}
 
 	/**
-	 * Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
-	 * is downloaded and should be used to update the state.
+	 * Called to refresh the latest summary state. This happens when either a pending summary is acked.
 	 */
-	public async refreshLatestSummary(
-		proposalHandle: string | undefined,
-		result: RefreshSummaryResult,
-		readAndParseBlob: ReadAndParseBlob,
-	): Promise<IGarbageCollectionSnapshotData | undefined> {
-		// If the latest summary was updated and the summary was tracked, this client is the one that generated this
-		// summary. So, update wasGCRunInLatestSummary.
-		// Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
-		// true in scenarios where GC is disabled but enabled in the snapshot we loaded from.
-		if (result.latestSummaryUpdated && result.wasSummaryTracked) {
-			this.wasGCRunInLatestSummary = this.configs.shouldRunGC;
-		}
-
-		if (!result.latestSummaryUpdated || !this.configs.shouldRunGC) {
-			return undefined;
-		}
-
-		// If the summary was tracked by this client, it was the one that generated the summary in the first place.
-		// Update latest state from pending.
-		if (result.wasSummaryTracked) {
-			this.latestSummaryGCVersion = this.configs.gcVersionInEffect;
-			this.latestSummaryData = this.pendingSummaryData;
-			this.pendingSummaryData = undefined;
-			this.updatedDSCountSinceLastSummary = 0;
-			return undefined;
-		}
-
-		// If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
-		const snapshotTree = result.snapshotTree;
-		const metadataBlobId = snapshotTree.blobs[metadataBlobName];
-		const metadata = metadataBlobId
-			? await readAndParseBlob<IContainerRuntimeMetadata>(metadataBlobId)
-			: undefined;
-		this.latestSummaryGCVersion = getGCVersion(metadata);
-
-		const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
-		// If GC ran in the container that generated this snapshot, it will have a GC tree.
-		this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
-
-		if (gcSnapshotTree === undefined) {
-			return undefined;
+	public async refreshLatestSummary(result: IRefreshSummaryResult): Promise<void> {
+		if (!result.isSummaryTracked) {
+			return;
 		}
 
-		let snapshotData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
+		// If the summary is tracked, this client is the one that generated it. So, update wasGCRunInLatestSummary.
+		// Note that this has to be updated if GC did not run too. Otherwise, `gcStateNeedsReset` will always return
+		// true in scenarios where GC is currently disabled but enabled in the snapshot we loaded from.
+		this.wasGCRunInLatestSummary = this.configs.shouldRunGC;
 
-		// If the GC version in the snapshot does not match the GC version currently in effect, the GC data
-		// in the snapshot cannot be interpreted correctly. Set everything to undefined except for deletedNodes
-		// because irrespective of GC versions, these nodes have been deleted and cannot be brought back. The
-		// deletedNodes info is needed to identify when these nodes are used.
-		if (getGCVersion(metadata) !== this.configs.gcVersionInEffect) {
-			snapshotData = {
-				gcState: undefined,
-				tombstones: undefined,
-				deletedNodes: snapshotData.deletedNodes,
-			};
+		if (!this.configs.shouldRunGC) {
+			return;
 		}
 
-		this.latestSummaryData = {
-			serializedGCState: JSON.stringify(snapshotData.gcState),
-			serializedTombstones: JSON.stringify(snapshotData.tombstones),
-			serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
-		};
-		return snapshotData;
+		this.latestSummaryGCVersion = this.configs.gcVersionInEffect;
+		this.latestSummaryData = this.pendingSummaryData;
+		this.pendingSummaryData = undefined;
+		this.updatedDSCountSinceLastSummary = 0;
+		return;
 	}
 
 	/**

package/src/gc/gcTelemetry.ts

@@ -3,13 +3,13 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryGenericEvent } from "@fluidframework/common-definitions";
+import { ITelemetryGenericEvent } from "@fluidframework/core-interfaces";
 import { IGarbageCollectionData } from "@fluidframework/runtime-definitions";
-import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
 import {
 	generateStack,
 	ITelemetryLoggerExt,
 	MonitoringContext,
+	tagCodeArtifacts,
 } from "@fluidframework/telemetry-utils";
 import { ICreateContainerMetadata } from "../summary";
 import {
@@ -23,7 +23,6 @@ import {
 	runSweepKey,
 } from "./gcDefinitions";
 import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
-import { tagAsCodeArtifact } from "./gcHelpers";
 
 type NodeUsageType = "Changed" | "Loaded" | "Revived";
 
@@ -161,7 +160,6 @@ export class GCTelemetryTracker {
 		const { usageType, currentReferenceTimestampMs, packagePath, id, fromId, ...propsToLog } =
 			nodeUsageProps;
 		const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
-			id: tagAsCodeArtifact(id),
 			type: nodeType,
 			unrefTime: nodeStateTracker.unreferencedTimestampMs,
 			age:
@@ -171,7 +169,7 @@ export class GCTelemetryTracker {
 				state === UnreferencedState.Inactive
 					? this.configs.inactiveTimeoutMs
 					: this.configs.sweepTimeoutMs,
-			fromId: fromId ? tagAsCodeArtifact(fromId) : undefined,
+			...tagCodeArtifacts({ id, fromId }),
 			...propsToLog,
 			...this.createContainerMetadata,
 		};
@@ -184,7 +182,7 @@ export class GCTelemetryTracker {
 				{
 					eventName: `GC_Tombstone_${nodeType}_Revived`,
 					category: "generic",
-					url: tagAsCodeArtifact(id),
+					...tagCodeArtifacts({ url: id }),
 					gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
 				},
 				undefined /* packagePath */,
@@ -211,7 +209,7 @@ export class GCTelemetryTracker {
 				const { id: taggedId, fromId: taggedFromId, ...otherProps } = eventProps;
 				const event = {
 					eventName: `${state}Object_${nodeUsageProps.usageType}`,
-					pkg: packagePathToTelemetryProperty(nodeUsageProps.packagePath),
+					pkg: tagCodeArtifacts({ pkg: nodeUsageProps.packagePath?.join("/") }).pkg,
 					stack: generateStack(),
 					id: taggedId,
 					fromId: taggedFromId,
@@ -277,8 +275,10 @@ export class GCTelemetryTracker {
 			if (missingExplicitRoutes.length > 0) {
 				logger.sendErrorEvent({
 					eventName: "gcUnknownOutboundReferences",
-					id: tagAsCodeArtifact(nodeId),
-					routes: tagAsCodeArtifact(JSON.stringify(missingExplicitRoutes)),
+					...tagCodeArtifacts({
+						id: nodeId,
+						routes: JSON.stringify(missingExplicitRoutes),
+					}),
 				});
 			}
 		}
@@ -318,8 +318,10 @@ export class GCTelemetryTracker {
 					}),
 					id,
 					fromId,
-					pkg: pkg ? tagAsCodeArtifact(pkg.join("/")) : undefined,
-					fromPkg: fromPkg ? tagAsCodeArtifact(fromPkg.join("/")) : undefined,
+					...tagCodeArtifacts({
+						pkg: pkg?.join("/"),
+						fromPkg: fromPkg?.join("/"),
+					}),
 				};
 
 				if (state === UnreferencedState.Inactive) {
@@ -379,7 +381,7 @@ export class GCTelemetryTracker {
 					lastSummaryTime,
 					...this.createContainerMetadata,
 				}),
-				id: tagAsCodeArtifact(JSON.stringify(deletedNodeIds)),
+				...tagCodeArtifacts({ id: JSON.stringify(deletedNodeIds) }),
 			});
 		}
 	}
@@ -398,7 +400,7 @@ export function sendGCUnexpectedUsageEvent(
 	packagePath: readonly string[] | undefined,
 	error?: unknown,
 ) {
-	event.pkg = packagePathToTelemetryProperty(packagePath);
+	event.pkg = tagCodeArtifacts({ pkg: packagePath?.join("/") })?.pkg;
 	event.tombstoneFlags = JSON.stringify({
 		DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
 		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),

package/src/gc/gcUnreferencedStateTracker.ts

@@ -3,7 +3,7 @@
  * Licensed under the MIT License.
  */
 
-import { assert, Timer } from "@fluidframework/common-utils";
+import { assert, Timer } from "@fluidframework/core-utils";
 import { UnreferencedState } from "./gcDefinitions";
 
 /** A wrapper around common-utils Timer that requires the timeout when calling start/restart */

package/src/gc/index.ts

@@ -29,8 +29,8 @@ export {
 	runSessionExpiryKey,
 	runSweepKey,
 	stableGCVersion,
-	sweepAttachmentBlobsKey,
-	sweepDatastoresKey,
+	disableAttachmentBlobSweepKey,
+	disableDatastoreSweepKey,
 	throwOnTombstoneLoadKey,
 	throwOnTombstoneUsageKey,
 	UnreferencedState,
@@ -38,12 +38,10 @@ export {
 export {
 	cloneGCData,
 	concatGarbageCollectionStates,
-	getGCDataFromSnapshot,
 	shouldAllowGcTombstoneEnforcement,
 	shouldAllowGcSweep,
 	trimLeadingAndTrailingSlashes,
 	unpackChildNodesGCDetails,
-	tagAsCodeArtifact,
 } from "./gcHelpers";
 export { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
 export {

package/src/id-compressor/appendOnlySortedMap.ts

@@ -3,10 +3,9 @@
  * Licensed under the MIT License.
  */
 
+/* eslint-disable tsdoc/syntax */
 /* eslint-disable no-bitwise */
-
-import { assert } from "@fluidframework/common-utils";
-import { fail } from "./utils";
+import { assert } from "@fluidframework/core-utils";
 
 /**
  * A map in which entries are always added in key-sorted order.
@@ -133,7 +132,7 @@ export class AppendOnlySortedMap<K, V> {
 	}
 
 	/**
-	 * Adds a new key/value pair to the map. `key` must be \> to all keys in the map.
+	 * Adds a new key/value pair to the map. `key` must be > to all keys in the map.
 	 * @param key - the key to add.
 	 * @param value - the value to add.
 	 */
@@ -141,7 +140,27 @@ export class AppendOnlySortedMap<K, V> {
 		const { elements } = this;
 		const { length } = elements;
 		if (length !== 0 && this.comparator(key, this.maxKey() as K) <= 0) {
-			fail("Inserted key must be > all others in the map.");
+			throw new Error("Inserted key must be > all others in the map.");
+		}
+		elements.push(key);
+		elements.push(value);
+	}
+
+	/**
+	 * Replaces the last key/value pair with the given one. If the map is empty, it simply appends.
+	 * `key` must be > to all keys in the map prior to the one replaced.
+	 * @param key - the key to add.
+	 * @param value - the value to add.
+	 */
+	public replaceLast(key: K, value: V): void {
+		const { elements, comparator } = this;
+		const { length } = elements;
+		if (length !== 0) {
+			elements.pop();
+			elements.pop();
+			if (comparator(key, this.maxKey() as K) <= 0) {
+				throw new Error("Inserted key must be > all others in the map.");
+			}
 		}
 		elements.push(key);
 		elements.push(value);
@@ -217,7 +236,7 @@ export class AppendOnlySortedMap<K, V> {
 			if (prev !== undefined) {
 				assert(
 					this.comparator(kv[0], prev[0]) > 0,
-					0x47f /* Keys in map must be sorted. */,
+					0x752 /* Keys in map must be sorted. */,
 				);
 			}
 			prev = kv;
@@ -338,90 +357,10 @@ export class AppendOnlySortedMap<K, V> {
 			} else if (c === 0) {
 				return keyIndex;
 			} else {
-				fail("Invalid comparator.");
+				throw new Error("Invalid comparator.");
 			}
 			mid = (low + high) >> 1;
 		}
 		return (mid * 2) ^ AppendOnlySortedMap.failureXor;
 	}
 }
-
-/**
- * A map in which entries are always added in both key-sorted and value-sorted order.
- * Supports appending and searching.
- */
-export class AppendOnlyDoublySortedMap<K, V, S> extends AppendOnlySortedMap<K, V> {
-	public constructor(
-		keyComparator: (a: K, b: K) => number,
-		private readonly extractSearchValue: (value: V) => S,
-		private readonly valueComparator: (search: S, value: S) => number,
-	) {
-		super(keyComparator);
-	}
-
-	public override append(key: K, value: V): void {
-		if (
-			this.elements.length !== 0 &&
-			this.valueComparator(
-				this.extractSearchValue(value),
-				this.extractSearchValue(this.maxValue() as V),
-			) <= 0
-		) {
-			fail("Inserted value must be > all others in the map.");
-		}
-		super.append(key, value);
-	}
-
-	private readonly compareValues = (search: S, _: K, value: V): number => {
-		return this.valueComparator(search, this.extractSearchValue(value));
-	};
-
-	// /**
-	//  * @param value - the value to lookup.
-	//  * @returns the key associated with `value` if such an entry exists, and undefined otherwise.
-	//  */
-	// public getByValue(value: S): K | undefined {
-	//     const index = AppendOnlySortedMap.keyIndexOf(this.elements, value, this.compareValues);
-	//     return this.elements[index]?.[0];
-	// }
-
-	/**
-	 * @param searchValue - the search value to lookup.
-	 * @returns the entry who's value, when run through the extractor provided to the constructor, matches
-	 * `searchValue`. If no such entry exists, this method returns the next lower entry as determined by the value
-	 * comparator provided to the constructor. If no such entry exists, this method returns undefined.
-	 */
-	public getPairOrNextLowerByValue(searchValue: S): readonly [K, V] | undefined {
-		return this.getPairOrNextLowerBy(searchValue, this.compareValues);
-	}
-
-	/**
-	 * @param searchValue - the search value to lookup.
-	 * @returns the entry who's value, when run through the extractor provided to the constructor, matches `searchValue`. If no such entry
-	 * exists, this method returns the next higher entry as determined by the value comparator provided to the constructor. If no such entry
-	 * exists, this method returns undefined.
-	 */
-	public getPairOrNextHigherByValue(searchValue: S): readonly [K, V] | undefined {
-		return this.getPairOrNextHigherBy(searchValue, this.compareValues);
-	}
-
-	/**
-	 * Test-only expensive assertions to check the internal validity of the data structure.
-	 */
-	public override assertValid(): void {
-		super.assertValid();
-		let prev: readonly [unknown, V] | undefined;
-		for (const kv of this.entries()) {
-			if (prev !== undefined) {
-				assert(
-					this.valueComparator(
-						this.extractSearchValue(kv[1]),
-						this.extractSearchValue(prev[1]),
-					) > 0,
-					0x480 /* Values in map must be sorted. */,
-				);
-			}
-			prev = kv;
-		}
-	}
-}

package/src/id-compressor/finalSpace.ts

@@ -0,0 +1,67 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { assert } from "@fluidframework/core-utils";
+import { IdCluster, clustersEqual, lastAllocatedFinal, lastFinalizedFinal } from "./sessions";
+import { FinalCompressedId } from "./identifiers";
+
+/**
+ * All IDs that have been finalized (acked), grouped into clusters sorted by their base final IDs.
+ * These clusters span the positive integer space and are not sparse, meaning a cluster's base final
+ * ID will always be one greater than the last final ID in the previous cluster (or 0 if there is not one).
+ */
+export class FinalSpace {
+	private readonly clusterList: IdCluster[] = [];
+
+	public get clusters(): readonly IdCluster[] {
+		return this.clusterList;
+	}
+
+	public getLastCluster(): IdCluster | undefined {
+		return this.clusterList[this.clusterList.length - 1];
+	}
+
+	public addCluster(newCluster: IdCluster) {
+		const lastCluster = this.getLastCluster();
+		assert(
+			lastCluster === undefined ||
+				// eslint-disable-next-line @typescript-eslint/restrict-plus-operands
+				newCluster.baseFinalId === lastCluster.baseFinalId + lastCluster.capacity,
+			0x753 /* Cluster insert to final_space is out of order. */,
+		);
+		this.clusterList.push(newCluster);
+	}
+
+	/**
+	 * @returns the upper bound (exclusive) of finalized IDs in final space, i.e. one greater than the last final ID in the last cluster.
+	 * Note: this does not include allocated but unfinalized space in clusters.
+	 */
+	public getFinalizedIdLimit(): FinalCompressedId {
+		const lastCluster = this.getLastCluster();
+		return lastCluster === undefined
+			? (0 as FinalCompressedId)
+			: (((lastFinalizedFinal(lastCluster) as number) + 1) as FinalCompressedId);
+	}
+
+	/**
+	 * @returns the upper bound (exclusive) of allocated IDs in final space, i.e. one greater than the last final ID in the last cluster.
+	 * Note: this does includes all allocated IDs in clusters.
+	 */
+	public getAllocatedIdLimit(): FinalCompressedId {
+		const lastCluster = this.getLastCluster();
+		return lastCluster === undefined
+			? (0 as FinalCompressedId)
+			: (((lastAllocatedFinal(lastCluster) as number) + 1) as FinalCompressedId);
+	}
+
+	public equals(other: FinalSpace): boolean {
+		for (let i = 0; i < this.clusterList.length; i++) {
+			if (!clustersEqual(this.clusterList[i], other.clusterList[i])) {
+				return false;
+			}
+		}
+		return this.clusterList.length === other.clusterList.length;
+	}
+}