@fluidframework/container-runtime 2.0.2 → 2.1.0-276326

package/src/gc/gcDefinitions.ts

@@ -24,6 +24,7 @@ import {
 } from "../summary/index.js";
 
 /**
+ * @legacy
  * @alpha
  */
 export type GCVersion = number;
@@ -60,8 +61,6 @@ export const gcDisableDataStoreSweepOptionName = "disableDataStoreSweep";
  */
 export const gcGenerationOptionName = "gcGeneration";
 
-/** Config key to turn GC sweep on / off. */
-export const runSweepKey = "Fluid.GarbageCollection.RunSweep";
 /** Config key to turn GC test mode on / off. */
 export const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
 /** Config key to expire a session after a set period of time. Defaults to true. */
@@ -97,6 +96,7 @@ export const defaultSweepGracePeriodMs = 1 * oneDayMs; // 1 day
 
 /**
  * @see IGCMetadata.gcFeatureMatrix and @see gcGenerationOptionName
+ * @legacy
  * @alpha
  */
 export type GCFeatureMatrix =
@@ -134,6 +134,7 @@ export interface IGCMetadata_Deprecated {
 /**
  * GC-specific metadata to be written into the summary.
  *
+ * @legacy
  * @alpha
  */
 export interface IGCMetadata {
@@ -181,6 +182,7 @@ export interface IGCMetadata {
 
 /**
  * The statistics of the system state after a garbage collection mark phase run.
+ * @legacy
  * @alpha
  */
 export interface IMarkPhaseStats {
@@ -206,6 +208,7 @@ export interface IMarkPhaseStats {
 
 /**
  * The statistics of the system state after a garbage collection sweep phase run.
+ * @legacy
  * @alpha
  */
 export interface ISweepPhaseStats {
@@ -225,12 +228,14 @@ export interface ISweepPhaseStats {
 
 /**
  * The statistics of the system state after a garbage collection run.
+ * @legacy
  * @alpha
  */
 export interface IGCStats extends IMarkPhaseStats, ISweepPhaseStats {}
 
 /**
  * The types of GC nodes in the GC reference graph.
+ * @legacy
  * @alpha
  */
 export const GCNodeType = {
@@ -245,6 +250,7 @@ export const GCNodeType = {
 } as const;
 
 /**
+ * @legacy
  * @alpha
  */
 export type GCNodeType = (typeof GCNodeType)[keyof typeof GCNodeType];
@@ -423,6 +429,7 @@ export interface IGarbageCollectorCreateParams {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface IGCRuntimeOptions {
@@ -476,6 +483,8 @@ export interface IGarbageCollectorConfigs {
 	 * throughout its lifetime.
 	 */
 	readonly sweepEnabled: boolean;
+	/** Is Tombstone AutoRecovery enabled? Useful for preventing the GC "TombstoneLoaded" op, for compatibility reasons */
+	readonly tombstoneAutorecoveryEnabled: boolean;
 	/**
 	 * Tracks if sweep phase should run or not, or if it should run only for attachment blobs.
 	 * Even if the sweep phase is allowed for a document (see sweepEnabled), it may be disabled or partially enabled

package/src/gc/gcTelemetry.ts

@@ -22,7 +22,6 @@ import {
 	IGarbageCollectorConfigs,
 	UnreferencedState,
 	disableTombstoneKey,
-	runSweepKey,
 	throwOnTombstoneLoadOverrideKey,
 	throwOnTombstoneUsageKey,
 } from "./gcDefinitions.js";
@@ -289,9 +288,6 @@ export class GCTelemetryTracker {
 				ThrowOnTombstoneUsage: this.mc.config.getBoolean(throwOnTombstoneUsageKey),
 				ThrowOnTombstoneLoad: this.mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
 			},
-			sweepFlags: {
-				EnableSweepFlag: this.mc.config.getBoolean(runSweepKey),
-			},
 		};
 
 		if (
@@ -431,9 +427,6 @@ export function sendGCUnexpectedUsageEvent(
 		ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
 		ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
 	});
-	event.sweepFlags = JSON.stringify({
-		EnableSweepFlag: mc.config.getBoolean(runSweepKey),
-	});
 	event.gcVersion = getGCVersionInEffect(mc.config);
 
 	mc.logger.sendTelemetryEvent(event, error);

package/src/gc/index.ts

@@ -31,7 +31,6 @@ export {
 	IGCStats,
 	oneDayMs,
 	runSessionExpiryKey,
-	runSweepKey,
 	stableGCVersion,
 	disableAutoRecoveryKey,
 	disableDatastoreSweepKey,

package/src/messageTypes.ts

@@ -17,6 +17,7 @@ import { IChunkedOp } from "./opLifecycle/index.js";
 import { IDocumentSchemaChangeMessage } from "./summary/index.js";
 
 /**
+ * @legacy
  * @alpha
  */
 export enum ContainerMessageType {

package/src/opLifecycle/batchManager.ts

@@ -56,6 +56,10 @@ export class BatchManager {
 			: this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
 	}
 
+	/**
+	 * The last-processed CSN when this batch started.
+	 * This is used to ensure that while the batch is open, no incoming ops are processed.
+	 */
 	private clientSequenceNumber: number | undefined;
 
 	constructor(public readonly options: IBatchManagerOptions) {}

package/src/opLifecycle/outbox.ts

@@ -41,7 +41,7 @@ export interface IOutboxParameters {
 	readonly submitBatchFn:
 		| ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number)
 		| undefined;
-	readonly legacySendBatchFn: (batch: IBatch) => void;
+	readonly legacySendBatchFn: (batch: IBatch) => number;
 	readonly config: IOutboxConfig;
 	readonly compressor: OpCompressor;
 	readonly splitter: OpSplitter;
@@ -126,10 +126,14 @@ export class Outbox {
 	}
 
 	/**
-	 * If we detect that the reference sequence number of the incoming message does not match
-	 * what was already in the batch managers, this means that batching has been interrupted so
+	 * Detect whether batching has been interrupted by an incoming message being processed. In this case,
 	 * we will flush the accumulated messages to account for that and create a new batch with the new
 	 * message as the first message.
+	 *
+	 * @remarks - To detect batch interruption, we compare both the reference sequence number
+	 * (i.e. last message processed by DeltaManager) and the client sequence number of the
+	 * last message processed by the ContainerRuntime. In the absence of op reentrancy, this
+	 * pair will remain stable during a single JS turn during which the batch is being built up.
 	 */
 	private maybeFlushPartialBatch() {
 		const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
@@ -254,6 +258,7 @@ export class Outbox {
 			return;
 		}
 
+		let clientSequenceNumber: number | undefined;
 		// Did we disconnect? (i.e. is shouldSend false?)
 		// If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
 		// Because flush() is a task that executes async (on clean stack), we can get here in disconnected state.
@@ -261,10 +266,10 @@ export class Outbox {
 			const processedBatch = this.compressBatch(
 				shouldGroup ? this.params.groupingManager.groupBatch(rawBatch) : rawBatch,
 			);
-			this.sendBatch(processedBatch);
+			clientSequenceNumber = this.sendBatch(processedBatch);
 		}
 
-		this.persistBatch(rawBatch.content);
+		this.params.pendingStateManager.onFlushBatch(rawBatch.content, clientSequenceNumber);
 	}
 
 	/**
@@ -355,11 +360,12 @@ export class Outbox {
 	 * Sends the batch object to the container context to be sent over the wire.
 	 *
 	 * @param batch - batch to be sent
+	 * @returns the clientSequenceNumber of the start of the batch, or undefined if nothing was sent
 	 */
 	private sendBatch(batch: IBatch) {
 		const length = batch.content.length;
 		if (length === 0) {
-			return;
+			return undefined; // Nothing submitted
 		}
 
 		const socketSize = estimateSocketSize(batch);
@@ -372,6 +378,7 @@ export class Outbox {
 			});
 		}
 
+		let clientSequenceNumber: number;
 		if (this.params.submitBatchFn === undefined) {
 			// Legacy path - supporting old loader versions. Can be removed only when LTS moves above
 			// version that has support for batches (submitBatchFn)
@@ -380,10 +387,10 @@ export class Outbox {
 				0x5a6 /* Compression should not have happened if the loader does not support it */,
 			);
 
-			this.params.legacySendBatchFn(batch);
+			clientSequenceNumber = this.params.legacySendBatchFn(batch);
 		} else {
 			assert(batch.referenceSequenceNumber !== undefined, 0x58e /* Batch must not be empty */);
-			this.params.submitBatchFn(
+			clientSequenceNumber = this.params.submitBatchFn(
 				batch.content.map((message) => ({
 					contents: message.contents,
 					metadata: message.metadata,
@@ -393,20 +400,11 @@ export class Outbox {
 				batch.referenceSequenceNumber,
 			);
 		}
-	}
 
-	private persistBatch(batch: BatchMessage[]) {
-		// Let the PendingStateManager know that a message was submitted.
-		// In future, need to shift toward keeping batch as a whole!
-		for (const message of batch) {
-			this.params.pendingStateManager.onSubmitMessage(
-				// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-				message.contents!,
-				message.referenceSequenceNumber,
-				message.localOpMetadata,
-				message.metadata,
-			);
-		}
+		// Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
+		clientSequenceNumber -= length - 1;
+		assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
+		return clientSequenceNumber;
 	}
 
 	public checkpoint() {

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-runtime";
-export const pkgVersion = "2.0.2";
+export const pkgVersion = "2.1.0-276326";

package/src/pendingStateManager.ts

@@ -16,6 +16,7 @@ import Deque from "double-ended-queue";
 
 import { InboundSequencedContainerRuntimeMessage } from "./messageTypes.js";
 import { IBatchMetadata } from "./metadata.js";
+import type { BatchMessage } from "./opLifecycle/index.js";
 import { pkgVersion } from "./packageVersion.js";
 
 /**
@@ -29,6 +30,7 @@ export interface IPendingMessage {
 	localOpMetadata: unknown;
 	opMetadata: Record<string, unknown> | undefined;
 	sequenceNumber?: number;
+	batchStartCsn?: number;
 }
 
 export interface IPendingLocalState {
@@ -187,27 +189,30 @@ export class PendingStateManager implements IDisposable {
 	public readonly dispose = () => this.disposeOnce.value;
 
 	/**
-	 * Called when a message is submitted locally. Adds the message and the associated details to the pending state
-	 * queue.
-	 * @param type - The container message type.
-	 * @param content - The message content.
-	 * @param localOpMetadata - The local metadata associated with the message.
+	 * The given batch has been flushed, and needs to be tracked locally until the corresponding
+	 * acks are processed, to ensure it is successfully sent.
+	 * @param batch - The batch that was flushed
+	 * @param clientSequenceNumber - The CSN of the first message in the batch,
+	 * or undefined if the batch was not yet sent (e.g. by the time we flushed we lost the connection)
 	 */
-	public onSubmitMessage(
-		content: string,
-		referenceSequenceNumber: number,
-		localOpMetadata: unknown,
-		opMetadata: Record<string, unknown> | undefined,
-	) {
-		const pendingMessage: IPendingMessage = {
-			type: "message",
-			referenceSequenceNumber,
-			content,
-			localOpMetadata,
-			opMetadata,
-		};
-
-		this.pendingMessages.push(pendingMessage);
+	public onFlushBatch(batch: BatchMessage[], clientSequenceNumber: number | undefined) {
+		for (const message of batch) {
+			const {
+				contents: content = "",
+				referenceSequenceNumber,
+				localOpMetadata,
+				metadata: opMetadata,
+			} = message;
+			const pendingMessage: IPendingMessage = {
+				type: "message",
+				referenceSequenceNumber,
+				content,
+				localOpMetadata,
+				opMetadata,
+				batchStartCsn: clientSequenceNumber,
+			};
+			this.pendingMessages.push(pendingMessage);
+		}
 	}
 
 	/**

package/src/summary/documentSchema.ts

@@ -13,6 +13,7 @@ import { pkgVersion } from "../packageVersion.js";
  * Please note that for all property types we should use undefined to indicate that particular capability is off.
  * Using false, or some string value (like "off") will result in clients who do not understand that property failing, whereas
  * we want them to continue to collaborate alongside clients who support that capability, but such capability is shipping dark for now.
+ * @legacy
  * @alpha
  */
 export type DocumentSchemaValueType = string | string[] | true | number | undefined;
@@ -27,6 +28,7 @@ export type DocumentSchemaValueType = string | string[] | true | number | undefi
  * undefined - ID compressor is not loaded.
  * While IContainerRuntime.generateDocumentUniqueId() is available, it will produce long IDs that are do not compress well.
  *
+ * @legacy
  * @alpha
  */
 export type IdCompressorMode = "on" | "delayed" | undefined;
@@ -55,6 +57,7 @@ export type IdCompressorMode = "on" | "delayed" | undefined;
  *
  * For now we are limiting it to just plain properties, and only really simple types, but that can be changed in the future.
  *
+ * @legacy
  * @alpha
  */
 export interface IDocumentSchema {
@@ -75,6 +78,7 @@ export interface IDocumentSchema {
  * ContainerMessageType.DocumentSchemaChange messages use CAS (Compare-and-swap) semantics, and convey
  * regSeq of last known schema change (known to a client proposing schema change).
  * @see ContainerRuntimeDocumentSchemaMessage
+ * @legacy
  * @alpha
  */
 export type IDocumentSchemaChangeMessage = IDocumentSchema;
@@ -85,6 +89,7 @@ export type IDocumentSchemaChangeMessage = IDocumentSchema;
  * WARNING: This type is used to infer IDocumentSchemaCurrent type!
  * Any changes here (including renaming of properties) are potentially changing document format and should be considered carefully!
  *
+ * @legacy
  * @alpha
  */
 export interface IDocumentSchemaFeatures {
@@ -114,12 +119,14 @@ export interface IDocumentSchemaFeatures {
  * This must be bumped whenever the format of document schema or protocol for changing the current document schema changes.
  * Ex: adding a new configuration property (under IDocumentSchema.runtime) does not require changing this version.
  * Ex: Changing the 'document schema acceptance' mechanism from convert-and-swap to one requiring consensus does require changing this version.
+ * @legacy
  * @alpha
  */
 export const currentDocumentVersionSchema = 1;
 
 /**
  * Current document schema.
+ * @legacy
  * @alpha
  */
 // eslint-disable-next-line @typescript-eslint/consistent-type-definitions
@@ -420,6 +427,7 @@ function arrayToProp(arr: string[]) {
  * Clients can retry, but current implementation is simply - they will not (and will rely on next session / reload to do
  * recalc and decide if schema needs to be changed or not).
  *
+ * @legacy
  * @alpha
  */
 export class DocumentsSchemaController {

package/src/summary/orderedClientElection.ts

@@ -231,6 +231,7 @@ export interface IOrderedClientElectionEvents extends IEvent {
 
 /**
  * Serialized state of IOrderedClientElection.
+ * @legacy
  * @alpha
  */
 export interface ISerializedElection {

package/src/summary/runWhileConnectedCoordinator.ts

@@ -13,6 +13,7 @@ import {
 
 /**
  * Similar to AbortController, but using promise instead of events
+ * @legacy
  * @alpha
  */
 export interface ICancellableSummarizerController extends ISummaryCancellationToken {

package/src/summary/summarizer.ts

@@ -67,6 +67,7 @@ export const createSummarizingWarning = (errorMessage: string, logged: boolean)
  * Summarizer is responsible for coordinating when to generate and send summaries.
  * It is the main entry point for summary work.
  * It is created only by summarizing container (i.e. one with clientType === "summarizer")
+ * @legacy
  * @alpha
  */
 export class Summarizer extends TypedEventEmitter<ISummarizerEvents> implements ISummarizer {

package/src/summary/summarizerTypes.ts

@@ -36,6 +36,7 @@ import { SummarizeReason } from "./summaryGenerator.js";
 /**
  * Similar to AbortSignal, but using promise instead of events
  * @param T - cancellation reason type
+ * @legacy
  * @alpha
  */
 export interface ICancellationToken<T> {
@@ -50,12 +51,14 @@ export interface ICancellationToken<T> {
 
 /**
  * Similar to AbortSignal, but using promise instead of events
+ * @legacy
  * @alpha
  */
 export type ISummaryCancellationToken = ICancellationToken<SummarizerStopReason>;
 
 /**
  * Data required to update internal tracking state after receiving a Summary Ack.
+ * @legacy
  * @alpha
  */
 export interface IRefreshSummaryAckOptions {
@@ -70,6 +73,7 @@ export interface IRefreshSummaryAckOptions {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizerInternalsProvider {
@@ -89,6 +93,7 @@ export interface ISummarizingWarning extends ContainerWarning {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface IConnectableRuntime {
@@ -99,6 +104,7 @@ export interface IConnectableRuntime {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizerRuntime extends IConnectableRuntime {
@@ -120,6 +126,7 @@ export interface ISummarizerRuntime extends IConnectableRuntime {
 
 /**
  * Options affecting summarize behavior.
+ * @legacy
  * @alpha
  */
 export interface ISummarizeOptions {
@@ -128,6 +135,7 @@ export interface ISummarizeOptions {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISubmitSummaryOptions extends ISummarizeOptions {
@@ -142,6 +150,7 @@ export interface ISubmitSummaryOptions extends ISummarizeOptions {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface IOnDemandSummarizeOptions extends ISummarizeOptions {
@@ -153,6 +162,7 @@ export interface IOnDemandSummarizeOptions extends ISummarizeOptions {
 
 /**
  * Options to use when enqueueing a summarize attempt.
+ * @legacy
  * @alpha
  */
 export interface IEnqueueSummarizeOptions extends IOnDemandSummarizeOptions {
@@ -171,6 +181,7 @@ export interface IEnqueueSummarizeOptions extends IOnDemandSummarizeOptions {
 /**
  * In addition to the normal summary tree + stats, this contains additional stats
  * only relevant at the root of the tree.
+ * @legacy
  * @alpha
  */
 export interface IGeneratedSummaryStats extends ISummaryStats {
@@ -190,6 +201,7 @@ export interface IGeneratedSummaryStats extends ISummaryStats {
 
 /**
  * Type for summarization failures that are retriable.
+ * @legacy
  * @alpha
  */
 export interface IRetriableFailureError extends Error {
@@ -198,6 +210,7 @@ export interface IRetriableFailureError extends Error {
 
 /**
  * Base results for all submitSummary attempts.
+ * @legacy
  * @alpha
  */
 export interface IBaseSummarizeResult {
@@ -211,6 +224,7 @@ export interface IBaseSummarizeResult {
 
 /**
  * Results of submitSummary after generating the summary tree.
+ * @legacy
  * @alpha
  */
 export interface IGenerateSummaryTreeResult extends Omit<IBaseSummarizeResult, "stage"> {
@@ -225,6 +239,7 @@ export interface IGenerateSummaryTreeResult extends Omit<IBaseSummarizeResult, "
 
 /**
  * Results of submitSummary after uploading the tree to storage.
+ * @legacy
  * @alpha
  */
 export interface IUploadSummaryResult extends Omit<IGenerateSummaryTreeResult, "stage"> {
@@ -237,6 +252,7 @@ export interface IUploadSummaryResult extends Omit<IGenerateSummaryTreeResult, "
 
 /**
  * Results of submitSummary after submitting the summarize op.
+ * @legacy
  * @alpha
  */
 export interface ISubmitSummaryOpResult extends Omit<IUploadSummaryResult, "stage" | "error"> {
@@ -262,6 +278,7 @@ export interface ISubmitSummaryOpResult extends Omit<IUploadSummaryResult, "stag
  * 3. "upload" - the summary was uploaded to storage, and the result contains the server-provided handle
  *
  * 4. "submit" - the summarize op was submitted, and the result contains the op client sequence number.
+ * @legacy
  * @alpha
  */
 export type SubmitSummaryResult =
@@ -272,12 +289,14 @@ export type SubmitSummaryResult =
 
 /**
  * The stages of Summarize, used to describe how far progress succeeded in case of a failure at a later stage.
+ * @legacy
  * @alpha
  */
 export type SummaryStage = SubmitSummaryResult["stage"] | "unknown";
 
 /**
  * The data in summarizer result when submit summary stage fails.
+ * @legacy
  * @alpha
  */
 export interface SubmitSummaryFailureData {
@@ -285,6 +304,7 @@ export interface SubmitSummaryFailureData {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface IBroadcastSummaryResult {
@@ -293,6 +313,7 @@ export interface IBroadcastSummaryResult {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface IAckSummaryResult {
@@ -301,6 +322,7 @@ export interface IAckSummaryResult {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface INackSummaryResult {
@@ -309,6 +331,7 @@ export interface INackSummaryResult {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export type SummarizeResultPart<TSuccess, TFailure = undefined> =
@@ -324,6 +347,7 @@ export type SummarizeResultPart<TSuccess, TFailure = undefined> =
 	  };
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizeResults {
@@ -340,6 +364,7 @@ export interface ISummarizeResults {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export type EnqueueSummarizeResult =
@@ -370,6 +395,7 @@ export type EnqueueSummarizeResult =
 	  };
 
 /**
+ * @legacy
  * @alpha
  */
 export type SummarizerStopReason =
@@ -399,6 +425,7 @@ export type SummarizerStopReason =
 	| "latestSummaryStateStale";
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizeEventProps {
@@ -409,6 +436,7 @@ export interface ISummarizeEventProps {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizerEvents extends IEvent {
@@ -416,6 +444,7 @@ export interface ISummarizerEvents extends IEvent {
 }
 
 /**
+ * @legacy
  * @alpha
  */
 export interface ISummarizer extends IEventProvider<ISummarizerEvents> {