@fluidframework/container-loader 1.4.0-121020 → 2.0.0-dev-rc.1.0.0.225277

package/src/location-redirection-utilities/index.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+export {
+	isLocationRedirectionError,
+	resolveWithLocationRedirectionHandling,
+} from "./resolveWithLocationRedirection";

package/src/location-redirection-utilities/resolveWithLocationRedirection.ts

@@ -0,0 +1,61 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ITelemetryBaseLogger, IRequest } from "@fluidframework/core-interfaces";
+import {
+	DriverErrorTypes,
+	ILocationRedirectionError,
+	IUrlResolver,
+} from "@fluidframework/driver-definitions";
+import { createChildLogger } from "@fluidframework/telemetry-utils";
+
+/**
+ * Checks if the error is location redirection error.
+ * @param error - error whose type is to be determined.
+ * @returns `true` is the error is location redirection error, otherwise `false`.
+ * @internal
+ */
+export function isLocationRedirectionError(error: any): error is ILocationRedirectionError {
+	return (
+		typeof error === "object" &&
+		error !== null &&
+		error.errorType === DriverErrorTypes.locationRedirection
+	);
+}
+
+/**
+ * Handles location redirection while fulfilling the loader request.
+ * @param api - Callback in which user can wrap the loader.resolve or loader.request call.
+ * @param request - request to be resolved.
+ * @param urlResolver - resolver used to resolve the url.
+ * @param logger - logger to send events.
+ * @returns Response from the API call.
+ * @alpha
+ */
+export async function resolveWithLocationRedirectionHandling<T>(
+	api: (request: IRequest) => Promise<T>,
+	request: IRequest,
+	urlResolver: IUrlResolver,
+	logger?: ITelemetryBaseLogger,
+): Promise<T> {
+	let req: IRequest = request;
+	const childLogger = createChildLogger({ logger, namespace: "LocationRedirection" });
+	for (;;) {
+		try {
+			return await api(req);
+		} catch (error: any) {
+			if (!isLocationRedirectionError(error)) {
+				throw error;
+			}
+			childLogger.sendTelemetryEvent({ eventName: "LocationRedirectionError" });
+			const resolvedUrl = error.redirectUrl;
+			// Generate the new request with new location details from the resolved url. For datastore/relative path,
+			// we don't need to pass "/" as host could have asked for a specific data store. So driver need to
+			// extract it from the resolved url.
+			const absoluteUrl = await urlResolver.getAbsoluteUrl(resolvedUrl, "", undefined);
+			req = { url: absoluteUrl, headers: req.headers };
+		}
+	}
+}

package/src/noopHeuristic.ts

@@ -0,0 +1,107 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { TypedEventEmitter } from "@fluid-internal/client-utils";
+import { assert, Timer } from "@fluidframework/core-utils";
+import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
+import { isRuntimeMessage } from "@fluidframework/driver-utils";
+import { IEvent } from "@fluidframework/core-interfaces";
+
+const defaultNoopTimeFrequency = 2000;
+const defaultNoopCountFrequency = 50;
+
+export interface INoopSenderEvents extends IEvent {
+	(event: "wantsNoop", listener: () => void);
+}
+
+// Here are key considerations when deciding conditions for when to send non-immediate noops:
+// 1. Sending them too often results in increase in file size and bandwidth, as well as catch up performance
+// 2. Sending too infrequently ensures that collab window is large, and as result Sequence DDS would have
+//    large catchUp blobs - see Issue #6364
+// 3. Similarly, processes that rely on "core" snapshot (and can't parse trailing ops, including above), like search
+//    parser in SPO, will result in non-accurate results due to presence of catch up blobs.
+// 4. Ordering service used 250ms timeout to coalesce non-immediate noops. It was changed to 2000 ms to allow more
+//    aggressive noop sending from client side.
+// 5. Number of ops sent by all clients is proportional to number of "write" clients (every client sends noops),
+//    but number of sequenced noops is a function of time (one op per 2 seconds at most).
+//    We should consider impact to both outbound traffic (might be huge, depends on number of clients) and file size.
+// Please also see Issue #5629 for more discussions.
+//
+// With that, the current algorithm is as follows:
+// 1. Sent noop 2000 ms of receiving an op if no ops were sent by this client within this timeframe.
+//    This will ensure that MSN moves forward with reasonable speed. If that results in too many sequenced noops,
+//    server timeout of 2000ms should be reconsidered to be increased.
+// 2. If there are more than 50 ops received without sending any ops, send noop to keep collab window small.
+//    Note that system ops (including noops themselves) are excluded, so it's 1 noop per 50 real ops.
+export class NoopHeuristic extends TypedEventEmitter<INoopSenderEvents> {
+	private opsProcessedSinceOpSent = 0;
+	private readonly timer: Timer | undefined;
+
+	constructor(
+		NoopTimeFrequency: number = defaultNoopTimeFrequency,
+		private readonly NoopCountFrequency: number = defaultNoopCountFrequency,
+	) {
+		super();
+		if (NoopTimeFrequency !== Infinity) {
+			this.timer = new Timer(NoopTimeFrequency, () => {
+				// We allow the timer to expire even if an op is sent or we disconnect.
+				// This condition is to guard against trying to send a noop anyway in that case.
+				if (this.opsProcessedSinceOpSent !== 0) {
+					this.emit("wantsNoop");
+				}
+			});
+		}
+	}
+
+	/**
+	 * Schedules as ack to the server to update the reference sequence number
+	 */
+	public notifyMessageProcessed(message: ISequencedDocumentMessage): void {
+		// We don't acknowledge no-ops to avoid acknowledgement cycles (i.e. ack the MSN
+		// update, which updates the MSN, then ack the update, etc...).
+		// Intent here is for runtime (and DDSes) not to keep too much tracking state / memory
+		// due to runtime ops from other clients.
+		if (!isRuntimeMessage(message)) {
+			return;
+		}
+
+		this.opsProcessedSinceOpSent++;
+		if (this.opsProcessedSinceOpSent === this.NoopCountFrequency) {
+			// Wait to send a noop if we are still synchronously processing ops.  This guards against two things:
+			// 1. If we're processing many ops, we may pass the frequency threshold many times.  We only need to send one noop at the very end in this case.
+			// 2. We may send another (non-noop) op in response to processing those ops, e.g. an Accept op.
+			queueMicrotask(() => {
+				if (this.opsProcessedSinceOpSent >= this.NoopCountFrequency) {
+					this.emit("wantsNoop");
+					assert(
+						this.opsProcessedSinceOpSent === 0,
+						0x243 /* "Expected a noop to be synchronously sent" */,
+					);
+				}
+				return;
+			});
+		}
+
+		if (this.timer !== undefined) {
+			// Start the timer if we newly have ops that want a noop.
+			// If the timer was already running (e.g. we surpassed the op count and sent a noop) this will reset it to its full duration.
+			if (this.opsProcessedSinceOpSent === 1) {
+				this.timer.restart();
+			}
+
+			assert(this.timer.hasTimer, 0x242 /* "has timer" */);
+		}
+	}
+
+	public notifyDisconnect(): void {
+		// No need to noop for any ops processed prior to disconnect - we are already removed from MSN calculation.
+		this.opsProcessedSinceOpSent = 0;
+	}
+
+	public notifyMessageSent(): void {
+		// Sending any message is as good as a noop for updating MSN.
+		this.opsProcessedSinceOpSent = 0;
+	}
+}

package/src/packageVersion.ts

@@ -6,4 +6,4 @@
  */
 
 export const pkgName = "@fluidframework/container-loader";
-export const pkgVersion = "1.4.0-121020";
+export const pkgVersion = "2.0.0-dev-rc.1.0.0.225277";

package/src/protocol.ts

@@ -0,0 +1,150 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { IAudienceOwner } from "@fluidframework/container-definitions";
+import { canBeCoalescedByService } from "@fluidframework/driver-utils";
+import {
+	IProtocolHandler as IBaseProtocolHandler,
+	IQuorumSnapshot,
+	ProtocolOpHandler,
+} from "@fluidframework/protocol-base";
+import {
+	IDocumentAttributes,
+	IProcessMessageResult,
+	ISequencedDocumentMessage,
+	ISignalClient,
+	ISignalMessage,
+	MessageType,
+} from "@fluidframework/protocol-definitions";
+
+// ADO: #1986: Start using enum from protocol-base.
+export enum SignalType {
+	ClientJoin = "join", // same value as MessageType.ClientJoin,
+	ClientLeave = "leave", // same value as MessageType.ClientLeave,
+	Clear = "clear", // used only by client for synthetic signals
+}
+
+/**
+ * Function to be used for creating a protocol handler.
+ * @alpha
+ */
+export type ProtocolHandlerBuilder = (
+	attributes: IDocumentAttributes,
+	snapshot: IQuorumSnapshot,
+	sendProposal: (key: string, value: any) => number,
+) => IProtocolHandler;
+
+/**
+ * @alpha
+ */
+export interface IProtocolHandler extends IBaseProtocolHandler {
+	readonly audience: IAudienceOwner;
+	processSignal(message: ISignalMessage);
+}
+
+export class ProtocolHandler extends ProtocolOpHandler implements IProtocolHandler {
+	constructor(
+		attributes: IDocumentAttributes,
+		quorumSnapshot: IQuorumSnapshot,
+		sendProposal: (key: string, value: any) => number,
+		public readonly audience: IAudienceOwner,
+		private readonly shouldClientHaveLeft: (clientId: string) => boolean,
+	) {
+		super(
+			attributes.minimumSequenceNumber,
+			attributes.sequenceNumber,
+			quorumSnapshot.members,
+			quorumSnapshot.proposals,
+			quorumSnapshot.values,
+			sendProposal,
+		);
+
+		// Join / leave signals are ignored for "write" clients in favor of join / leave ops
+		this.quorum.on("addMember", (clientId, details) =>
+			audience.addMember(clientId, details.client),
+		);
+		this.quorum.on("removeMember", (clientId) => audience.removeMember(clientId));
+		for (const [clientId, details] of this.quorum.getMembers()) {
+			this.audience.addMember(clientId, details.client);
+		}
+	}
+
+	public processMessage(
+		message: ISequencedDocumentMessage,
+		local: boolean,
+	): IProcessMessageResult {
+		// Check and report if we're getting messages from a clientId that we previously
+		// flagged as shouldHaveLeft, or from a client that's not in the quorum but should be
+		if (message.clientId != null) {
+			const client = this.quorum.getMember(message.clientId);
+
+			if (client === undefined && message.type !== MessageType.ClientJoin) {
+				// pre-0.58 error message: messageClientIdMissingFromQuorum
+				throw new Error("Remote message's clientId is missing from the quorum");
+			}
+
+			// Here checking canBeCoalescedByService is used as an approximation of "is benign to process despite being unexpected".
+			// It's still not good to see these messages from unexpected clientIds, but since they don't harm the integrity of the
+			// document we don't need to blow up aggressively.
+			if (this.shouldClientHaveLeft(message.clientId) && !canBeCoalescedByService(message)) {
+				// pre-0.58 error message: messageClientIdShouldHaveLeft
+				throw new Error("Remote message's clientId already should have left");
+			}
+		}
+
+		return super.processMessage(message, local);
+	}
+
+	public processSignal(message: ISignalMessage) {
+		const innerContent = message.content as { content: any; type: string };
+		switch (innerContent.type) {
+			case SignalType.Clear: {
+				const members = this.audience.getMembers();
+				for (const [clientId, client] of members) {
+					if (client.mode === "read") {
+						this.audience.removeMember(clientId);
+					}
+				}
+				break;
+			}
+			case SignalType.ClientJoin: {
+				const newClient = innerContent.content as ISignalClient;
+				// Ignore write clients - quorum will control such clients.
+				if (newClient.client.mode === "read") {
+					this.audience.addMember(newClient.clientId, newClient.client);
+				}
+				break;
+			}
+			case SignalType.ClientLeave: {
+				const leftClientId = innerContent.content as string;
+				// Ignore write clients - quorum will control such clients.
+				if (this.audience.getMember(leftClientId)?.mode === "read") {
+					this.audience.removeMember(leftClientId);
+				}
+				break;
+			}
+			default:
+				break;
+		}
+	}
+}
+
+/**
+ * Function to check whether the protocol handler should process the Signal.
+ * The protocol handler should strictly handle only ClientJoin, ClientLeave
+ * and Clear signal types.
+ */
+export function protocolHandlerShouldProcessSignal(message: ISignalMessage) {
+	// Signal originates from server
+	if (message.clientId === null) {
+		const innerContent = message.content as { content: unknown; type: string };
+		return (
+			innerContent.type === SignalType.Clear ||
+			innerContent.type === SignalType.ClientJoin ||
+			innerContent.type === SignalType.ClientLeave
+		);
+	}
+	return false;
+}

package/src/protocolTreeDocumentStorageService.ts

@@ -3,43 +3,43 @@
  * Licensed under the MIT License.
  */
 
-import { IDisposable } from "@fluidframework/common-definitions";
-import {
-    IDocumentStorageService,
-    ISummaryContext,
-} from "@fluidframework/driver-definitions";
-import { combineAppAndProtocolSummary } from "@fluidframework/driver-utils";
-import {
-    ISummaryTree,
-} from "@fluidframework/protocol-definitions";
+import { IDisposable } from "@fluidframework/core-interfaces";
+import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
+import { ISummaryTree } from "@fluidframework/protocol-definitions";
 
+/**
+ * A storage service wrapper whose sole job is to intercept calls to uploadSummaryWithContext and ensure they include
+ * the protocol summary, using the provided callback to add it if necessary.
+ */
 export class ProtocolTreeStorageService implements IDocumentStorageService, IDisposable {
-    constructor(
-        private readonly internalStorageService: IDocumentStorageService & IDisposable,
-        private readonly generateProtocolTree: () => ISummaryTree,
-    ) {
-    }
-    public get policies() {
-        return this.internalStorageService.policies;
-    }
-    public get repositoryUrl() {
-        return this.internalStorageService.repositoryUrl;
-    }
-    public get disposed() {
-        return this.internalStorageService.disposed;
-    }
+	constructor(
+		private readonly internalStorageService: IDocumentStorageService & IDisposable,
+		private readonly addProtocolSummaryIfMissing: (summaryTree: ISummaryTree) => ISummaryTree,
+	) {}
+	public get policies() {
+		return this.internalStorageService.policies;
+	}
+	public get repositoryUrl() {
+		return this.internalStorageService.repositoryUrl;
+	}
+	public get disposed() {
+		return this.internalStorageService.disposed;
+	}
 
-    getSnapshotTree = this.internalStorageService.getSnapshotTree.bind(this.internalStorageService);
-    getVersions = this.internalStorageService.getVersions.bind(this.internalStorageService);
-    createBlob = this.internalStorageService.createBlob.bind(this.internalStorageService);
-    readBlob = this.internalStorageService.readBlob.bind(this.internalStorageService);
-    downloadSummary = this.internalStorageService.downloadSummary.bind(this.internalStorageService);
-    dispose = this.internalStorageService.dispose.bind(this.internalStorageService);
+	getSnapshotTree = this.internalStorageService.getSnapshotTree.bind(this.internalStorageService);
+	getVersions = this.internalStorageService.getVersions.bind(this.internalStorageService);
+	createBlob = this.internalStorageService.createBlob.bind(this.internalStorageService);
+	readBlob = this.internalStorageService.readBlob.bind(this.internalStorageService);
+	downloadSummary = this.internalStorageService.downloadSummary.bind(this.internalStorageService);
+	dispose = this.internalStorageService.dispose.bind(this.internalStorageService);
 
-    async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
-        return this.internalStorageService.uploadSummaryWithContext(
-            combineAppAndProtocolSummary(summary, this.generateProtocolTree()),
-            context,
-        );
-    }
+	async uploadSummaryWithContext(
+		summary: ISummaryTree,
+		context: ISummaryContext,
+	): Promise<string> {
+		return this.internalStorageService.uploadSummaryWithContext(
+			this.addProtocolSummaryIfMissing(summary),
+			context,
+		);
+	}
 }

package/src/quorum.ts

@@ -2,58 +2,19 @@
  * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
  * Licensed under the MIT License.
  */
-import { assert, EventForwarder, doIfNotDisposed } from "@fluidframework/common-utils";
 import { IFluidCodeDetails } from "@fluidframework/core-interfaces";
-import {
-    ICommittedProposal,
-    IQuorum,
-    IQuorumEvents,
-    ISequencedClient,
-} from "@fluidframework/protocol-definitions";
-
-/**
- * Proxies Quorum events.
- */
-export class QuorumProxy extends EventForwarder<IQuorumEvents> implements IQuorum {
-    public readonly propose: (key: string, value: any) => Promise<void>;
-    public readonly has: (key: string) => boolean;
-    public readonly get: (key: string) => any;
-    public readonly getMembers: () => Map<string, ISequencedClient>;
-    public readonly getMember: (clientId: string) => ISequencedClient | undefined;
-
-    constructor(quorum: IQuorum) {
-        super(quorum);
-
-        // This is heavily used object, increase limit at which Node prints warnings.
-        super.setMaxListeners(50);
-
-        this.propose = doIfNotDisposed(this, quorum.propose.bind(quorum));
-        this.has = doIfNotDisposed(this, quorum.has.bind(quorum));
-        this.get = doIfNotDisposed(this, quorum.get.bind(quorum));
-        this.getMembers = doIfNotDisposed(this, quorum.getMembers.bind(quorum));
-        this.getMember = doIfNotDisposed(this, quorum.getMember.bind(quorum));
-    }
-}
-
-export function getCodeDetailsFromQuorumValues(
-    quorumValues: [string, ICommittedProposal][],
-): IFluidCodeDetails {
-    const qValuesMap = new Map(quorumValues);
-    const proposal = qValuesMap.get("code");
-    assert(proposal !== undefined, 0x2dc /* "Cannot find code proposal" */);
-    return proposal?.value as IFluidCodeDetails;
-}
+import { ICommittedProposal } from "@fluidframework/protocol-definitions";
 
 export function initQuorumValuesFromCodeDetails(
-    source: IFluidCodeDetails,
+	source: IFluidCodeDetails,
 ): [string, ICommittedProposal][] {
-    // Seed the base quorum to be an empty list with a code quorum set
-    const committedCodeProposal: ICommittedProposal = {
-        key: "code",
-        value: source,
-        approvalSequenceNumber: 0,
-        commitSequenceNumber: 0,
-        sequenceNumber: 0,
-    };
-    return [["code", committedCodeProposal]];
+	// Seed the base quorum to be an empty list with a code quorum set
+	const committedCodeProposal: ICommittedProposal = {
+		key: "code",
+		value: source,
+		approvalSequenceNumber: 0,
+		commitSequenceNumber: 0,
+		sequenceNumber: 0,
+	};
+	return [["code", committedCodeProposal]];
 }