@fluidframework/routerlicious-driver 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/src/contracts.ts

@@ -0,0 +1,60 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { ISnapshotTree } from "@fluidframework/protocol-definitions";
+
+/*
+ *
+ * Whole Snapshot Download Data Structures
+ *
+ */
+
+export interface IWholeFlatSnapshotTreeEntryTree {
+	path: string;
+	type: "tree";
+	// Indicates that this tree entry is unreferenced. If this is not present, the tree entry is considered referenced.
+	unreferenced?: true;
+}
+
+export interface IWholeFlatSnapshotTreeEntryBlob {
+	id: string;
+	path: string;
+	type: "blob";
+}
+
+export type IWholeFlatSnapshotTreeEntry =
+	| IWholeFlatSnapshotTreeEntryTree
+	| IWholeFlatSnapshotTreeEntryBlob;
+
+export interface IWholeFlatSnapshotTree {
+	entries: IWholeFlatSnapshotTreeEntry[];
+	id: string;
+	sequenceNumber: number;
+}
+
+export interface IWholeFlatSnapshotBlob {
+	content: string;
+	encoding: "base64" | "utf-8";
+	id: string;
+	size: number;
+}
+
+export interface IWholeFlatSnapshot {
+	// The same as the id of the first snapshot tree in the trees array.
+	id: string;
+	// Receive an array of snapshot trees for future-proofing, however, always length 1 for now.
+	trees: IWholeFlatSnapshotTree[];
+	blobs?: IWholeFlatSnapshotBlob[];
+}
+
+/**
+ * Normalized Whole Summary with decoded blobs and unflattened snapshot tree.
+ */
+export interface INormalizedWholeSnapshot {
+	blobs: Map<string, ArrayBuffer>;
+	snapshotTree: ISnapshotTree;
+	sequenceNumber: number | undefined;
+	id: string;
+}

package/src/createNewUtils.ts

@@ -3,37 +3,39 @@
  * Licensed under the MIT License.
  */
 
-import { Uint8ArrayToString } from "@fluidframework/common-utils";
+import { Uint8ArrayToString } from "@fluid-internal/client-utils";
 import { ISummaryTree, SummaryType } from "@fluidframework/protocol-definitions";
 
 /**
  * Utility api to convert ISummaryTree to a summary tree where blob contents are only utf8 strings.
  * @param summary - Summary supplied by the runtime to upload.
- * @returns - Modified summary tree where the blob contents could be utf8 string only.
+ * @returns Modified summary tree where the blob contents could be utf8 string only.
  */
 export function convertSummaryToCreateNewSummary(summary: ISummaryTree): ISummaryTree {
-    const keys = Object.keys(summary.tree);
-    for (const key of keys) {
-        const summaryObject = summary.tree[key];
+	const keys = Object.keys(summary.tree);
+	for (const key of keys) {
+		const summaryObject = summary.tree[key];
 
-        switch (summaryObject.type) {
-            case SummaryType.Tree: {
-                summary.tree[key] = convertSummaryToCreateNewSummary(summaryObject);
-                break;
-            }
-            case SummaryType.Blob: {
-                summaryObject.content = typeof summaryObject.content === "string" ?
-                    summaryObject.content : Uint8ArrayToString(summaryObject.content, "utf8");
-                break;
-            }
-            case SummaryType.Handle: {
-                throw new Error("No handle should be present for first summary!!");
-            }
-            default: {
-                throw new Error(`Unknown tree type ${summaryObject.type}`);
-            }
-        }
-    }
+		switch (summaryObject.type) {
+			case SummaryType.Tree: {
+				summary.tree[key] = convertSummaryToCreateNewSummary(summaryObject);
+				break;
+			}
+			case SummaryType.Blob: {
+				summaryObject.content =
+					typeof summaryObject.content === "string"
+						? summaryObject.content
+						: Uint8ArrayToString(summaryObject.content, "utf8");
+				break;
+			}
+			case SummaryType.Handle: {
+				throw new Error("No handle should be present for first summary!!");
+			}
+			default: {
+				throw new Error(`Unknown tree type ${summaryObject.type}`);
+			}
+		}
+	}
 
-    return summary;
+	return summary;
 }

package/src/defaultTokenProvider.ts

@@ -7,24 +7,23 @@ import { ITokenProvider, ITokenResponse } from "./tokens";
 
 /**
  * Default token provider in case the host does not provide one. It simply caches the provided jwt and returns it back.
+ * @internal
  */
 
 export class DefaultTokenProvider implements ITokenProvider {
-    constructor(private readonly jwt: string) {
+	constructor(private readonly jwt: string) {}
 
-    }
+	public async fetchOrdererToken(): Promise<ITokenResponse> {
+		return {
+			fromCache: true,
+			jwt: this.jwt,
+		};
+	}
 
-    public async fetchOrdererToken(): Promise<ITokenResponse> {
-        return {
-            fromCache: true,
-            jwt: this.jwt,
-        };
-    }
-
-    public async fetchStorageToken(): Promise<ITokenResponse> {
-        return {
-            fromCache: true,
-            jwt: this.jwt,
-        };
-    }
+	public async fetchStorageToken(): Promise<ITokenResponse> {
+		return {
+			fromCache: true,
+			jwt: this.jwt,
+		};
+	}
 }

package/src/definitions.ts

@@ -6,6 +6,6 @@
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
 
 export interface ISnapshotTreeVersion {
-    id: string;
-    snapshotTree: ISnapshotTree;
+	id: string;
+	snapshotTree: ISnapshotTree;
 }

package/src/deltaStorageService.ts

@@ -3,118 +3,172 @@
  * Licensed under the MIT License.
  */
 
+import { ITelemetryProperties } from "@fluidframework/core-interfaces";
+import { getW3CData, validateMessages } from "@fluidframework/driver-base";
 import {
-    IDeltaStorageService,
-    IDocumentDeltaStorageService,
-    IDeltasFetchResult,
-    IStream,
+	IDeltaStorageService,
+	IDocumentDeltaStorageService,
+	IDeltasFetchResult,
+	IStream,
 } from "@fluidframework/driver-definitions";
 import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
-import { readAndParse, requestOps, emptyMessageStream } from "@fluidframework/driver-utils";
-import { TelemetryNullLogger } from "@fluidframework/common-utils";
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { PerformanceEvent } from "@fluidframework/telemetry-utils";
-import { RestWrapper } from "@fluidframework/server-services-client";
+import {
+	readAndParse,
+	requestOps,
+	emptyMessageStream,
+	streamObserver,
+} from "@fluidframework/driver-utils";
+import { ITelemetryLoggerExt, PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { DocumentStorageService } from "./documentStorageService";
+import { RestWrapper } from "./restWrapperBase";
 
-const MaxBatchDeltas = 2000; // Maximum number of ops we can fetch at a time
+/**
+ * Maximum number of ops we can fetch at a time. This should be kept at 2k, as
+ * server determines whether to try to fallback to long-term storage if the ops range requested is larger than
+ * what they have locally available in short-term storage. So if we request 2k ops, they know it is not a
+ * specific request and they don't fall to long term storage which takes time.
+ * Please coordinate to AFR team if this value need to be changed.
+ */
+const MaxBatchDeltas = 2000;
 
 /**
  * Storage service limited to only being able to fetch documents for a specific document
  */
 export class DocumentDeltaStorageService implements IDocumentDeltaStorageService {
-    constructor(
-        private readonly tenantId: string,
-        private readonly id: string,
-        private readonly deltaStorageService: IDeltaStorageService,
-        private readonly documentStorageService: DocumentStorageService) {
-    }
+	constructor(
+		private readonly tenantId: string,
+		private readonly id: string,
+		private readonly deltaStorageService: IDeltaStorageService,
+		private readonly documentStorageService: DocumentStorageService,
+		private readonly logger: ITelemetryLoggerExt,
+	) {}
+
+	private logtailSha: string | undefined = this.documentStorageService.logTailSha;
+	private snapshotOps: ISequencedDocumentMessage[] | undefined;
+
+	fetchMessages(
+		fromTotal: number,
+		toTotal: number | undefined,
+		abortSignal?: AbortSignal,
+		cachedOnly?: boolean,
+		fetchReason?: string,
+	): IStream<ISequencedDocumentMessage[]> {
+		if (cachedOnly) {
+			return emptyMessageStream;
+		}
 
-    private logtailSha: string | undefined = this.documentStorageService.logTailSha;
+		let opsFromSnapshot = 0;
+		let opsFromStorage = 0;
+		const requestCallback = async (
+			from: number,
+			to: number,
+			telemetryProps: ITelemetryProperties,
+		) => {
+			this.snapshotOps = this.logtailSha
+				? await readAndParse<ISequencedDocumentMessage[]>(
+						this.documentStorageService,
+						this.logtailSha,
+				  )
+				: [];
+			this.logtailSha = undefined;
 
-    fetchMessages(from: number,
-        to: number | undefined,
-        abortSignal?: AbortSignal,
-        cachedOnly?: boolean,
-        fetchReason?: string,
-    ): IStream<ISequencedDocumentMessage[]> {
-        if (cachedOnly) {
-            return emptyMessageStream;
-        }
-        return requestOps(
-            this.getCore.bind(this),
-            // Staging: starting with no concurrency, listening for feedback first.
-            // In future releases we will switch to actual concurrency
-            1, // concurrency
-            from, // inclusive
-            to, // exclusive
-            MaxBatchDeltas,
-            new TelemetryNullLogger(),
-            abortSignal,
-            fetchReason,
-        );
-    }
+			if (this.snapshotOps !== undefined && this.snapshotOps.length !== 0) {
+				const messages = this.snapshotOps.filter(
+					(op) => op.sequenceNumber >= from && op.sequenceNumber < to,
+				);
+				validateMessages("snapshotOps", messages, from, this.logger, false /* strict */);
+				if (messages.length > 0 && messages[0].sequenceNumber === from) {
+					this.snapshotOps = this.snapshotOps.filter((op) => op.sequenceNumber >= to);
+					opsFromSnapshot += messages.length;
+					return { messages, partialResult: true };
+				}
+				this.snapshotOps = undefined;
+			}
 
-    private async getCore(from: number, to: number): Promise<IDeltasFetchResult> {
-        const opsFromLogTail = this.logtailSha
-            ? await readAndParse<ISequencedDocumentMessage[]>(this.documentStorageService, this.logtailSha)
-            : [];
+			const ops = await this.deltaStorageService.get(this.tenantId, this.id, from, to);
+			validateMessages("storage", ops.messages, from, this.logger, false /* strict */);
+			opsFromStorage += ops.messages.length;
+			return ops;
+		};
 
-        this.logtailSha = undefined;
-        if (opsFromLogTail.length > 0) {
-            const messages = opsFromLogTail.filter((op) =>
-                op.sequenceNumber >= from,
-            );
-            if (messages.length > 0) {
-                return { messages, partialResult: true };
-            }
-        }
+		const stream = requestOps(
+			async (from: number, to: number, telemetryProps: ITelemetryProperties) => {
+				const result = await requestCallback(from, to, telemetryProps);
+				// Catch all case, just in case
+				validateMessages("catch all", result.messages, from, this.logger);
+				return result;
+			},
+			// Staging: starting with no concurrency, listening for feedback first.
+			// In future releases we will switch to actual concurrency
+			1, // concurrency
+			fromTotal, // inclusive
+			toTotal, // exclusive
+			MaxBatchDeltas,
+			this.logger,
+			abortSignal,
+			fetchReason,
+		);
 
-        return this.deltaStorageService.get(this.tenantId, this.id, from, to);
-    }
+		return streamObserver(stream, (result) => {
+			if (result.done && opsFromSnapshot + opsFromStorage !== 0) {
+				this.logger.sendPerformanceEvent({
+					eventName: "CacheOpsRetrieved",
+					opsFromSnapshot,
+					opsFromStorage,
+				});
+			}
+		});
+	}
 }
 
 /**
  * Provides access to the underlying delta storage on the server for routerlicious driver.
  */
 export class DeltaStorageService implements IDeltaStorageService {
-    constructor(
-        private readonly url: string,
-        private readonly restWrapper: RestWrapper,
-        private readonly logger: ITelemetryLogger,
-        private readonly getRestWrapper: () => Promise<RestWrapper> = async () => this.restWrapper,
-        private readonly getDeltaStorageUrl: () => string = () => this.url,
-    ) {
-    }
+	constructor(
+		private readonly url: string,
+		private readonly restWrapper: RestWrapper,
+		private readonly logger: ITelemetryLoggerExt,
+		private readonly getRestWrapper: () => Promise<RestWrapper> = async () => this.restWrapper,
+		private readonly getDeltaStorageUrl: () => string = () => this.url,
+	) {}
 
-    public async get(
-        tenantId: string,
-        id: string,
-        from: number, // inclusive
-        to: number, // exclusive
-    ): Promise<IDeltasFetchResult> {
-        const ops = await PerformanceEvent.timedExecAsync(
-            this.logger,
-            {
-                eventName: "getDeltas",
-                from,
-                to,
-            },
-            async (event) => {
-                const restWrapper = await this.getRestWrapper();
-                const url = this.getDeltaStorageUrl();
-                const response = await restWrapper.get<ISequencedDocumentMessage[]>(
-                    url,
-                    { from: from - 1, to });
-                event.end({
-                    count: response.length,
-                });
-                return response;
-            },
-        );
+	public async get(
+		tenantId: string,
+		id: string,
+		from: number, // inclusive
+		to: number, // exclusive
+	): Promise<IDeltasFetchResult> {
+		const ops = await PerformanceEvent.timedExecAsync(
+			this.logger,
+			{
+				eventName: "OpsFetch",
+				from,
+				to,
+			},
+			async (event) => {
+				const restWrapper = await this.getRestWrapper();
+				const url = this.getDeltaStorageUrl();
+				const response = await restWrapper.get<ISequencedDocumentMessage[]>(url, {
+					from: from - 1,
+					to,
+				});
+				event.end({
+					length: response.content.length,
+					details: JSON.stringify({
+						firstOpSeqNumber: response.content[0]?.sequenceNumber,
+						lastOpSeqNumber:
+							response.content[response.content.length - 1]?.sequenceNumber,
+					}),
+					...response.propsToLog,
+					...getW3CData(response.requestUrl, "xmlhttprequest"),
+				});
+				return response.content;
+			},
+		);
 
-        // It is assumed that server always returns all the ops that it has in the range that was requested.
-        // This may change in the future, if so, we need to adjust and receive "end" value from server in such case.
-        return { messages: ops, partialResult: false };
-    }
+		// It is assumed that server always returns all the ops that it has in the range that was requested.
+		// This may change in the future, if so, we need to adjust and receive "end" value from server in such case.
+		return { messages: ops, partialResult: false };
+	}
 }

package/src/documentDeltaConnection.ts

@@ -3,10 +3,9 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
+import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
 import { DocumentDeltaConnection } from "@fluidframework/driver-base";
-import { IDocumentDeltaConnection } from "@fluidframework/driver-definitions";
-import { IAnyDriverError } from "@fluidframework/driver-utils";
+import { IAnyDriverError, IDocumentDeltaConnection } from "@fluidframework/driver-definitions";
 import { IClient, IConnect } from "@fluidframework/protocol-definitions";
 import type { io as SocketIOClientStatic } from "socket.io-client";
 import { errorObjectFromSocketError, IR11sSocketError } from "./errorUtils";
@@ -18,57 +17,60 @@ const protocolVersions = ["^0.4.0", "^0.3.0", "^0.2.0", "^0.1.0"];
  * Wrapper over the shared one for driver specific translation.
  */
 export class R11sDocumentDeltaConnection extends DocumentDeltaConnection {
-    public static async create(
-        tenantId: string,
-        id: string,
-        token: string | null,
-        io: typeof SocketIOClientStatic,
-        client: IClient,
-        url: string,
-        logger: ITelemetryLogger,
-        timeoutMs = 20000): Promise<IDocumentDeltaConnection> {
-        const socket = io(
-            url,
-            {
-                query: {
-                    documentId: id,
-                    tenantId,
-                },
-                reconnection: false,
-                // Default to websocket connection, with long-polling disabled
-                transports: ["websocket"],
-                timeout: timeoutMs,
-            });
+	public static async create(
+		tenantId: string,
+		id: string,
+		token: string | null,
+		io: typeof SocketIOClientStatic,
+		client: IClient,
+		url: string,
+		logger: ITelemetryLoggerExt,
+		timeoutMs = 20000,
+		enableLongPollingDowngrade = true,
+	): Promise<IDocumentDeltaConnection> {
+		const socket = io(url, {
+			query: {
+				documentId: id,
+				tenantId,
+			},
+			reconnection: false,
+			// Default to websocket connection, with long-polling disabled
+			transports: ["websocket"],
+			timeout: timeoutMs,
+		});
 
-        const connectMessage: IConnect = {
-            client,
-            id,
-            mode: client.mode,
-            tenantId,
-            token,  // Token is going to indicate tenant level information, etc...
-            versions: protocolVersions,
-            relayUserAgent: [client.details.environment, ` driverVersion:${driverVersion}`].join(";"),
-        };
+		const connectMessage: IConnect = {
+			client,
+			id,
+			mode: client.mode,
+			tenantId,
+			token, // Token is going to indicate tenant level information, etc...
+			versions: protocolVersions,
+			relayUserAgent: [client.details.environment, ` driverVersion:${driverVersion}`].join(
+				";",
+			),
+		};
 
-        // TODO: expose to host at factory level
-        const enableLongPollingDowngrades = true;
-        const deltaConnection = new R11sDocumentDeltaConnection(socket, id, logger, enableLongPollingDowngrades);
+		const deltaConnection = new R11sDocumentDeltaConnection(
+			socket,
+			id,
+			logger,
+			enableLongPollingDowngrade,
+		);
 
-        await deltaConnection.initialize(connectMessage, timeoutMs);
-        return deltaConnection;
-    }
+		await deltaConnection.initialize(connectMessage, timeoutMs);
+		return deltaConnection;
+	}
 
-    /**
-     * Error raising for socket.io issues
-     */
-    protected createErrorObject(handler: string, error?: any, canRetry = true): IAnyDriverError {
-        // Note: we suspect the incoming error object is either:
-        // - a socketError: add it to the R11sError object for driver to be able to parse it and reason over it.
-        // - anything else: let base class handle it
-        if (canRetry && Number.isInteger(error?.code) && typeof error?.message === "string") {
-            return errorObjectFromSocketError(error as IR11sSocketError, handler);
-        } else {
-            return super.createErrorObject(handler, error, canRetry);
-        }
-    }
+	/**
+	 * Error raising for socket.io issues
+	 */
+	protected createErrorObject(handler: string, error?: any, canRetry = true): IAnyDriverError {
+		// Note: we suspect the incoming error object is either:
+		// - a socketError: add it to the R11sError object for driver to be able to parse it and reason over it.
+		// - anything else: let base class handle it
+		return canRetry && Number.isInteger(error?.code) && typeof error?.message === "string"
+			? errorObjectFromSocketError(error as IR11sSocketError, handler)
+			: super.createErrorObject(handler, error, canRetry);
+	}
 }