@fluidframework/driver-utils 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/src/prefetchDocumentStorageService.ts

@@ -3,90 +3,95 @@
  * Licensed under the MIT License.
  */
 import { LoaderCachingPolicy } from "@fluidframework/driver-definitions";
-import {
-    ISnapshotTree,
-    IVersion,
-} from "@fluidframework/protocol-definitions";
+import { ISnapshotTree, IVersion } from "@fluidframework/protocol-definitions";
 import { DocumentStorageServiceProxy } from "./documentStorageServiceProxy";
 import { canRetryOnError } from "./network";
 
+/**
+ * @internal
+ */
 export class PrefetchDocumentStorageService extends DocumentStorageServiceProxy {
-    // BlobId -> blob prefetchCache cache
-    private readonly prefetchCache = new Map<string, Promise<ArrayBufferLike>>();
-    private prefetchEnabled = true;
+	// BlobId -> blob prefetchCache cache
+	private readonly prefetchCache = new Map<string, Promise<ArrayBufferLike>>();
+	private prefetchEnabled = true;
 
-    public get policies() {
-        const policies = this.internalStorageService.policies;
-        if (policies) {
-            return { ...policies, caching: LoaderCachingPolicy.NoCaching };
-        }
-    }
+	public get policies() {
+		const policies = this.internalStorageService.policies;
+		if (policies) {
+			return { ...policies, caching: LoaderCachingPolicy.NoCaching };
+		}
+	}
 
-    public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
-        const p = this.internalStorageService.getSnapshotTree(version);
-        if (this.prefetchEnabled) {
-            // We don't care if the prefetch succeeds
-            void p.then((tree: ISnapshotTree | null | undefined) => {
-                if (tree === null || tree === undefined) { return; }
-                this.prefetchTree(tree);
-            });
-        }
-        return p;
-    }
+	public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
+		const p = this.internalStorageService.getSnapshotTree(version);
+		if (this.prefetchEnabled) {
+			// We don't care if the prefetch succeeds
+			void p.then((tree: ISnapshotTree | null | undefined) => {
+				if (tree === null || tree === undefined) {
+					return;
+				}
+				this.prefetchTree(tree);
+			});
+		}
+		return p;
+	}
 
-    public async readBlob(blobId: string): Promise<ArrayBufferLike> {
-        return this.cachedRead(blobId);
-    }
-    public stopPrefetch() {
-        this.prefetchEnabled = false;
-        this.prefetchCache.clear();
-    }
+	public async readBlob(blobId: string): Promise<ArrayBufferLike> {
+		return this.cachedRead(blobId);
+	}
+	public stopPrefetch() {
+		this.prefetchEnabled = false;
+		this.prefetchCache.clear();
+	}
 
-    private async cachedRead(blobId: string): Promise<ArrayBufferLike> {
-        if (this.prefetchEnabled) {
-            const prefetchedBlobP = this.prefetchCache.get(blobId);
-            if (prefetchedBlobP !== undefined) {
-                return prefetchedBlobP;
-            }
-            const prefetchedBlobPFromStorage = this.internalStorageService.readBlob(blobId);
-            this.prefetchCache.set(blobId, prefetchedBlobPFromStorage.catch((error) => {
-                if (canRetryOnError(error)) {
-                    this.prefetchCache.delete(blobId);
-                }
-                throw error;
-            }));
-            return prefetchedBlobPFromStorage;
-        }
-        return this.internalStorageService.readBlob(blobId);
-    }
+	private async cachedRead(blobId: string): Promise<ArrayBufferLike> {
+		if (this.prefetchEnabled) {
+			const prefetchedBlobP = this.prefetchCache.get(blobId);
+			if (prefetchedBlobP !== undefined) {
+				return prefetchedBlobP;
+			}
+			const prefetchedBlobPFromStorage = this.internalStorageService.readBlob(blobId);
+			this.prefetchCache.set(
+				blobId,
+				prefetchedBlobPFromStorage.catch((error) => {
+					if (canRetryOnError(error)) {
+						this.prefetchCache.delete(blobId);
+					}
+					throw error;
+				}),
+			);
+			return prefetchedBlobPFromStorage;
+		}
+		return this.internalStorageService.readBlob(blobId);
+	}
 
-    private prefetchTree(tree: ISnapshotTree) {
-        const secondary: string[] = [];
-        this.prefetchTreeCore(tree, secondary);
+	private prefetchTree(tree: ISnapshotTree) {
+		const secondary: string[] = [];
+		this.prefetchTreeCore(tree, secondary);
 
-        for (const blob of secondary) {
-            // We don't care if the prefetch succeeds
-            void this.cachedRead(blob);
-        }
-    }
+		for (const blob of secondary) {
+			// We don't care if the prefetch succeeds
+			void this.cachedRead(blob);
+		}
+	}
 
-    private prefetchTreeCore(tree: ISnapshotTree, secondary: string[]) {
-        for (const blobKey of Object.keys(tree.blobs)) {
-            const blob = tree.blobs[blobKey];
-            if (blobKey.startsWith(".") || blobKey === "header" || blobKey.startsWith("quorum")) {
-                if (blob !== null) {
-                    // We don't care if the prefetch succeeds
-                    void this.cachedRead(blob);
-                }
-            } else if (!blobKey.startsWith("deltas")) {
-                if (blob !== null) {
-                    secondary.push(blob);
-                }
-            }
-        }
+	private prefetchTreeCore(tree: ISnapshotTree, secondary: string[]) {
+		for (const blobKey of Object.keys(tree.blobs)) {
+			const blob = tree.blobs[blobKey];
+			if (blobKey.startsWith(".") || blobKey === "header" || blobKey.startsWith("quorum")) {
+				if (blob !== null) {
+					// We don't care if the prefetch succeeds
+					void this.cachedRead(blob);
+				}
+			} else if (!blobKey.startsWith("deltas")) {
+				if (blob !== null) {
+					secondary.push(blob);
+				}
+			}
+		}
 
-        for (const subTree of Object.keys(tree.trees)) {
-            this.prefetchTreeCore(tree.trees[subTree], secondary);
-        }
-    }
+		for (const subTree of Object.keys(tree.trees)) {
+			this.prefetchTreeCore(tree.trees[subTree], secondary);
+		}
+	}
 }

package/src/rateLimiter.ts

@@ -3,41 +3,44 @@
  * Licensed under the MIT License.
  */
 
-import { assert } from "@fluidframework/common-utils";
+import { assert } from "@fluidframework/core-utils";
 
+/**
+ * @alpha
+ */
 export class RateLimiter {
-    private readonly tasks: (() => void)[] = [];
-    constructor(private maxRequests: number) {
-        assert(maxRequests > 0, 0x0ae /* "Tried to create rate limiter with 0 max requests!" */);
-    }
+	private readonly tasks: (() => void)[] = [];
+	constructor(private maxRequests: number) {
+		assert(maxRequests > 0, 0x0ae /* "Tried to create rate limiter with 0 max requests!" */);
+	}
 
-    public get waitQueueLength(): number {
-        return this.tasks.length;
-    }
+	public get waitQueueLength(): number {
+		return this.tasks.length;
+	}
 
-    // Run when one of the tasks finished running.
-    // Release next task if we have one, or allow more tasks to run in future.
-    protected readonly release = () => {
-        const task = this.tasks.shift();
-        if (task !== undefined) {
-            return task();
-        }
-        this.maxRequests++;
-    };
+	// Run when one of the tasks finished running.
+	// Release next task if we have one, or allow more tasks to run in future.
+	protected readonly release = () => {
+		const task = this.tasks.shift();
+		if (task !== undefined) {
+			return task();
+		}
+		this.maxRequests++;
+	};
 
-    protected async acquire() {
-        if (this.maxRequests > 0) {
-            this.maxRequests--;
-            return;
-        }
+	protected async acquire() {
+		if (this.maxRequests > 0) {
+			this.maxRequests--;
+			return;
+		}
 
-        return new Promise<void>((resolve) => {
-            this.tasks.push(resolve);
-        });
-    }
+		return new Promise<void>((resolve) => {
+			this.tasks.push(resolve);
+		});
+	}
 
-    public async schedule<T>(work: () => Promise<T>) {
-        await this.acquire();
-        return work().finally(this.release);
-    }
+	public async schedule<T>(work: () => Promise<T>) {
+		await this.acquire();
+		return work().finally(this.release);
+	}
 }

package/src/readAndParse.ts

@@ -3,18 +3,27 @@
  * Licensed under the MIT License.
  */
 
-import { bufferToString } from "@fluidframework/common-utils";
+import { bufferToString } from "@fluid-internal/client-utils";
 import { IDocumentStorageService } from "@fluidframework/driver-definitions";
 
 /**
- * Read a blob from IDocumentStorageService and JSON.parse it into object of type T
+ * Read a blob from {@link @fluidframework/driver-definitions#IDocumentStorageService} and
+ * {@link https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/JSON/parse | JSON.parse}
+ * it into object of type `T`.
  *
- * @param storage - the IDocumentStorageService to read from
- * @param id - the id of the blob to read and parse
- * @returns the object that we decoded and JSON.parse
+ * @param storage - The `DocumentStorageService` to read from.
+ * @param id - The ID of the blob to read and parse.
+ *
+ * @typeParam T - Output type matching JSON format of inpyt blob data.
+ *
+ * @returns The object that we decoded and parsed via `JSON.parse`.
+ * @internal
  */
-export async function readAndParse<T>(storage: Pick<IDocumentStorageService, "readBlob">, id: string): Promise<T> {
-    const blob = await storage.readBlob(id);
-    const decoded = bufferToString(blob, "utf8");
-    return JSON.parse(decoded) as T;
+export async function readAndParse<T>(
+	storage: Pick<IDocumentStorageService, "readBlob">,
+	id: string,
+): Promise<T> {
+	const blob = await storage.readBlob(id);
+	const decoded = bufferToString(blob, "utf8");
+	return JSON.parse(decoded) as T;
 }

package/src/runWithRetry.ts

@@ -3,103 +3,163 @@
  * Licensed under the MIT License.
  */
 
-import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { delay, performance } from "@fluidframework/common-utils";
-import { DriverErrorType } from "@fluidframework/driver-definitions";
-import { canRetryOnError, getRetryDelayFromError } from "./network";
+import { ITelemetryLoggerExt, isFluidError } from "@fluidframework/telemetry-utils";
+import { performance } from "@fluid-internal/client-utils";
+import { delay } from "@fluidframework/core-utils";
+import { DriverErrorTypes } from "@fluidframework/driver-definitions";
+import { canRetryOnError, getRetryDelayFromError, NonRetryableError } from "./network";
 import { pkgVersion } from "./packageVersion";
-import { NonRetryableError } from ".";
 
 /**
  * Interface describing an object passed to various network APIs.
  * It allows caller to control cancellation, as well as learn about any delays.
+ * @internal
  */
 export interface IProgress {
-    /**
-     * Abort signal used to cancel operation
-     * Note that most of the layers do not use this signal yet. We need to change that over time.
-     * Please consult with API documentation / implementation.
-     * Note that  number of layers may not check this signal while holding this request in a queue,
-     * so it may take a while it takes effect. This can be improved in the future.
-     * Layers in question are:
-     *    - driver (RateLimiter)
-     *    - runWithRetry
-     */
-    cancel?: AbortSignal;
+	/**
+	 * Abort signal used to cancel operation.
+	 *
+	 * @remarks Note that most of the layers do not use this signal yet. We need to change that over time.
+	 * Please consult with API documentation / implementation.
+	 * Note that  number of layers may not check this signal while holding this request in a queue,
+	 * so it may take a while it takes effect. This can be improved in the future.
+	 *
+	 * The layers in question are:
+	 *
+	 * - driver (RateLimiter)
+	 *
+	 * - runWithRetry
+	 */
+	cancel?: AbortSignal;
 
-    /**
-     * Called whenever api returns cancellable error and the call is going to be retried.
-     * Any exception thrown from this call back result in cancellation of operation
-     * and propagation of thrown exception.
-     * @param delayInMs - delay before next retry. This value will depend on internal back-off logic,
-     * as well as information provided by service (like 429 error asking to wait for some time before retry)
-     * @param error - error object returned from the call.
-     */
-    onRetry?(delayInMs: number, error: any): void;
+	/**
+	 * Called whenever api returns cancellable error and the call is going to be retried.
+	 * Any exception thrown from this call back result in cancellation of operation
+	 * and propagation of thrown exception.
+	 * @param delayInMs - delay before next retry. This value will depend on internal back-off logic,
+	 * as well as information provided by service (like 429 error asking to wait for some time before retry)
+	 * @param error - error object returned from the call.
+	 */
+	onRetry?(delayInMs: number, error: any): void;
 }
 
+/**
+ * @internal
+ */
 export async function runWithRetry<T>(
-    api: (cancel?: AbortSignal) => Promise<T>,
-    fetchCallName: string,
-    logger: ITelemetryLogger,
-    progress: IProgress,
+	api: (cancel?: AbortSignal) => Promise<T>,
+	fetchCallName: string,
+	logger: ITelemetryLoggerExt,
+	progress: IProgress,
 ): Promise<T> {
-    let result: T | undefined;
-    let success = false;
-    let retryAfterMs = 1000; // has to be positive!
-    let numRetries = 0;
-    const startTime = performance.now();
-    let lastError: any;
-    do {
-        try {
-            result = await api(progress.cancel);
-            success = true;
-        } catch (err) {
-            // If it is not retriable, then just throw the error.
-            if (!canRetryOnError(err)) {
-                logger.sendTelemetryEvent({
-                    eventName: `${fetchCallName}_cancel`,
-                    retry: numRetries,
-                    duration: performance.now() - startTime,
-                    fetchCallName,
-                }, err);
-                throw err;
-            }
+	let result: T | undefined;
+	let success = false;
+	// We double this value in first try in when we calculate time to wait for in "calculateMaxWaitTime" function.
+	let retryAfterMs = 500; // has to be positive!
+	let numRetries = 0;
+	const startTime = performance.now();
+	let lastError: any;
+	do {
+		try {
+			result = await api(progress.cancel);
+			success = true;
+		} catch (err) {
+			// If it is not retriable, then just throw the error.
+			if (!canRetryOnError(err)) {
+				logger.sendTelemetryEvent(
+					{
+						eventName: `${fetchCallName}_cancel`,
+						retry: numRetries,
+						duration: performance.now() - startTime,
+						fetchCallName,
+					},
+					err,
+				);
+				throw err;
+			}
+
+			if (progress.cancel?.aborted === true) {
+				logger.sendTelemetryEvent(
+					{
+						eventName: `${fetchCallName}_runWithRetryAborted`,
+						retry: numRetries,
+						duration: performance.now() - startTime,
+						fetchCallName,
+						reason: progress.cancel.reason,
+					},
+					err,
+				);
+				throw new NonRetryableError(
+					"runWithRetry was Aborted",
+					DriverErrorTypes.genericError,
+					{
+						driverVersion: pkgVersion,
+						fetchCallName,
+						reason: progress.cancel.reason,
+					},
+				);
+			}
+
+			// logging the first failed retry instead of every attempt. We want to avoid filling telemetry
+			// when we have tight loop of retrying in offline mode, but we also want to know what caused
+			// the failure in the first place
+			if (numRetries === 0) {
+				logger.sendTelemetryEvent(
+					{
+						eventName: `${fetchCallName}_firstFailed`,
+						duration: performance.now() - startTime,
+						fetchCallName,
+					},
+					err,
+				);
+			}
 
-            if (progress.cancel?.aborted === true) {
-                logger.sendTelemetryEvent({
-                    eventName: `${fetchCallName}_runWithRetryAborted`,
-                    retry: numRetries,
-                    duration: performance.now() - startTime,
-                    fetchCallName,
-                }, err);
-                throw new NonRetryableError(
-                    "runWithRetry was Aborted",
-                    DriverErrorType.genericError,
-                    { driverVersion: pkgVersion, fetchCallName },
-                );
-            }
+			numRetries++;
+			lastError = err;
+			// Wait for the calculated time before retrying.
+			retryAfterMs = calculateMaxWaitTime(retryAfterMs, err);
+			if (progress.onRetry) {
+				progress.onRetry(retryAfterMs, err);
+			}
+			await delay(retryAfterMs);
+		}
+	} while (!success);
+	if (numRetries > 0) {
+		logger.sendTelemetryEvent(
+			{
+				eventName: `${fetchCallName}_lastError`,
+				retry: numRetries,
+				duration: performance.now() - startTime,
+				fetchCallName,
+			},
+			lastError,
+		);
+	}
+	// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+	return result!;
+}
+
+const MaxReconnectDelayInMsWhenEndpointIsReachable = 60000;
+const MaxReconnectDelayInMsWhenEndpointIsNotReachable = 8000;
 
-            numRetries++;
-            lastError = err;
-            // If the error is throttling error, then wait for the specified time before retrying.
-            // If the waitTime is not specified, then we start with retrying immediately to max of 8s.
-            retryAfterMs = getRetryDelayFromError(err) ?? Math.min(retryAfterMs * 2, 8000);
-            if (progress.onRetry) {
-                progress.onRetry(retryAfterMs, err);
-            }
-            await delay(retryAfterMs);
-        }
-    } while (!success);
-    if (numRetries > 0) {
-        logger.sendTelemetryEvent({
-            eventName: `${fetchCallName}_lastError`,
-            retry: numRetries,
-            duration: performance.now() - startTime,
-            fetchCallName,
-        },
-        lastError);
-    }
-    // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
-    return result!;
+/**
+ * Calculates time to wait for after an error based on the error and wait time for previous iteration.
+ * In case endpoint(service or socket) is not reachable, then we maybe offline or may have got some
+ * transient error not related to endpoint, in that case we want to try at faster pace and hence the
+ * max wait is lesser 8s as compared to when endpoint is reachable in which case it is 60s.
+ * @param delayMs - wait time for previous iteration
+ * @param error - error based on which we decide wait time.
+ * @returns Wait time to wait for.
+ * @internal
+ */
+export function calculateMaxWaitTime(delayMs: number, error: unknown): number {
+	const retryDelayFromError = getRetryDelayFromError(error);
+	let newDelayMs = Math.max(retryDelayFromError ?? 0, delayMs * 2);
+	newDelayMs = Math.min(
+		delayMs,
+		isFluidError(error) && error.getTelemetryProperties().endpointReached === true
+			? MaxReconnectDelayInMsWhenEndpointIsReachable
+			: MaxReconnectDelayInMsWhenEndpointIsNotReachable,
+	);
+	return newDelayMs;
 }

package/src/summaryForCreateNew.ts

@@ -4,53 +4,72 @@
  */
 
 import {
-    ISummaryTree,
-    SummaryType,
-    ISummaryBlob,
-    ICommittedProposal,
-    IDocumentAttributes,
+	ISummaryTree,
+	SummaryType,
+	ISummaryBlob,
+	ICommittedProposal,
+	IDocumentAttributes,
 } from "@fluidframework/protocol-definitions";
 
 /**
- * Combine the app summary and protocol summary in 1 tree.
- * @param appSummary - Summary of the app.
- * @param protocolSummary - Summary of the protocol.
+ * Defines the current layout of an .app + .protocol summary tree
+ * this is used internally for create new, and single commit summary
+ * @internal
  */
-export function combineAppAndProtocolSummary(
-    appSummary: ISummaryTree,
-    protocolSummary: ISummaryTree,
-): ISummaryTree {
-    const createNewSummary: ISummaryTree = {
-        type: SummaryType.Tree,
-        tree: {
-            ".protocol": protocolSummary,
-            ".app": appSummary,
-        },
-    };
-    return createNewSummary;
+export interface CombinedAppAndProtocolSummary extends ISummaryTree {
+	tree: {
+		[".app"]: ISummaryTree;
+		[".protocol"]: ISummaryTree;
+	};
+}
+
+/**
+ * Validates the current layout of an .app + .protocol summary tree
+ * this is used internally for create new, and single commit summary
+ * @internal
+ */
+export function isCombinedAppAndProtocolSummary(
+	summary: ISummaryTree | undefined,
+	...optionalRootTrees: string[]
+): summary is CombinedAppAndProtocolSummary {
+	if (
+		summary?.tree === undefined ||
+		summary.tree?.[".app"]?.type !== SummaryType.Tree ||
+		summary.tree?.[".protocol"]?.type !== SummaryType.Tree
+	) {
+		return false;
+	}
+	const treeKeys = Object.keys(summary.tree).filter((t) => !optionalRootTrees.includes(t));
+	if (treeKeys.length !== 2) {
+		return false;
+	}
+	return true;
 }
 
 /**
  * Extract the attributes from the protocol summary.
  * @param protocolSummary - protocol summary from which the values are to be extracted.
+ * @internal
  */
 export function getDocAttributesFromProtocolSummary(
-    protocolSummary: ISummaryTree,
+	protocolSummary: ISummaryTree,
 ): IDocumentAttributes {
-    const attributesBlob = protocolSummary.tree.attributes as ISummaryBlob;
-    const documentAttributes = JSON.parse(attributesBlob.content as string) as IDocumentAttributes;
-    documentAttributes.term = documentAttributes.term ?? 1;
-    return documentAttributes;
+	const attributesBlob = protocolSummary.tree.attributes as ISummaryBlob;
+	return JSON.parse(attributesBlob.content as string) as IDocumentAttributes;
 }
 
 /**
  * Extract quorum values from the protocol summary.
  * @param protocolSummary - protocol summary from which the values are to be extracted.
+ * @internal
  */
 export function getQuorumValuesFromProtocolSummary(
-    protocolSummary: ISummaryTree,
+	protocolSummary: ISummaryTree,
 ): [string, ICommittedProposal][] {
-    const quorumValuesBlob = protocolSummary.tree.quorumValues as ISummaryBlob;
-    const quorumValues = JSON.parse(quorumValuesBlob.content as string) as [string, ICommittedProposal][];
-    return quorumValues;
+	const quorumValuesBlob = protocolSummary.tree.quorumValues as ISummaryBlob;
+	const quorumValues = JSON.parse(quorumValuesBlob.content as string) as [
+		string,
+		ICommittedProposal,
+	][];
+	return quorumValues;
 }