@fluidframework/odsp-driver 0.51.3 → 0.53.0-46105

package/src/epochTracker.ts

@@ -6,8 +6,7 @@
 import { v4 as uuid } from "uuid";
 import { assert, Deferred } from "@fluidframework/common-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { fluidEpochMismatchError, throwOdspNetworkError } from "@fluidframework/odsp-doclib-utils";
-import { ThrottlingError, RateLimiter } from "@fluidframework/driver-utils";
+import { ThrottlingError, RateLimiter, NonRetryableError } from "@fluidframework/driver-utils";
 import { IConnected } from "@fluidframework/protocol-definitions";
 import {
     snapshotKey,
@@ -18,7 +17,7 @@ import {
     IOdspError,
 } from "@fluidframework/odsp-driver-definitions";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
-import { PerformanceEvent, isValidLegacyError, isFluidError, normalizeError } from "@fluidframework/telemetry-utils";
+import { PerformanceEvent, isFluidError, normalizeError } from "@fluidframework/telemetry-utils";
 import { fetchAndParseAsJSONHelper, fetchArray, IOdspResponse } from "./odspUtils";
 import {
     IOdspCache,
@@ -34,6 +33,8 @@ export type FetchTypeInternal = FetchType | "cache";
 
 export const Odsp409Error = "Odsp409Error";
 
+export const defaultCacheExpiryTimeoutMs: number = 2 * 24 * 60 * 60 * 1000;
+
 /**
  * This class is a wrapper around fetch calls. It adds epoch to the request made so that the
  * server can match it with its epoch value in order to match the version.
@@ -75,16 +76,37 @@ export class EpochTracker implements IPersistedFileCache {
         entry: IEntry,
     ): Promise<any> {
         try {
+            // Return undefined so that the ops/snapshots are grabbed from the server instead of the cache
             const value: IVersionedValueWithEpoch = await this.cache.get(this.fileEntryFromEntry(entry));
+            // Version mismatch between what the runtime expects and what it recieved.
+            // The cached value should not be used
             if (value === undefined || value.version !== persistedCacheValueVersion) {
                 return undefined;
             }
             assert(value.fluidEpoch !== undefined, 0x1dc /* "all entries have to have epoch" */);
             if (this._fluidEpoch === undefined) {
                 this.setEpoch(value.fluidEpoch, true, "cache");
+            // Epoch mismatch, the cached value is considerably different from what the current state of
+            // the runtime and should not be used
             } else if (this._fluidEpoch !== value.fluidEpoch) {
                 return undefined;
             }
+            // Expire the cached snapshot if it's older than the defaultCacheExpiryTimeoutMs and immediately
+            // expire all old caches that do not have cacheEntryTime
+            if (entry.type === snapshotKey) {
+                const cacheTime = value.value?.cacheEntryTime;
+                const currentTime = Date.now();
+                if (cacheTime === undefined || currentTime - cacheTime >= defaultCacheExpiryTimeoutMs) {
+                    this.logger.sendTelemetryEvent(
+                        {
+                            eventName: "odspVersionsCacheExpired",
+                            duration: currentTime - cacheTime,
+                            maxCacheAgeMs: defaultCacheExpiryTimeoutMs,
+                        });
+                    await this.removeEntries();
+                    return undefined;
+                }
+            }
             // eslint-disable-next-line @typescript-eslint/no-unsafe-return
             return value.value;
         } catch (error) {
@@ -95,6 +117,11 @@ export class EpochTracker implements IPersistedFileCache {
 
     public async put(entry: IEntry, value: any) {
         assert(this._fluidEpoch !== undefined, 0x1dd /* "no epoch" */);
+        // For snapshots, the value should have the cacheEntryTime. This will be used to expire snapshots older
+        // than the defaultCacheExpiryTimeoutMs.
+        if (entry.type === snapshotKey) {
+            value.cacheEntryTime = value.cacheEntryTime ?? Date.now();
+        }
         const data: IVersionedValueWithEpoch = {
             value,
             version: persistedCacheValueVersion,
@@ -143,9 +170,9 @@ export class EpochTracker implements IPersistedFileCache {
         fetchType: FetchType,
         addInBody: boolean = false,
     ): Promise<IOdspResponse<T>> {
-        const clientCorelationId = this.formatClientCorelationId();
+        const clientCorrelationId = this.formatClientCorrelationId();
         // Add epoch in fetch request.
-        this.addEpochInRequest(fetchOptions, addInBody, clientCorelationId);
+        this.addEpochInRequest(fetchOptions, addInBody, clientCorrelationId);
         let epochFromResponse: string | undefined;
         return this.rateLimiter.schedule(
             async () => fetchAndParseAsJSONHelper<T>(url, fetchOptions),
@@ -154,7 +181,7 @@ export class EpochTracker implements IPersistedFileCache {
             this.validateEpochFromResponse(epochFromResponse, fetchType);
             response.commonSpoHeaders = {
                 ...response.commonSpoHeaders,
-                "X-RequestStats": clientCorelationId,
+                "X-RequestStats": clientCorrelationId,
             };
             return response;
         }).catch(async (error) => {
@@ -166,8 +193,7 @@ export class EpochTracker implements IPersistedFileCache {
             await this.checkForEpochError(error, epochFromResponse, fetchType);
             throw error;
         }).catch((error) => {
-            const fluidError = normalizeError(error, {props: {"X-RequestStats": clientCorelationId}});
-            // eslint-disable-next-line @typescript-eslint/no-throw-literal
+            const fluidError = normalizeError(error, { props: { XRequestStatsHeader: clientCorrelationId }});
             throw fluidError;
         });
     }
@@ -185,9 +211,9 @@ export class EpochTracker implements IPersistedFileCache {
         fetchType: FetchType,
         addInBody: boolean = false,
     ) {
-        const clientCorelationId = this.formatClientCorelationId();
+        const clientCorrelationId = this.formatClientCorrelationId();
         // Add epoch in fetch request.
-        this.addEpochInRequest(fetchOptions, addInBody, clientCorelationId);
+        this.addEpochInRequest(fetchOptions, addInBody, clientCorrelationId);
         let epochFromResponse: string | undefined;
         return this.rateLimiter.schedule(
             async () => fetchArray(url, fetchOptions),
@@ -196,7 +222,7 @@ export class EpochTracker implements IPersistedFileCache {
             this.validateEpochFromResponse(epochFromResponse, fetchType);
             response.commonSpoHeaders = {
                 ...response.commonSpoHeaders,
-                "X-RequestStats": clientCorelationId,
+                "X-RequestStats": clientCorrelationId,
             };
             return response;
         }).catch(async (error) => {
@@ -208,8 +234,7 @@ export class EpochTracker implements IPersistedFileCache {
             await this.checkForEpochError(error, epochFromResponse, fetchType);
             throw error;
         }).catch((error) => {
-            const fluidError = normalizeError(error, {props: {"X-RequestStats": clientCorelationId}});
-            // eslint-disable-next-line @typescript-eslint/no-throw-literal
+            const fluidError = normalizeError(error, { props: { XRequestStatsHeader: clientCorrelationId }});
             throw fluidError;
         });
     }
@@ -217,11 +242,11 @@ export class EpochTracker implements IPersistedFileCache {
     private addEpochInRequest(
         fetchOptions: RequestInit,
         addInBody: boolean,
-        clientCorelationId: string,
+        clientCorrelationId: string,
     ) {
         if (addInBody) {
             const headers: {[key: string]: string} = {};
-            headers["X-RequestStats"] = clientCorelationId;
+            headers["X-RequestStats"] = clientCorrelationId;
             if (this.fluidEpoch !== undefined) {
                 headers["x-fluid-epoch"] = this.fluidEpoch;
             }
@@ -234,7 +259,7 @@ export class EpochTracker implements IPersistedFileCache {
                 assert(fetchOptions.headers !== undefined, 0x282 /* "Headers should be present now" */);
                 fetchOptions.headers[key] = val;
             };
-            addHeader("X-RequestStats", clientCorelationId);
+            addHeader("X-RequestStats", clientCorrelationId);
             if (this.fluidEpoch !== undefined) {
                 addHeader("x-fluid-epoch", this.fluidEpoch);
             }
@@ -259,7 +284,7 @@ export class EpochTracker implements IPersistedFileCache {
         fetchOptions.body = formParams.join("\r\n");
     }
 
-    private formatClientCorelationId() {
+    private formatClientCorrelationId() {
         return `driverId=${this.driverId}, RequestNumber=${this.networkCallNumber++}`;
     }
 
@@ -268,7 +293,10 @@ export class EpochTracker implements IPersistedFileCache {
         fetchType: FetchTypeInternal,
         fromCache: boolean = false,
     ) {
-        this.checkForEpochErrorCore(epochFromResponse);
+        const error = this.checkForEpochErrorCore(epochFromResponse);
+        if (error !== undefined) {
+            throw error;
+        }
         if (epochFromResponse !== undefined) {
             if (this._fluidEpoch === undefined) {
                 this.setEpoch(epochFromResponse, fromCache, fetchType);
@@ -283,11 +311,9 @@ export class EpochTracker implements IPersistedFileCache {
         fromCache: boolean = false,
     ) {
         if (isFluidError(error) && error.errorType === DriverErrorType.fileOverwrittenInStorage) {
-            try {
-                // This will only throw if it is an epoch error.
-                this.checkForEpochErrorCore(epochFromResponse);
-            } catch (epochError) {
-                assert(isValidLegacyError(epochError),
+            const epochError = this.checkForEpochErrorCore(epochFromResponse);
+            if (epochError !== undefined) {
+                assert(isFluidError(epochError),
                     0x21f /* "epochError expected to be thrown by throwOdspNetworkError and of known type" */);
                 epochError.addTelemetryProperties({
                     fromCache,
@@ -297,7 +323,6 @@ export class EpochTracker implements IPersistedFileCache {
                 this.logger.sendErrorEvent({ eventName: "fileOverwrittenInStorage" }, epochError);
                 // If the epoch mismatches, then clear all entries for such file entry from cache.
                 await this.removeEntries();
-                // eslint-disable-next-line @typescript-eslint/no-throw-literal
                 throw epochError;
             }
             // If it was categorized as epoch error but the epoch returned in response matches with the client epoch
@@ -312,7 +337,9 @@ export class EpochTracker implements IPersistedFileCache {
         // initializes this value. Sometimes response does not contain epoch as it is still in
         // implementation phase at server side. In that case also, don't compare it with our epoch value.
         if (this.fluidEpoch && epochFromResponse && (this.fluidEpoch !== epochFromResponse)) {
-            throwOdspNetworkError("epochMismatch", fluidEpochMismatchError);
+            // This is similar in nature to how fluidEpochMismatchError (409) is handled.
+            // Difference - client detected mismatch, instead of server detecting it.
+            return new NonRetryableError("epochMismatch", "Epoch mismatch", DriverErrorType.fileOverwrittenInStorage);
         }
     }
 

package/src/fetchSnapshot.ts

@@ -101,10 +101,11 @@ export async function fetchSnapshotWithRedeem(
             logger.sendErrorEvent({
                 eventName: "RedeemFallback",
                 errorType: error.errorType,
-            });
+            }, error);
             await redeemSharingLink(odspResolvedUrl, storageTokenFetcher, logger);
             const odspResolvedUrlWithoutShareLink: IOdspResolvedUrl =
                 { ...odspResolvedUrl, sharingLinkToRedeem: undefined };
+
             if(odspResolvedUrlWithoutShareLink.shareLinkInfo) {
                 odspResolvedUrlWithoutShareLink.shareLinkInfo = {
                     ...odspResolvedUrlWithoutShareLink.shareLinkInfo,

package/src/getFileLink.ts

@@ -4,9 +4,10 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { delay, PromiseCache } from "@fluidframework/common-utils";
-import { canRetryOnError, getRetryDelayFromError } from "@fluidframework/driver-utils";
+import { assert, delay } from "@fluidframework/common-utils";
+import { canRetryOnError, getRetryDelayFromError, NonRetryableError } from "@fluidframework/driver-utils";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
+import { DriverErrorType } from "@fluidframework/driver-definitions";
 import {
     IOdspUrlParts,
     OdspResourceTokenFetchOptions,
@@ -18,7 +19,7 @@ import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth";
 import { fetchHelper, getWithRetryForTokenRefresh } from "./odspUtils";
 
 // Store cached responses for the lifetime of web session as file link remains the same for given file item
-const fileLinkCache = new PromiseCache<string, string | undefined>();
+const fileLinkCache = new Map<string, Promise<string>>();
 
 /**
  * Returns file link for a file with given drive and item ids.
@@ -39,10 +40,11 @@ export async function getFileLink(
     odspUrlParts: IOdspUrlParts,
     identityType: IdentityType,
     logger: ITelemetryLogger,
-): Promise<string | undefined> {
+): Promise<string> {
     const cacheKey = `${odspUrlParts.siteUrl}_${odspUrlParts.driveId}_${odspUrlParts.itemId}`;
-    if (fileLinkCache.has(cacheKey)) {
-        return fileLinkCache.get(cacheKey);
+    const maybeFileLinkCacheEntry = fileLinkCache.get(cacheKey);
+    if (maybeFileLinkCacheEntry !== undefined) {
+        return maybeFileLinkCacheEntry;
     }
 
     const valueGenerator = async function() {
@@ -54,10 +56,11 @@ export async function getFileLink(
                 result = await getFileLinkCore(getToken, odspUrlParts, identityType, logger);
                 success = true;
             } catch (err) {
-                // If it is not retriable, then just return undefined
+                // If it is not retriable, then just throw
                 if (!canRetryOnError(err)) {
-                    fileLinkCache.remove(cacheKey);
-                    return undefined;
+                    // Delete from the cache to permit retrying later.
+                    fileLinkCache.delete(cacheKey);
+                    throw err;
                 }
                 // If the error is throttling error, then wait for the specified time before retrying.
                 // If the waitTime is not specified, then we start with retrying immediately to max of 8s.
@@ -65,10 +68,14 @@ export async function getFileLink(
                 await delay(retryAfterMs);
             }
         } while (!success);
+
+        // We are guaranteed to run the getFileLinkCore at least once with successful result (which must be a string)
+        assert(result !== undefined, 0x292 /* "Unexpected undefined result from getFileLinkCore" */);
         return result;
     };
-    fileLinkCache.add(cacheKey, valueGenerator);
-    return fileLinkCache.get(cacheKey);
+    const fileLink = valueGenerator();
+    fileLinkCache.set(cacheKey, fileLink);
+    return fileLink;
 }
 
 async function getFileLinkCore(
@@ -76,11 +83,8 @@ async function getFileLinkCore(
     odspUrlParts: IOdspUrlParts,
     identityType: IdentityType,
     logger: ITelemetryLogger,
-): Promise<string | undefined> {
+): Promise<string> {
     const fileItem = await getFileItemLite(getToken, odspUrlParts, logger);
-    if (!fileItem) {
-        return undefined;
-    }
 
     // ODC canonical link does not require any additional processing
     if (identityType === "Consumer") {
@@ -111,11 +115,17 @@ async function getFileLinkCore(
                 };
                 const response = await fetchHelper(url, requestInit);
                 additionalProps = response.commonSpoHeaders;
-                if (response.content.ok) {
-                    const sharingInfo = await response.content.json();
-                    return sharingInfo?.d?.directUrl as string;
+
+                const sharingInfo = await response.content.json();
+                const directUrl = sharingInfo?.d?.directUrl;
+                if (typeof directUrl !== "string") {
+                    // This will retry once in getWithRetryForTokenRefresh
+                    throw new NonRetryableError(
+                        "getFileLinkCoreMalformedResponse",
+                        "Malformed GetSharingInformation response",
+                        DriverErrorType.incorrectServerResponse);
                 }
-                return undefined;
+                return directUrl;
             });
             event.end({ ...additionalProps, attempts });
             return fileLink;
@@ -131,11 +141,18 @@ interface FileItemLite {
     webDavUrl: string;
 }
 
+const isFileItemLite = (maybeFileItemLite: any): maybeFileItemLite is FileItemLite => {
+    if (typeof maybeFileItemLite.webUrl !== "string" || typeof maybeFileItemLite.webDavUrl !== "string") {
+        return false;
+    }
+    return true;
+};
+
 async function getFileItemLite(
     getToken: TokenFetcher<OdspResourceTokenFetchOptions>,
     odspUrlParts: IOdspUrlParts,
     logger: ITelemetryLogger,
-): Promise<FileItemLite | undefined> {
+): Promise<FileItemLite> {
     return PerformanceEvent.timedExecAsync(
         logger,
         { eventName: "odspFileLink", requestName: "getFileItemLite" },
@@ -153,16 +170,19 @@ async function getFileItemLite(
                 const requestInit = { method: "GET", headers };
                 const response = await fetchHelper(url, requestInit);
                 additionalProps = response.commonSpoHeaders;
-                if (response.content.ok) {
-                    return await response.content.json() as FileItemLite;
+
+                const responseJson = await response.content.json();
+                if (!isFileItemLite(responseJson)) {
+                    // This will retry once in getWithRetryForTokenRefresh
+                    throw new NonRetryableError(
+                        "getFileItemLiteMalformedResponse",
+                        "Malformed getFileItemLite response",
+                        DriverErrorType.incorrectServerResponse);
                 }
-                return undefined;
+                return responseJson;
             });
             event.end({ ...additionalProps, attempts });
-            if (fileItem && fileItem.webDavUrl && fileItem.webUrl) {
-                return fileItem;
-            }
-            return undefined;
+            return fileItem;
         },
     );
 }

package/src/odspDeltaStorageService.ts

@@ -41,6 +41,7 @@ export class OdspDeltaStorageService {
         from: number,
         to: number,
         telemetryProps: ITelemetryProperties,
+        fetchReason?: string,
     ): Promise<IDeltasFetchResult> {
         return getWithRetryForTokenRefresh(async (options) => {
             // Note - this call ends up in getSocketStorageDiscovery() and can refresh token
@@ -52,6 +53,9 @@ export class OdspDeltaStorageService {
             let postBody = `--${formBoundary}\r\n`;
             postBody += `Authorization: Bearer ${storageToken}\r\n`;
             postBody += `X-HTTP-Method-Override: GET\r\n`;
+            if (fetchReason !== undefined) {
+                postBody += `fetchReason: ${fetchReason}\r\n`;
+            }
 
             postBody += `_post: 1\r\n`;
             postBody += `\r\n--${formBoundary}--`;
@@ -123,7 +127,8 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
         private readonly getFromStorage: (
             from: number,
             to: number,
-            telemetryProps: ITelemetryProperties) => Promise<IDeltasFetchResult>,
+            telemetryProps: ITelemetryProperties,
+            fetchReason?: string) => Promise<IDeltasFetchResult>,
         private readonly getCached: (from: number, to: number) => Promise<ISequencedDocumentMessage[]>,
         private readonly requestFromSocket: (from: number, to: number) => void,
         private readonly opsReceived: (ops: ISequencedDocumentMessage[]) => void,
@@ -151,7 +156,8 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
         fromTotal: number,
         toTotal: number | undefined,
         abortSignal?: AbortSignal,
-        cachedOnly?: boolean)
+        cachedOnly?: boolean,
+        fetchReason?: string)
     {
         // We do not control what's in the cache. Current API assumes that fetchMessages() keeps banging on
         // storage / cache until it gets ops it needs. This would result in deadlock if fixed range is asked from
@@ -198,7 +204,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
                 return { messages: [], partialResult: false };
             }
 
-            const ops = await this.getFromStorage(from, to, telemetryProps);
+            const ops = await this.getFromStorage(from, to, telemetryProps, fetchReason);
             this.validateMessages("storage", ops.messages, from);
             opsFromStorage += ops.messages.length;
             this.opsReceived(ops.messages);
@@ -220,6 +226,7 @@ export class OdspDeltaStorageWithCache implements IDocumentDeltaStorageService {
             this.batchSize,
             this.logger,
             abortSignal,
+            fetchReason,
         );
 
         return streamObserver(stream, (result) => {

package/src/odspDocumentDeltaConnection.ts

@@ -20,6 +20,7 @@ import { v4 as uuid } from "uuid";
 import { IOdspSocketError, IGetOpsResponse, IFlushOpsResponse } from "./contracts";
 import { EpochTracker } from "./epochTracker";
 import { errorObjectFromSocketError } from "./odspError";
+import { pkgVersion } from "./packageVersion";
 
 const protocolVersions = ["^0.4.0", "^0.3.0", "^0.2.0", "^0.1.0"];
 const feature_get_ops = "api_get_ops";
@@ -226,6 +227,7 @@ export class OdspDocumentDeltaConnection extends DocumentDeltaConnection {
             versions: protocolVersions,
             nonce: uuid(),
             epoch: epochTracker.fluidEpoch,
+            relayUserAgent: [client.details.environment, ` driverVersion:${pkgVersion}`].join(";"),
         };
 
         // Reference to this client supporting get_ops flow.

package/src/odspDocumentService.ts

@@ -14,8 +14,8 @@ import {
     IDocumentStorageService,
     IDocumentServicePolicies,
 } from "@fluidframework/driver-definitions";
-import { DeltaStreamConnectionForbiddenError } from "@fluidframework/driver-utils";
-import { fetchTokenErrorCode, IFacetCodes, throwOdspNetworkError } from "@fluidframework/odsp-doclib-utils";
+import { DeltaStreamConnectionForbiddenError, NonRetryableError } from "@fluidframework/driver-utils";
+import { IFacetCodes } from "@fluidframework/odsp-doclib-utils";
 import {
     IClient,
     ISequencedDocumentMessage,
@@ -26,6 +26,7 @@ import {
     IEntry,
     HostStoragePolicy,
     InstrumentedStorageTokenFetcher,
+    OdspErrorType,
 } from "@fluidframework/odsp-driver-definitions";
 import { HostStoragePolicyInternal, ISocketStorageDiscovery } from "./contracts";
 import { IOdspCache } from "./odspCache";
@@ -211,7 +212,7 @@ export class OdspDocumentService implements IDocumentService {
             this.logger,
             batchSize,
             concurrency,
-            async (from, to, telemetryProps) => service.get(from, to, telemetryProps),
+            async (from, to, telemetryProps, fetchReason) => service.get(from, to, telemetryProps, fetchReason),
             async (from, to) => {
                 const res = await this.opsCache?.get(from, to);
                 return res as ISequencedDocumentMessage[] ?? [];
@@ -270,7 +271,10 @@ export class OdspDocumentService implements IDocumentService {
 
             const finalWebsocketToken = websocketToken ?? (websocketEndpoint.socketToken || null);
             if (finalWebsocketToken === null) {
-                throwOdspNetworkError("pushTokenIsNull", fetchTokenErrorCode);
+                throw new NonRetryableError(
+                    "pushTokenIsNull",
+                    "Websocket token is null",
+                    OdspErrorType.fetchTokenError);
             }
             try {
                 const connection = await this.connectToDeltaStreamWithRetry(