@fluidframework/odsp-driver 1.2.1 → 2.0.0-internal.1.0.0

package/src/epochTracker.ts

@@ -6,7 +6,12 @@
 import { v4 as uuid } from "uuid";
 import { assert, Deferred } from "@fluidframework/common-utils";
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { ThrottlingError, RateLimiter, NonRetryableError } from "@fluidframework/driver-utils";
+import {
+    ThrottlingError,
+    RateLimiter,
+    NonRetryableError,
+    LocationRedirectionError,
+} from "@fluidframework/driver-utils";
 import { IConnected } from "@fluidframework/protocol-definitions";
 import {
     snapshotKey,
@@ -15,9 +20,16 @@ import {
     IFileEntry,
     IPersistedCache,
     IOdspError,
+    IOdspErrorAugmentations,
+    IOdspResolvedUrl,
 } from "@fluidframework/odsp-driver-definitions";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
-import { PerformanceEvent, isFluidError, normalizeError } from "@fluidframework/telemetry-utils";
+import {
+    PerformanceEvent,
+    isFluidError,
+    normalizeError,
+    loggerToMonitoringContext,
+} from "@fluidframework/telemetry-utils";
 import { fetchAndParseAsJSONHelper, fetchArray, fetchHelper, getOdspResolvedUrl, IOdspResponse } from "./odspUtils";
 import {
     IOdspCache,
@@ -27,6 +39,7 @@ import {
 import { IVersionedValueWithEpoch, persistedCacheValueVersion } from "./contracts";
 import { ClpCompliantAppHeader } from "./contractsPublic";
 import { pkgVersion as driverVersion } from "./packageVersion";
+import { patchOdspResolvedUrl } from "./odspLocationRedirection";
 
 export type FetchType = "blob" | "createBlob" | "createFile" | "joinSession" | "ops" | "test" | "snapshotTree" |
     "treesLatest" | "uploadSummary" | "push" | "versions";
@@ -47,6 +60,7 @@ export const defaultCacheExpiryTimeoutMs: number = 2 * 24 * 60 * 60 * 1000;
 export class EpochTracker implements IPersistedFileCache {
     private _fluidEpoch: string | undefined;
 
+    private readonly snapshotCacheExpiryTimeoutMs: number;
     public readonly rateLimiter: RateLimiter;
     private readonly driverId = uuid();
     // This tracks the request number made by the driver instance.
@@ -59,6 +73,12 @@ export class EpochTracker implements IPersistedFileCache {
     ) {
         // Limits the max number of concurrent requests to 24.
         this.rateLimiter = new RateLimiter(24);
+
+        // We need this for GC testing until we properly plumb through the snapshot expiration policy (see PR #11168)
+        this.snapshotCacheExpiryTimeoutMs =
+            loggerToMonitoringContext(logger).config.getBoolean("Fluid.Driver.Odsp.TestOverride.DisableSnapshotCache")
+                ? 0
+                : defaultCacheExpiryTimeoutMs;
     }
 
     // public for UT purposes only!
@@ -95,17 +115,17 @@ export class EpochTracker implements IPersistedFileCache {
             } else if (this._fluidEpoch !== value.fluidEpoch) {
                 return undefined;
             }
-            // Expire the cached snapshot if it's older than the defaultCacheExpiryTimeoutMs and immediately
+            // Expire the cached snapshot if it's older than snapshotCacheExpiryTimeoutMs and immediately
             // expire all old caches that do not have cacheEntryTime
             if (entry.type === snapshotKey) {
                 const cacheTime = value.value?.cacheEntryTime;
                 const currentTime = Date.now();
-                if (cacheTime === undefined || currentTime - cacheTime >= defaultCacheExpiryTimeoutMs) {
+                if (cacheTime === undefined || currentTime - cacheTime >= this.snapshotCacheExpiryTimeoutMs) {
                     this.logger.sendTelemetryEvent(
                         {
                             eventName: "odspVersionsCacheExpired",
                             duration: currentTime - cacheTime,
-                            maxCacheAgeMs: defaultCacheExpiryTimeoutMs,
+                            maxCacheAgeMs: this.snapshotCacheExpiryTimeoutMs,
                         });
                     await this.removeEntries();
                     return undefined;
@@ -121,8 +141,8 @@ export class EpochTracker implements IPersistedFileCache {
 
     public async put(entry: IEntry, value: any) {
         assert(this._fluidEpoch !== undefined, 0x1dd /* "no epoch" */);
-        // For snapshots, the value should have the cacheEntryTime. This will be used to expire snapshots older
-        // than the defaultCacheExpiryTimeoutMs.
+        // For snapshots, the value should have the cacheEntryTime.
+        // This will be used to expire snapshots older than snapshotCacheExpiryTimeoutMs.
         if (entry.type === snapshotKey) {
             value.cacheEntryTime = value.cacheEntryTime ?? Date.now();
         }
@@ -224,6 +244,26 @@ export class EpochTracker implements IPersistedFileCache {
             }
             await this.checkForEpochError(error, epochFromResponse, fetchType);
             throw error;
+        }).catch((error) => {
+            // If the error is about location redirection, then we need to generate new resolved url with correct
+            // location info.
+            if (isFluidError(error) && error.errorType === DriverErrorType.fileNotFoundOrAccessDeniedError) {
+                const redirectLocation = (error as IOdspErrorAugmentations).redirectLocation;
+                if (redirectLocation !== undefined) {
+                    const redirectUrl: IOdspResolvedUrl = patchOdspResolvedUrl(
+                        this.fileEntry.resolvedUrl,
+                        redirectLocation,
+                    );
+                    const locationRedirectionError = new LocationRedirectionError(
+                        error.message,
+                        redirectUrl,
+                        { driverVersion, redirectLocation },
+                    );
+                    locationRedirectionError.addTelemetryProperties(error.getTelemetryProperties());
+                    throw locationRedirectionError;
+                }
+            }
+            throw error;
         }).catch((error) => {
             const fluidError = normalizeError(error, { props: { XRequestStatsHeader: clientCorrelationId } });
             throw fluidError;

package/src/fetchSnapshot.ts

@@ -16,7 +16,7 @@ import {
     InstrumentedStorageTokenFetcher,
 } from "@fluidframework/odsp-driver-definitions";
 import { ISnapshotTree } from "@fluidframework/protocol-definitions";
-import { isRuntimeMessage, NonRetryableError } from "@fluidframework/driver-utils";
+import { DriverErrorTelemetryProps, isRuntimeMessage, NonRetryableError } from "@fluidframework/driver-utils";
 import { IOdspSnapshot, ISnapshotCachedEntry, IVersionedValueWithEpoch, persistedCacheValueVersion } from "./contracts";
 import { getQueryString } from "./getQueryString";
 import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth";
@@ -234,19 +234,30 @@ async function fetchLatestSnapshotCore(
                     snapshotOptions,
                     controller,
                 );
+
                 const odspResponse = response.odspResponse;
                 const contentType = odspResponse.headers.get("content-type");
-                odspResponse.propsToLog = {
+
+                const propsToLog: DriverErrorTelemetryProps = {
                     ...odspResponse.propsToLog,
                     contentType,
                     accept: response.requestHeaders.accept,
+                    driverVersion: pkgVersion,
                 };
+
+                // Measure how much time we spend processing payload
+                const snapshotParseEvent = PerformanceEvent.start(logger, {
+                    eventName: "SnapshotParse",
+                    ...propsToLog,
+                });
+
                 let parsedSnapshotContents: IOdspResponse<ISnapshotContents> | undefined;
+
                 try {
                     switch (contentType) {
                         case "application/json": {
                             const text = await odspResponse.content.text();
-                            odspResponse.propsToLog.bodySize = text.length;
+                            propsToLog.bodySize = text.length;
                             const content: IOdspSnapshot = JSON.parse(text);
                             validateBlobsAndTrees(content);
                             const snapshotContents: ISnapshotContents =
@@ -256,7 +267,7 @@ async function fetchLatestSnapshotCore(
                         }
                         case "application/ms-fluid": {
                             const content = await odspResponse.content.arrayBuffer();
-                            odspResponse.propsToLog.bodySize = content.byteLength;
+                            propsToLog.bodySize = content.byteLength;
                             const snapshotContents: ISnapshotContents = parseCompactSnapshotResponse(
                                 new ReadBuffer(new Uint8Array(content)));
                             if (snapshotContents.snapshotTree.trees === undefined ||
@@ -264,7 +275,7 @@ async function fetchLatestSnapshotCore(
                                     throw new NonRetryableError(
                                         "Returned odsp snapshot is malformed. No trees or blobs!",
                                         DriverErrorType.incorrectServerResponse,
-                                        { driverVersion: pkgVersion, ...odspResponse.propsToLog },
+                                        propsToLog,
                                     );
                                 }
                             parsedSnapshotContents = { ...odspResponse, content: snapshotContents };
@@ -274,12 +285,12 @@ async function fetchLatestSnapshotCore(
                             throw new NonRetryableError(
                                 "Unknown snapshot content type",
                                 DriverErrorType.incorrectServerResponse,
-                                { driverVersion: pkgVersion, ...odspResponse.propsToLog },
+                                propsToLog,
                             );
                     }
                 } catch (error) {
                     if (isFluidError(error)) {
-                        error.addTelemetryProperties({ driverVersion: pkgVersion, ...odspResponse.propsToLog });
+                        error.addTelemetryProperties(propsToLog);
                         throw error;
                     }
                     const enhancedError = wrapError(
@@ -287,11 +298,14 @@ async function fetchLatestSnapshotCore(
                         (errorMessage) => new NonRetryableError(
                             `Error parsing snapshot response: ${errorMessage}`,
                             DriverErrorType.genericError,
-                            { driverVersion: pkgVersion, ...odspResponse.propsToLog }));
+                            propsToLog));
                     throw enhancedError;
                 }
+
                 assert(parsedSnapshotContents !== undefined, 0x312 /* snapshot should be parsed */);
                 const snapshot = parsedSnapshotContents.content;
+                const { trees, numBlobs, encodedBlobsSize } = evalBlobsAndTrees(snapshot);
+
                 // From: https://developer.mozilla.org/en-US/docs/Web/API/PerformanceResourceTiming
                 // fetchStart: immediately before the browser starts to fetch the resource.
                 // requestStart: immediately before the browser starts requesting the resource from the server
@@ -344,9 +358,6 @@ async function fetchLatestSnapshotCore(
                     }
                 }
 
-                const { numTrees, numBlobs, encodedBlobsSize } =
-                    evalBlobsAndTrees(parsedSnapshotContents.content);
-
                 // There are some scenarios in ODSP where we cannot cache, trees/latest will explicitly tell us when we
                 // cannot cache using an HTTP response header.
                 const canCache =
@@ -375,8 +386,11 @@ async function fetchLatestSnapshotCore(
                     // eslint-disable-next-line @typescript-eslint/no-floating-promises
                     putInCache(valueWithEpoch);
                 }
+
+                snapshotParseEvent.end();
+
                 event.end({
-                    trees: numTrees,
+                    trees,
                     blobs: snapshot.blobs?.size ?? 0,
                     leafNodes: numBlobs,
                     encodedBlobsSize,
@@ -406,7 +420,7 @@ async function fetchLatestSnapshotCore(
                     // Azure Fluid Relay service is the redeem status (S means success), and FRP is a flag to indicate
                     // if the permission has changed.
                     sltelemetry: odspResponse.headers.get("x-fluid-sltelemetry"),
-                    ...odspResponse.propsToLog,
+                    ...propsToLog,
                 });
                 return snapshot;
             },
@@ -466,13 +480,13 @@ function getFormBodyAndHeaders(
 }
 
 function evalBlobsAndTrees(snapshot: ISnapshotContents) {
-    const numTrees = countTreesInSnapshotTree(snapshot.snapshotTree);
+    const trees = countTreesInSnapshotTree(snapshot.snapshotTree);
     const numBlobs = snapshot.blobs.size;
     let encodedBlobsSize = 0;
     for (const [_, blobContent] of snapshot.blobs) {
         encodedBlobsSize += blobContent.byteLength;
     }
-    return { numTrees, numBlobs, encodedBlobsSize };
+    return { trees, numBlobs, encodedBlobsSize };
 }
 
 export function validateBlobsAndTrees(snapshot: IOdspSnapshot) {

package/src/getFileLink.ts

@@ -4,19 +4,15 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { assert, delay } from "@fluidframework/common-utils";
-import { canRetryOnError, getRetryDelayFromError, NonRetryableError } from "@fluidframework/driver-utils";
+import { assert } from "@fluidframework/common-utils";
+import { canRetryOnError, NonRetryableError } from "@fluidframework/driver-utils";
 import { PerformanceEvent } from "@fluidframework/telemetry-utils";
 import { DriverErrorType } from "@fluidframework/driver-definitions";
-import {
-    IOdspUrlParts,
-    OdspResourceTokenFetchOptions,
-    IdentityType,
-    TokenFetcher,
-} from "@fluidframework/odsp-driver-definitions";
+import { IOdspUrlParts, OdspResourceTokenFetchOptions, TokenFetcher } from "@fluidframework/odsp-driver-definitions";
 import { getUrlAndHeadersWithAuth } from "./getUrlAndHeadersWithAuth";
 import { fetchHelper, getWithRetryForTokenRefresh, toInstrumentedOdspTokenFetcher } from "./odspUtils";
 import { pkgVersion as driverVersion } from "./packageVersion";
+import { runWithRetry } from "./retryUtils";
 
 // Store cached responses for the lifetime of web session as file link remains the same for given file item
 const fileLinkCache = new Map<string, Promise<string>>();
@@ -28,17 +24,13 @@ const fileLinkCache = new Map<string, Promise<string>>();
  * throttling error. In future, we are thinking of app allowing to pass some cancel token, with which
  * we would be able to stop retrying.
  * @param getToken - used to fetch access tokens needed to execute operation
- * @param siteUrl - url of the site that contains the file
- * @param driveId - drive where file is stored
- * @param itemId - file id
- * @param identityType - type of client account
+ * @param odspUrlParts - object describing file storage identity
  * @param logger - used to log results of operation, including any error
  * @returns Promise which resolves to file link url when successful; otherwise, undefined.
  */
 export async function getFileLink(
     getToken: TokenFetcher<OdspResourceTokenFetchOptions>,
     odspUrlParts: IOdspUrlParts,
-    identityType: IdentityType,
     logger: ITelemetryLogger,
 ): Promise<string> {
     const cacheKey = `${odspUrlParts.siteUrl}_${odspUrlParts.driveId}_${odspUrlParts.itemId}`;
@@ -47,33 +39,29 @@ export async function getFileLink(
         return maybeFileLinkCacheEntry;
     }
 
-    const valueGenerator = async function() {
-        let result: string | undefined;
-        let success = false;
-        let retryAfterMs = 1000;
-        do {
-            try {
-                result = await getFileLinkCore(getToken, odspUrlParts, identityType, logger);
-                success = true;
-            } catch (err) {
-                // If it is not retriable, then just throw
-                if (!canRetryOnError(err)) {
-                    // Delete from the cache to permit retrying later.
-                    fileLinkCache.delete(cacheKey);
-                    throw err;
-                }
-                // If the error is throttling error, then wait for the specified time before retrying.
-                // If the waitTime is not specified, then we start with retrying immediately to max of 8s.
-                retryAfterMs = getRetryDelayFromError(err) ?? Math.min(retryAfterMs * 2, 8000);
-                await delay(retryAfterMs);
+    const fileLinkGenerator = async function() {
+        let fileLinkCore: string;
+        try {
+            fileLinkCore = await runWithRetry(
+                async () => getFileLinkCore(getToken, odspUrlParts, logger),
+                "getFileLinkCore",
+                logger,
+            );
+        } catch (err) {
+            // runWithRetry throws a non retriable error after it hits the max # of attempts
+            // or encounters an unexpected error type
+            if (!canRetryOnError(err)) {
+                // Delete from the cache to permit retrying later.
+                fileLinkCache.delete(cacheKey);
             }
-        } while (!success);
+            throw err;
+        }
 
         // We are guaranteed to run the getFileLinkCore at least once with successful result (which must be a string)
-        assert(result !== undefined, 0x292 /* "Unexpected undefined result from getFileLinkCore" */);
-        return result;
+        assert(fileLinkCore !== undefined, 0x292 /* "Unexpected undefined result from getFileLinkCore" */);
+        return fileLinkCore;
     };
-    const fileLink = valueGenerator();
+    const fileLink = fileLinkGenerator();
     fileLinkCache.set(cacheKey, fileLink);
     return fileLink;
 }
@@ -81,15 +69,14 @@ export async function getFileLink(
 async function getFileLinkCore(
     getToken: TokenFetcher<OdspResourceTokenFetchOptions>,
     odspUrlParts: IOdspUrlParts,
-    identityType: IdentityType,
     logger: ITelemetryLogger,
 ): Promise<string> {
-    const fileItem = await getFileItemLite(getToken, odspUrlParts, logger, identityType === "Consumer");
+    const fileItem = await getFileItemLite(getToken, odspUrlParts, logger, true);
 
     // ODSP link requires extra call to return link that is resistant to file being renamed or moved to different folder
     return PerformanceEvent.timedExecAsync(
         logger,
-        { eventName: "odspFileLink", requestName: "getSharingLink" },
+        { eventName: "odspFileLink", requestName: "getSharingInformation" },
         async (event) => {
             let attempts = 0;
             let additionalProps;
@@ -106,11 +93,11 @@ async function getFileLinkCore(
                     0x2bb /* "Instrumented token fetcher with throwOnNullToken = true should never return null" */);
 
                 const { url, headers } = getUrlAndHeadersWithAuth(
-                    `${odspUrlParts.siteUrl}/_api/web/GetFileByServerRelativeUrl(@a1)/Linkingurl?@a1=${
-                        encodeURIComponent(`'${new URL(fileItem.webDavUrl).pathname}'`)
+                    `${odspUrlParts.siteUrl}/_api/web/GetFileByUrl(@a1)/ListItemAllFields/GetSharingInformation?@a1=${
+                        encodeURIComponent(`'${fileItem.webDavUrl}'`)
                     }`,
                     storageToken,
-                    false,
+                    true,
                 );
                 const requestInit = {
                     method: "POST",
@@ -124,15 +111,15 @@ async function getFileLinkCore(
                 additionalProps = response.propsToLog;
 
                 const sharingInfo = await response.content.json();
-                const linkingUrl = sharingInfo?.d?.LinkingUrl;
-                if (typeof linkingUrl !== "string") {
+                const directUrl = sharingInfo?.d?.directUrl;
+                if (typeof directUrl !== "string") {
                     // This will retry once in getWithRetryForTokenRefresh
                     throw new NonRetryableError(
-                        "Malformed GetSharingLink response",
+                        "Malformed GetSharingInformation response",
                         DriverErrorType.incorrectServerResponse,
                         { driverVersion });
                 }
-                return linkingUrl;
+                return directUrl;
             });
             event.end({ ...additionalProps, attempts });
             return fileLink;

package/src/odspDeltaStorageService.ts

@@ -35,13 +35,14 @@ export class OdspDeltaStorageService {
      * @param from - inclusive
      * @param to - exclusive
      * @param telemetryProps - properties to add when issuing telemetry events
+     * @param scenarioName - reason for fetching ops
      * @returns ops retrieved & info if result was partial (i.e. more is available)
      */
      public async get(
         from: number,
         to: number,
         telemetryProps: ITelemetryProperties,
-        fetchReason?: string,
+        scenarioName?: string,
     ): Promise<IDeltasFetchResult> {
         return getWithRetryForTokenRefresh(async (options) => {
             // Note - this call ends up in getSocketStorageDiscovery() and can refresh token
@@ -78,7 +79,7 @@ export class OdspDeltaStorageService {
                 },
                 "ops",
                 true,
-                fetchReason,
+                scenarioName,
             );
             clearTimeout(timer);
             const deltaStorageResponse = response.content;
@@ -99,6 +100,7 @@ export class OdspDeltaStorageService {
                 from,
                 to,
                 ...telemetryProps,
+                reason: scenarioName,
             });
 
             // It is assumed that server always returns all the ops that it has in the range that was requested.

package/src/odspDocumentService.ts

@@ -4,7 +4,7 @@
  */
 
 import { ITelemetryLogger } from "@fluidframework/common-definitions";
-import { performance } from "@fluidframework/common-utils";
+import { assert, performance } from "@fluidframework/common-utils";
 import {
     ChildLogger,
     IFluidErrorBase,
@@ -21,12 +21,7 @@ import {
     IDocumentServicePolicies,
     DriverErrorType,
 } from "@fluidframework/driver-definitions";
-import {
-    canRetryOnError,
-    DeltaStreamConnectionForbiddenError,
-    NonRetryableError,
-} from "@fluidframework/driver-utils";
-import { IFacetCodes } from "@fluidframework/odsp-doclib-utils";
+import { canRetryOnError, DeltaStreamConnectionForbiddenError, NonRetryableError } from "@fluidframework/driver-utils";
 import {
     IClient,
     ISequencedDocumentMessage,
@@ -39,6 +34,7 @@ import {
     InstrumentedStorageTokenFetcher,
     OdspErrorType,
 } from "@fluidframework/odsp-driver-definitions";
+import { hasFacetCodes } from "@fluidframework/odsp-doclib-utils";
 import type { io as SocketIOClientStatic } from "socket.io-client";
 import { HostStoragePolicyInternal, ISocketStorageDiscovery } from "./contracts";
 import { IOdspCache } from "./odspCache";
@@ -113,6 +109,8 @@ export class OdspDocumentService implements IDocumentService {
 
     private currentConnection?: OdspDocumentDeltaConnection;
 
+    private relayServiceTenantAndSessionId: string | undefined;
+
     /**
      * @param odspResolvedUrl - resolved url identifying document that will be managed by this service instance.
      * @param getStorageToken - function that can provide the storage token. This is is also referred to as
@@ -189,6 +187,11 @@ export class OdspDocumentService implements IDocumentService {
                     }
                     throw new Error("Disconnected while uploading summary (attempt to perform flush())");
                 },
+                () => {
+                    assert(this.relayServiceTenantAndSessionId !== undefined,
+                        0x37b /* relayServiceTenantAndSessionId should be present */);
+                    return this.relayServiceTenantAndSessionId;
+                },
                 this.mc.config.getNumber("Fluid.Driver.Odsp.snapshotFormatFetchType"),
             );
         }
@@ -271,7 +274,7 @@ export class OdspDocumentService implements IDocumentService {
                     this.socketIoClientFactory().catch(annotateAndRethrowConnectionError("socketIoClientFactory")),
                 ]);
 
-            const finalWebsocketToken = websocketToken ?? (websocketEndpoint.socketToken || null);
+            const finalWebsocketToken = websocketToken ?? (websocketEndpoint.socketToken ?? null);
             if (finalWebsocketToken === null) {
                 throw this.annotateConnectionError(
                     new NonRetryableError(
@@ -344,10 +347,9 @@ export class OdspDocumentService implements IDocumentService {
         requestSocketToken: boolean,
         options: TokenFetchOptionsEx,
     ) {
-        return this.joinSessionCore(requestSocketToken, options).catch((e) => {
-            const likelyFacetCodes = e as IFacetCodes;
-            if (Array.isArray(likelyFacetCodes.facetCodes)) {
-                for (const code of likelyFacetCodes.facetCodes) {
+        const response = await this.joinSessionCore(requestSocketToken, options).catch((e) => {
+            if (hasFacetCodes(e) && e.facetCodes !== undefined) {
+                for (const code of e.facetCodes) {
                     switch (code) {
                         case "sessionForbiddenOnPreservedFiles":
                         case "sessionForbiddenOnModerationEnabledLibrary":
@@ -364,6 +366,8 @@ export class OdspDocumentService implements IDocumentService {
             }
             throw e;
         });
+        this.relayServiceTenantAndSessionId = `${response.tenantId}/${response.id}`;
+        return response;
     }
 
     private async joinSessionCore(