@fluid-tools/fetch-tool 2.1.0-276326 → 2.1.0-281041

package/src/fluidFetchArgs.ts

@@ -15,15 +15,6 @@ export let paramSnapshotVersionIndex: number | undefined;
 export let paramNumSnapshotVersions = 10;
 export let paramActualFormatting = false;
 
-let paramForceTokenReauth = false;
-
-// Only return true once, to reauth on first call.
-export function getForceTokenReauth() {
-	const result = paramForceTokenReauth;
-	paramForceTokenReauth = false;
-	return result;
-}
-
 export let paramSaveDir: string | undefined;
 export const messageTypeFilter = new Set<string>();
 
@@ -33,12 +24,12 @@ export let paramJWT: string;
 export let connectToWebSocket = false;
 
 export let localDataOnly = false;
+export let loginHint: string | undefined;
 
 const optionsArray = [
 	["--dump:rawmessage", "dump all messages"],
 	["--dump:snapshotVersion", "dump a list of snapshot version"],
 	["--dump:snapshotTree", "dump the snapshot trees"],
-	["--forceTokenReauth", "Force reauthorize token (SPO only)"],
 	["--stat:message", "show message type, channel type, data type statistics"],
 	["--stat:snapshot", "show a table of snapshot path and blob size"],
 	["--stat", "Show both messages & snapshot stats"],
@@ -53,6 +44,7 @@ const optionsArray = [
 	["--snapshotVersionIndex <number>", "Index of the version to dump"],
 	["--websocket", "Connect to web socket to download initial messages"],
 	["--local", "Do not connect to storage, use earlier downloaded data. Requires --saveDir."],
+	["--loginHint", "login hint for the user with document access."],
 ];
 
 function printUsage() {
@@ -120,9 +112,6 @@ export function parseArguments() {
 			case "--jwt":
 				paramJWT = parseStrArg(i++, "jwt token");
 				break;
-			case "--forceTokenReauth":
-				paramForceTokenReauth = true;
-				break;
 			case "--snapshotVersionIndex":
 				paramSnapshotVersionIndex = parseIntArg(i++, "version index", true);
 				break;
@@ -141,6 +130,9 @@ export function parseArguments() {
 			case "--local":
 				localDataOnly = true;
 				break;
+			case "--loginHint":
+				loginHint = parseStrArg(i++, "login hint");
+				break;
 			default:
 				try {
 					const url = new URL(arg);

package/src/fluidFetchInit.ts

@@ -65,8 +65,6 @@ async function initializeODSPCore(
 			},
 			server,
 			clientConfig,
-			undefined,
-			true,
 		);
 	};
 	// eslint-disable-next-line @typescript-eslint/promise-function-async

package/src/fluidFetchMessages.ts

@@ -96,28 +96,44 @@ async function* loadAllSequencedMessages(
 	let requests = 0;
 	let opsStorage = 0;
 
+	console.log("fetching cached messages");
 	// reading only 1 op to test if there is mismatch
 	const teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);
 
-	let statusCode;
-	let innerMostErrorCode;
-	let response;
-
 	try {
 		await teststream.read();
 	} catch (error: any) {
-		statusCode = error.getTelemetryProperties().statusCode;
-		innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
-		// if there is gap between ops, catch the error and check it is the error we need
+		const statusCode = error.getTelemetryProperties().statusCode;
+		const innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
 		if (statusCode !== 410 || innerMostErrorCode !== "fluidDeltaDataNotAvailable") {
 			throw error;
 		}
-		// get firstAvailableDelta from the error response, and set current sequence number to that
-		response = JSON.parse(error.getTelemetryProperties().response);
-		firstAvailableDelta = response.error.firstAvailableDelta;
-		lastSeq = firstAvailableDelta - 1;
+
+		// This indicates we tried to fetch ops from storage that have been deleted (because they are past some retention policy).
+		// In that case, the error message should indicate the first sequence number that is available.
+		// We make a best-effort attempt for the original query (fetch all ops) by starting from that sequence number.
+		const props = error.getTelemetryProperties();
+		const { responseMessage } = props;
+		const [_, seq] =
+			typeof responseMessage === "string"
+				? responseMessage.match(/GenesisSequenceNumber '(\d+)'/) ?? []
+				: [];
+		if (seq !== undefined) {
+			lastSeq = parseInt(seq, 10);
+			firstAvailableDelta = lastSeq + 1;
+			console.log(
+				`Not all ops are available (older ops may have been deleted from storage). Starting from sequenceNumber: ${firstAvailableDelta}.`,
+			);
+		} else {
+			console.log(props);
+			throw new Error(
+				`Unexpected structure for 410 error: ${error.message}. Further error properties were logged above. This indicates a problem with fetch-tool.`,
+			);
+		}
 	}
 
+	console.log("fetching remaining messages from delta storage");
+
 	// continue reading rest of the ops
 	const stream = deltaStorage.fetchMessages(
 		lastSeq + 1, // inclusive left

package/src/fluidFetchSharePoint.ts

@@ -3,8 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import child_process from "child_process";
-
+import { InteractiveBrowserCredential, useIdentityPlugin } from "@azure/identity";
+import { cachePersistencePlugin } from "@azure/identity-cache-persistence";
 import { DriverErrorTypes } from "@fluidframework/driver-definitions/internal";
 import {
 	IPublicClientConfig,
@@ -13,16 +13,16 @@ import {
 	getChildrenByDriveItem,
 	getDriveItemByServerRelativePath,
 	getDriveItemFromDriveAndItem,
-	getOdspRefreshTokenFn,
+	getAadTenant,
+	getOdspScope,
 } from "@fluidframework/odsp-doclib-utils/internal";
-import {
-	IOdspTokenManagerCacheKey,
-	OdspTokenConfig,
-	OdspTokenManager,
-	odspTokensCache,
-} from "@fluidframework/tool-utils/internal";
 
-import { getForceTokenReauth } from "./fluidFetchArgs.js";
+import { loginHint } from "./fluidFetchArgs.js";
+
+// Note: the following page may be helpful for debugging auth issues:
+// https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/identity/identity/TROUBLESHOOTING.md
+// See e.g. the section on setting 'AZURE_LOG_LEVEL'.
+useIdentityPlugin(cachePersistencePlugin);
 
 export const fetchToolClientConfig: IPublicClientConfig = {
 	get clientId(): string {
@@ -41,40 +41,43 @@ export async function resolveWrapper<T>(
 	server: string,
 	clientConfig: IPublicClientConfig,
 	forceTokenReauth = false,
-	forToken = false,
 ): Promise<T> {
 	try {
-		const odspTokenManager = new OdspTokenManager(odspTokensCache);
-		const tokenConfig: OdspTokenConfig = {
-			type: "browserLogin",
-			navigator: fluidFetchWebNavigator,
-		};
-		const tokens = await odspTokenManager.getOdspTokens(
-			server,
-			clientConfig,
-			tokenConfig,
-			undefined /* forceRefresh */,
-			forceTokenReauth || getForceTokenReauth(),
-		);
+		const credential = new InteractiveBrowserCredential({
+			clientId: fetchToolClientConfig.clientId,
+			tenantId: getAadTenant(server),
+			// NOTE: fetch-tool flows using multiple sets of user credentials haven't been well-tested.
+			// Some of the @azure/identity docs suggest we may need to manage authentication records and choose
+			// which one to use explicitly here if we have such scenarios.
+			// If we start doing this, it may be worth considering using disableAutomaticAuthentication here so we
+			// have better control over when interactive auth may be triggered.
+			// For now, fetch-tool doesn't work against personal accounts anyway so the only flow that might necessitate this
+			// would be grabbing documents using several identities (e.g. test accounts we use for stress testing).
+			// In that case, a simple workaround is to delete the cache that @azure/identity uses before running the tool.
+			// See docs on `tokenCachePersistenceOptions.name` for information on where this cache is stored.
+			loginHint,
+			tokenCachePersistenceOptions: {
+				enabled: true,
+				name: "fetch-tool",
+			},
+		});
+
+		const scope = getOdspScope(server);
 
-		const result = await callback({
-			accessToken: tokens.accessToken,
-			refreshTokenFn: getOdspRefreshTokenFn(server, clientConfig, tokens),
+		const { token } = await credential.getToken(scope);
+
+		return await callback({
+			accessToken: token,
+			refreshTokenFn: async () => {
+				await credential.authenticate(scope);
+				const result = await credential.getToken(scope);
+				return result.token;
+			},
 		});
-		// If this is used for getting a token, then refresh the cache with new token.
-		if (forToken) {
-			const key: IOdspTokenManagerCacheKey = { isPush: false, userOrServer: server };
-			await odspTokenManager.updateTokensCache(key, {
-				accessToken: result as any as string,
-				refreshToken: tokens.refreshToken,
-			});
-			return result;
-		}
-		return result;
 	} catch (e: any) {
 		if (e.errorType === DriverErrorTypes.authorizationError && !forceTokenReauth) {
 			// Re-auth
-			return resolveWrapper<T>(callback, server, clientConfig, true, forToken);
+			return resolveWrapper<T>(callback, server, clientConfig, true);
 		}
 		throw e;
 	}
@@ -156,13 +159,3 @@ export async function getSingleSharePointFile(server: string, drive: string, ite
 		fetchToolClientConfig,
 	);
 }
-
-const fluidFetchWebNavigator = (url: string) => {
-	let message = "Please open browser and navigate to this URL:";
-	if (process.platform === "win32") {
-		child_process.exec(`start "fluid-fetch" /B "${url}"`);
-		message =
-			"Opening browser to get authorization code.  If that doesn't open, please go to this URL manually";
-	}
-	console.log(`${message}\n  ${url}`);
-};