@fluid-tools/fetch-tool 2.1.0-274160 → 2.1.0-276985

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@fluid-tools/fetch-tool",
-  "version": "2.1.0-274160",
+  "version": "2.1.0-276985",
   "description": "Console tool to fetch Fluid data from relay service",
   "homepage": "https://fluidframework.com",
   "repository": {
@@ -15,20 +15,22 @@
     "fluid-fetch": "bin/fluid-fetch"
   },
   "dependencies": {
-    "@fluid-internal/client-utils": "2.1.0-274160",
-    "@fluidframework/container-runtime": "2.1.0-274160",
-    "@fluidframework/core-interfaces": "2.1.0-274160",
-    "@fluidframework/core-utils": "2.1.0-274160",
-    "@fluidframework/datastore": "2.1.0-274160",
-    "@fluidframework/driver-definitions": "2.1.0-274160",
-    "@fluidframework/odsp-doclib-utils": "2.1.0-274160",
-    "@fluidframework/odsp-driver": "2.1.0-274160",
-    "@fluidframework/odsp-driver-definitions": "2.1.0-274160",
-    "@fluidframework/odsp-urlresolver": "2.1.0-274160",
-    "@fluidframework/routerlicious-driver": "2.1.0-274160",
-    "@fluidframework/routerlicious-urlresolver": "2.1.0-274160",
-    "@fluidframework/runtime-definitions": "2.1.0-274160",
-    "@fluidframework/tool-utils": "2.1.0-274160"
+    "@azure/identity": "^4.2.0",
+    "@azure/identity-cache-persistence": "^1.1.0",
+    "@fluid-internal/client-utils": "2.1.0-276985",
+    "@fluidframework/container-runtime": "2.1.0-276985",
+    "@fluidframework/core-interfaces": "2.1.0-276985",
+    "@fluidframework/core-utils": "2.1.0-276985",
+    "@fluidframework/datastore": "2.1.0-276985",
+    "@fluidframework/driver-definitions": "2.1.0-276985",
+    "@fluidframework/odsp-doclib-utils": "2.1.0-276985",
+    "@fluidframework/odsp-driver": "2.1.0-276985",
+    "@fluidframework/odsp-driver-definitions": "2.1.0-276985",
+    "@fluidframework/odsp-urlresolver": "2.1.0-276985",
+    "@fluidframework/routerlicious-driver": "2.1.0-276985",
+    "@fluidframework/routerlicious-urlresolver": "2.1.0-276985",
+    "@fluidframework/runtime-definitions": "2.1.0-276985",
+    "@fluidframework/tool-utils": "2.1.0-276985"
   },
   "devDependencies": {
     "@biomejs/biome": "^1.7.3",

package/src/fluidFetchArgs.ts

@@ -15,15 +15,6 @@ export let paramSnapshotVersionIndex: number | undefined;
 export let paramNumSnapshotVersions = 10;
 export let paramActualFormatting = false;
 
-let paramForceTokenReauth = false;
-
-// Only return true once, to reauth on first call.
-export function getForceTokenReauth() {
-	const result = paramForceTokenReauth;
-	paramForceTokenReauth = false;
-	return result;
-}
-
 export let paramSaveDir: string | undefined;
 export const messageTypeFilter = new Set<string>();
 
@@ -33,12 +24,12 @@ export let paramJWT: string;
 export let connectToWebSocket = false;
 
 export let localDataOnly = false;
+export let loginHint: string | undefined;
 
 const optionsArray = [
 	["--dump:rawmessage", "dump all messages"],
 	["--dump:snapshotVersion", "dump a list of snapshot version"],
 	["--dump:snapshotTree", "dump the snapshot trees"],
-	["--forceTokenReauth", "Force reauthorize token (SPO only)"],
 	["--stat:message", "show message type, channel type, data type statistics"],
 	["--stat:snapshot", "show a table of snapshot path and blob size"],
 	["--stat", "Show both messages & snapshot stats"],
@@ -53,6 +44,7 @@ const optionsArray = [
 	["--snapshotVersionIndex <number>", "Index of the version to dump"],
 	["--websocket", "Connect to web socket to download initial messages"],
 	["--local", "Do not connect to storage, use earlier downloaded data. Requires --saveDir."],
+	["--loginHint", "login hint for the user with document access."],
 ];
 
 function printUsage() {
@@ -120,9 +112,6 @@ export function parseArguments() {
 			case "--jwt":
 				paramJWT = parseStrArg(i++, "jwt token");
 				break;
-			case "--forceTokenReauth":
-				paramForceTokenReauth = true;
-				break;
 			case "--snapshotVersionIndex":
 				paramSnapshotVersionIndex = parseIntArg(i++, "version index", true);
 				break;
@@ -141,6 +130,9 @@ export function parseArguments() {
 			case "--local":
 				localDataOnly = true;
 				break;
+			case "--loginHint":
+				loginHint = parseStrArg(i++, "login hint");
+				break;
 			default:
 				try {
 					const url = new URL(arg);

package/src/fluidFetchInit.ts

@@ -20,10 +20,9 @@ import {
 } from "@fluidframework/odsp-urlresolver/internal";
 import * as r11s from "@fluidframework/routerlicious-driver/internal";
 import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver/internal";
-import { getMicrosoftConfiguration } from "@fluidframework/tool-utils/internal";
 
 import { localDataOnly, paramJWT } from "./fluidFetchArgs.js";
-import { resolveWrapper } from "./fluidFetchSharePoint.js";
+import { resolveWrapper, fetchToolClientConfig } from "./fluidFetchSharePoint.js";
 
 export let latestVersionsId: string = "";
 export let connectionInfo: any;
@@ -66,8 +65,6 @@ async function initializeODSPCore(
 			},
 			server,
 			clientConfig,
-			undefined,
-			true,
 		);
 	};
 	// eslint-disable-next-line @typescript-eslint/promise-function-async
@@ -173,7 +170,7 @@ export async function fluidFetchInit(urlStr: string) {
 		return initializeODSPCore(
 			odspResolvedUrl,
 			new URL(odspResolvedUrl.siteUrl).host,
-			getMicrosoftConfiguration(),
+			fetchToolClientConfig,
 		);
 	} else if (resolvedInfo.serviceType === "r11s") {
 		const url = new URL(urlStr);

package/src/fluidFetchMessages.ts

@@ -96,28 +96,44 @@ async function* loadAllSequencedMessages(
 	let requests = 0;
 	let opsStorage = 0;
 
+	console.log("fetching cached messages");
 	// reading only 1 op to test if there is mismatch
 	const teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);
 
-	let statusCode;
-	let innerMostErrorCode;
-	let response;
-
 	try {
 		await teststream.read();
 	} catch (error: any) {
-		statusCode = error.getTelemetryProperties().statusCode;
-		innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
-		// if there is gap between ops, catch the error and check it is the error we need
+		const statusCode = error.getTelemetryProperties().statusCode;
+		const innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
 		if (statusCode !== 410 || innerMostErrorCode !== "fluidDeltaDataNotAvailable") {
 			throw error;
 		}
-		// get firstAvailableDelta from the error response, and set current sequence number to that
-		response = JSON.parse(error.getTelemetryProperties().response);
-		firstAvailableDelta = response.error.firstAvailableDelta;
-		lastSeq = firstAvailableDelta - 1;
+
+		// This indicates we tried to fetch ops from storage that have been deleted (because they are past some retention policy).
+		// In that case, the error message should indicate the first sequence number that is available.
+		// We make a best-effort attempt for the original query (fetch all ops) by starting from that sequence number.
+		const props = error.getTelemetryProperties();
+		const { responseMessage } = props;
+		const [_, seq] =
+			typeof responseMessage === "string"
+				? responseMessage.match(/GenesisSequenceNumber '(\d+)'/) ?? []
+				: [];
+		if (seq !== undefined) {
+			lastSeq = parseInt(seq, 10);
+			firstAvailableDelta = lastSeq + 1;
+			console.log(
+				`Not all ops are available (older ops may have been deleted from storage). Starting from sequenceNumber: ${firstAvailableDelta}.`,
+			);
+		} else {
+			console.log(props);
+			throw new Error(
+				`Unexpected structure for 410 error: ${error.message}. Further error properties were logged above. This indicates a problem with fetch-tool.`,
+			);
+		}
 	}
 
+	console.log("fetching remaining messages from delta storage");
+
 	// continue reading rest of the ops
 	const stream = deltaStorage.fetchMessages(
 		lastSeq + 1, // inclusive left

package/src/fluidFetchSharePoint.ts

@@ -3,8 +3,8 @@
  * Licensed under the MIT License.
  */
 
-import child_process from "child_process";
-
+import { InteractiveBrowserCredential, useIdentityPlugin } from "@azure/identity";
+import { cachePersistencePlugin } from "@azure/identity-cache-persistence";
 import { DriverErrorTypes } from "@fluidframework/driver-definitions/internal";
 import {
 	IPublicClientConfig,
@@ -13,57 +13,71 @@ import {
 	getChildrenByDriveItem,
 	getDriveItemByServerRelativePath,
 	getDriveItemFromDriveAndItem,
-	getOdspRefreshTokenFn,
+	getAadTenant,
+	getOdspScope,
 } from "@fluidframework/odsp-doclib-utils/internal";
-import {
-	IOdspTokenManagerCacheKey,
-	OdspTokenConfig,
-	OdspTokenManager,
-	getMicrosoftConfiguration,
-	odspTokensCache,
-} from "@fluidframework/tool-utils/internal";
 
-import { getForceTokenReauth } from "./fluidFetchArgs.js";
+import { loginHint } from "./fluidFetchArgs.js";
+
+// Note: the following page may be helpful for debugging auth issues:
+// https://github.com/Azure/azure-sdk-for-js/blob/main/sdk/identity/identity/TROUBLESHOOTING.md
+// See e.g. the section on setting 'AZURE_LOG_LEVEL'.
+useIdentityPlugin(cachePersistencePlugin);
+
+export const fetchToolClientConfig: IPublicClientConfig = {
+	get clientId(): string {
+		const clientId = process.env.fetch__tool__clientId;
+		if (clientId === undefined) {
+			throw new Error(
+				"Client ID environment variable not set: fetch__tool__clientId. Use the getkeys tool to populate it.",
+			);
+		}
+		return clientId;
+	},
+};
 
 export async function resolveWrapper<T>(
 	callback: (authRequestInfo: IOdspAuthRequestInfo) => Promise<T>,
 	server: string,
 	clientConfig: IPublicClientConfig,
 	forceTokenReauth = false,
-	forToken = false,
 ): Promise<T> {
 	try {
-		const odspTokenManager = new OdspTokenManager(odspTokensCache);
-		const tokenConfig: OdspTokenConfig = {
-			type: "browserLogin",
-			navigator: fluidFetchWebNavigator,
-		};
-		const tokens = await odspTokenManager.getOdspTokens(
-			server,
-			clientConfig,
-			tokenConfig,
-			undefined /* forceRefresh */,
-			forceTokenReauth || getForceTokenReauth(),
-		);
+		const credential = new InteractiveBrowserCredential({
+			clientId: fetchToolClientConfig.clientId,
+			tenantId: getAadTenant(server),
+			// NOTE: fetch-tool flows using multiple sets of user credentials haven't been well-tested.
+			// Some of the @azure/identity docs suggest we may need to manage authentication records and choose
+			// which one to use explicitly here if we have such scenarios.
+			// If we start doing this, it may be worth considering using disableAutomaticAuthentication here so we
+			// have better control over when interactive auth may be triggered.
+			// For now, fetch-tool doesn't work against personal accounts anyway so the only flow that might necessitate this
+			// would be grabbing documents using several identities (e.g. test accounts we use for stress testing).
+			// In that case, a simple workaround is to delete the cache that @azure/identity uses before running the tool.
+			// See docs on `tokenCachePersistenceOptions.name` for information on where this cache is stored.
+			loginHint,
+			tokenCachePersistenceOptions: {
+				enabled: true,
+				name: "fetch-tool",
+			},
+		});
+
+		const scope = getOdspScope(server);
+
+		const { token } = await credential.getToken(scope);
 
-		const result = await callback({
-			accessToken: tokens.accessToken,
-			refreshTokenFn: getOdspRefreshTokenFn(server, clientConfig, tokens),
+		return await callback({
+			accessToken: token,
+			refreshTokenFn: async () => {
+				await credential.authenticate(scope);
+				const result = await credential.getToken(scope);
+				return result.token;
+			},
 		});
-		// If this is used for getting a token, then refresh the cache with new token.
-		if (forToken) {
-			const key: IOdspTokenManagerCacheKey = { isPush: false, userOrServer: server };
-			await odspTokenManager.updateTokensCache(key, {
-				accessToken: result as any as string,
-				refreshToken: tokens.refreshToken,
-			});
-			return result;
-		}
-		return result;
 	} catch (e: any) {
 		if (e.errorType === DriverErrorTypes.authorizationError && !forceTokenReauth) {
 			// Re-auth
-			return resolveWrapper<T>(callback, server, clientConfig, true, forToken);
+			return resolveWrapper<T>(callback, server, clientConfig, true);
 		}
 		throw e;
 	}
@@ -101,12 +115,10 @@ export async function getSharepointFiles(
 	serverRelativePath: string,
 	recurse: boolean,
 ) {
-	const clientConfig = getMicrosoftConfiguration();
-
 	const fileInfo = await resolveDriveItemByServerRelativePath(
 		server,
 		serverRelativePath,
-		clientConfig,
+		fetchToolClientConfig,
 	);
 	console.log(fileInfo);
 	const pendingFolder: { path: string; folder: IOdspDriveItem }[] = [];
@@ -124,7 +136,7 @@ export async function getSharepointFiles(
 			break;
 		}
 		const { path, folder } = folderInfo;
-		const children = await resolveChildrenByDriveItem(server, folder, clientConfig);
+		const children = await resolveChildrenByDriveItem(server, folder, fetchToolClientConfig);
 		for (const child of children) {
 			const childPath = `${path}/${child.name}`;
 			if (child.isFolder) {
@@ -140,22 +152,10 @@ export async function getSharepointFiles(
 }
 
 export async function getSingleSharePointFile(server: string, drive: string, item: string) {
-	const clientConfig = getMicrosoftConfiguration();
-
 	return resolveWrapper<IOdspDriveItem>(
 		// eslint-disable-next-line @typescript-eslint/promise-function-async
 		(authRequestInfo) => getDriveItemFromDriveAndItem(server, drive, item, authRequestInfo),
 		server,
-		clientConfig,
+		fetchToolClientConfig,
 	);
 }
-
-const fluidFetchWebNavigator = (url: string) => {
-	let message = "Please open browser and navigate to this URL:";
-	if (process.platform === "win32") {
-		child_process.exec(`start "fluid-fetch" /B "${url}"`);
-		message =
-			"Opening browser to get authorization code.  If that doesn't open, please go to this URL manually";
-	}
-	console.log(`${message}\n  ${url}`);
-};