@fluid-tools/fetch-tool 2.74.0 → 2.81.0

package/src/fluidAnalyzeMessages.ts

@@ -23,7 +23,7 @@ import type { IAttachMessage, IEnvelope } from "@fluidframework/runtime-definiti
 const noClientName = "No Client";
 const objectTypePrefix = "https://graph.microsoft.com/types/";
 
-function incr(map: Map<string, [number, number]>, key: string, size: number, count = 1) {
+function incr(map: Map<string, [number, number]>, key: string, size: number, count = 1): void {
 	const value = map.get(key);
 	if (value === undefined) {
 		map.set(key, [count, size]);
@@ -50,7 +50,7 @@ interface IMessageAnalyzer {
  * Helper class to track session statistics
  */
 class ActiveSession {
-	public static create(email: string, message: ISequencedDocumentMessage) {
+	public static create(email: string, message: ISequencedDocumentMessage): ActiveSession {
 		return new ActiveSession(email, message);
 	}
 
@@ -61,7 +61,7 @@ class ActiveSession {
 		private readonly startMessage: ISequencedDocumentMessage,
 	) {}
 
-	public reportOp(timestamp: number) {
+	public reportOp(timestamp: number): void {
 		this.opCount++;
 	}
 
@@ -90,7 +90,7 @@ function dumpStats(
 		removeTotals?: boolean;
 		reverseSort?: boolean;
 	},
-) {
+): void {
 	const fieldSizes = [10, 14];
 	const nameLength = 72;
 	const fieldsLength = fieldSizes[0] + fieldSizes[1] + 1;
@@ -174,7 +174,7 @@ function dumpStats(
 	}
 }
 
-const getObjectId = (dataStoreId: string, id: string) => `[${dataStoreId}]/${id}`;
+const getObjectId = (dataStoreId: string, id: string): string => `[${dataStoreId}]/${id}`;
 
 /**
  * Analyzer for sessions
@@ -477,11 +477,11 @@ class MessageDumper implements IMessageAnalyzer {
 }
 
 export async function printMessageStats(
-	generator, // AsyncGenerator<ISequencedDocumentMessage[]>,
+	generator: AsyncGenerator<ISequencedDocumentMessage[]>,
 	dumpMessageStats: boolean,
 	dumpMessages: boolean,
 	messageTypeFilter: Set<string> = new Set<string>(),
-) {
+): Promise<void> {
 	let lastMessage: ISequencedDocumentMessage | undefined;
 
 	const analyzers: IMessageAnalyzer[] = [
@@ -498,7 +498,7 @@ export async function printMessageStats(
 	}
 
 	for await (const messages of generator) {
-		for (const message of messages as ISequencedDocumentMessage[]) {
+		for (const message of messages) {
 			const msgSize = JSON.stringify(message).length;
 			lastMessage = message;
 
@@ -531,7 +531,7 @@ function processOp(
 	dataTypeStats: Map<string, [number, number]>,
 	messageTypeStats: Map<string, [number, number]>,
 	chunkMap: Map<string, { chunks: string[]; totalSize: number }>,
-) {
+): void {
 	let type = runtimeMessage.type;
 	let recorded = false;
 	let totalMsgSize = msgSize;
@@ -675,7 +675,7 @@ function processOp(
 function processDataStoreAttachOp(
 	attachMessage: IAttachMessage | string,
 	dataType: Map<string, string>,
-) {
+): void {
 	// dataType.set(getObjectId(attachMessage.id), attachMessage.type);
 
 	// That's data store, and it brings a bunch of data structures.
@@ -703,7 +703,7 @@ function reportOpenSessions(
 	sessionsInProgress: Map<string, ActiveSession>,
 	sessions: Map<string, [number, number]>,
 	users: Map<string, [number, number]>,
-) {
+): void {
 	const activeSessions = new Map<string, [number, number]>();
 
 	for (const [clientId, ses] of sessionsInProgress) {
@@ -739,10 +739,11 @@ function reportOpenSessions(
 function calcChannelStats(
 	dataType: Map<string, string>,
 	objectStats: Map<string, [number, number]>,
-) {
+): Map<string, [number, number]> {
 	const channelStats = new Map<string, [number, number]>();
 	for (const [objectId, type] of dataType) {
 		let value = objectStats.get(objectId);
+		// eslint-disable-next-line @typescript-eslint/prefer-nullish-coalescing -- using ??= could change behavior if value is falsy
 		if (value === undefined) {
 			value = [0, 0];
 		}
@@ -761,7 +762,7 @@ function processQuorumMessages(
 	sessionsInProgress: Map<string, ActiveSession>,
 	sessions: Map<string, [number, number]>,
 	users: Map<string, [number, number]>,
-) {
+): ActiveSession | undefined {
 	let session: ActiveSession | undefined;
 	const dataString = (message as any).data;
 	if (message.type === "join") {

package/src/fluidFetch.ts

@@ -17,7 +17,7 @@ import { fluidFetchMessages } from "./fluidFetchMessages.js";
 import { getSharepointFiles, getSingleSharePointFile } from "./fluidFetchSharePoint.js";
 import { fluidFetchSnapshot } from "./fluidFetchSnapshot.js";
 
-async function fluidFetchOneFile(urlStr: string, name?: string) {
+async function fluidFetchOneFile(urlStr: string, name?: string): Promise<void> {
 	const documentService = await fluidFetchInit(urlStr);
 	const saveDir =
 		paramSaveDir !== undefined
@@ -41,7 +41,10 @@ async function fluidFetchOneFile(urlStr: string, name?: string) {
 	await fluidFetchMessages(documentService, saveDir);
 }
 
-async function tryFluidFetchOneSharePointFile(server: string, driveItem: IOdspDriveItem) {
+async function tryFluidFetchOneSharePointFile(
+	server: string,
+	driveItem: IOdspDriveItem,
+): Promise<void> {
 	const { path, name, driveId, itemId } = driveItem;
 	console.log(`File: ${path}/${name}`);
 	await fluidFetchOneFile(
@@ -61,7 +64,7 @@ function getSharePointSpecificDriveItem(
 	}
 }
 
-function getSharepointServerRelativePathFromURL(url: URL) {
+function getSharepointServerRelativePathFromURL(url: URL): string | undefined {
 	if (url.pathname.startsWith("/_api/v2.1/drives/")) {
 		return undefined;
 	}
@@ -83,7 +86,7 @@ function getSharepointServerRelativePathFromURL(url: URL) {
 	return decodeURI(sitePath);
 }
 
-async function fluidFetchMain() {
+async function fluidFetchMain(): Promise<void> {
 	if (paramURL === undefined) {
 		return;
 	}
@@ -128,7 +131,9 @@ fluidFetchMain()
 					if (key !== "message" && key !== "stack") {
 						extraMsg += `\n${key}: ${JSON.stringify(error[key], undefined, 2)}`;
 					}
-				} catch (_) {}
+				} catch (_) {
+					// TODO: document why we are ignoring the error here
+				}
 			}
 			console.error(`ERROR: ${error.stack}${extraMsg}`);
 		} else if (typeof error === "object") {

package/src/fluidFetchArgs.ts

@@ -47,7 +47,7 @@ const optionsArray = [
 	["--loginHint", "login hint for the user with document access."],
 ];
 
-function printUsage() {
+function printUsage(): void {
 	console.log("Usage: fluid-fetch [options] URL");
 	console.log("URL: <ODSP URL>|<Routerlicious URL>");
 	console.log("Options:");
@@ -66,7 +66,7 @@ export function setArguments(values: {
 	dumpSnapshotStats?: boolean;
 	dumpSnapshotTrees?: boolean;
 	overWrite?: boolean;
-}) {
+}): void {
 	paramSaveDir = values.saveDir;
 	paramURL = values.paramURL;
 	dumpMessages = values.dumpMessages ?? dumpMessages;
@@ -76,7 +76,7 @@ export function setArguments(values: {
 	overWrite = values.overWrite ?? overWrite;
 }
 
-export function parseArguments() {
+export function parseArguments(): void {
 	for (let i = 2; i < process.argv.length; i++) {
 		const arg = process.argv[i];
 		switch (arg) {
@@ -157,7 +157,7 @@ export function parseArguments() {
 	checkArgs();
 }
 
-function parseStrArg(i: number, name: string) {
+function parseStrArg(i: number, name: string): string {
 	if (i + 1 >= process.argv.length) {
 		console.error(`ERROR: Missing ${name}`);
 		printUsage();
@@ -165,7 +165,7 @@ function parseStrArg(i: number, name: string) {
 	}
 	return process.argv[i + 1];
 }
-function parseIntArg(i: number, name: string, allowZero: boolean) {
+function parseIntArg(i: number, name: string, allowZero: boolean): number {
 	if (i + 1 >= process.argv.length) {
 		console.error(`ERROR: Missing ${name}`);
 		printUsage();
@@ -181,7 +181,7 @@ function parseIntArg(i: number, name: string, allowZero: boolean) {
 	return paramNumber;
 }
 
-function checkArgs() {
+function checkArgs(): void {
 	if (paramSnapshotVersionIndex !== undefined) {
 		paramNumSnapshotVersions = Math.max(
 			paramSnapshotVersionIndex + 1,

package/src/fluidFetchInit.ts

@@ -4,7 +4,10 @@
  */
 
 import type { IRequest } from "@fluidframework/core-interfaces";
-import type { IResolvedUrl } from "@fluidframework/driver-definitions/internal";
+import type {
+	IDocumentService,
+	IResolvedUrl,
+} from "@fluidframework/driver-definitions/internal";
 import type {
 	IPublicClientConfig,
 	IOdspAuthRequestInfo,
@@ -31,7 +34,7 @@ async function initializeODSPCore(
 	odspResolvedUrl: IOdspResolvedUrl,
 	server: string,
 	clientConfig: IPublicClientConfig,
-) {
+): Promise<IDocumentService | undefined> {
 	const { driveId, itemId } = odspResolvedUrl;
 
 	connectionInfo = {
@@ -52,7 +55,9 @@ async function initializeODSPCore(
   item:   ${itemId}
   docId:  ${docId}`);
 
-	const getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {
+	const getStorageTokenStub = async (
+		options: OdspResourceTokenFetchOptions,
+	): Promise<string> => {
 		return resolveWrapper(
 			async (authRequestInfo: IOdspAuthRequestInfo) => {
 				if (
@@ -68,7 +73,7 @@ async function initializeODSPCore(
 		);
 	};
 	// eslint-disable-next-line @typescript-eslint/promise-function-async
-	const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) =>
+	const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions): Promise<string> =>
 		Promise.resolve("");
 	const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(
 		getStorageTokenStub,
@@ -86,7 +91,7 @@ async function initializeR11s(
 	server: string,
 	pathname: string,
 	r11sResolvedUrl: IResolvedUrl,
-) {
+): Promise<IDocumentService | undefined> {
 	const path = pathname.split("/");
 	let tenantId: string;
 	let documentId: string;
@@ -108,7 +113,7 @@ async function initializeR11s(
 	};
 
 	if (localDataOnly) {
-		return;
+		return undefined;
 	}
 
 	console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
@@ -159,7 +164,7 @@ async function resolveUrl(url: string): Promise<IResolvedInfo | undefined> {
 	return undefined;
 }
 
-export async function fluidFetchInit(urlStr: string) {
+export async function fluidFetchInit(urlStr: string): Promise<IDocumentService | undefined> {
 	const resolvedInfo = await resolveUrl(urlStr);
 	if (resolvedInfo === undefined) {
 		throw new Error(`Unknown URL ${urlStr}`);

package/src/fluidFetchMessages.ts

@@ -33,7 +33,7 @@ async function* loadAllSequencedMessages(
 	documentService?: IDocumentService,
 	dir?: string,
 	files?: string[],
-) {
+): AsyncGenerator<ISequencedDocumentMessage[]> {
 	let lastSeq = 0;
 	// flag for mismatch between last sequence number read and new one to be read
 	let seqNumMismatch = false;
@@ -210,10 +210,10 @@ async function* loadAllSequencedMessages(
 }
 
 async function* saveOps(
-	gen, // AsyncGenerator<ISequencedDocumentMessage[]>,
+	gen: AsyncGenerator<ISequencedDocumentMessage[]>,
 	dir: string,
 	files: string[],
-) {
+): AsyncGenerator<ISequencedDocumentMessage[]> {
 	// Split into 100K ops
 	const chunk = 100 * 1000;
 
@@ -287,7 +287,7 @@ async function* saveOps(
 export async function fluidFetchMessages(
 	documentService?: IDocumentService,
 	saveDir?: string,
-) {
+): Promise<void> {
 	const messageStats = dumpMessageStats || dumpMessages;
 	if (!messageStats && (saveDir === undefined || documentService === undefined)) {
 		return;
@@ -316,6 +316,7 @@ export async function fluidFetchMessages(
 		return printMessageStats(generator, dumpMessageStats, dumpMessages, messageTypeFilter);
 	} else {
 		let item;
+		// eslint-disable-next-line no-empty -- TODO: Investigate / document what is going on here.
 		for await (item of generator) {
 		}
 	}

package/src/fluidFetchSharePoint.ts

@@ -113,7 +113,7 @@ async function resolveDriveItemByServerRelativePath(
 	server: string,
 	serverRelativePath: string,
 	clientConfig: IPublicClientConfig,
-) {
+): Promise<IOdspDriveItem> {
 	return resolveWrapper<IOdspDriveItem>(
 		// eslint-disable-next-line @typescript-eslint/promise-function-async
 		(authRequestInfo) =>
@@ -127,7 +127,7 @@ async function resolveChildrenByDriveItem(
 	server: string,
 	folderDriveItem: IOdspDriveItem,
 	clientConfig: IPublicClientConfig,
-) {
+): Promise<IOdspDriveItem[]> {
 	return resolveWrapper<IOdspDriveItem[]>(
 		// eslint-disable-next-line @typescript-eslint/promise-function-async
 		(authRequestInfo) => getChildrenByDriveItem(folderDriveItem, server, authRequestInfo),
@@ -140,7 +140,7 @@ export async function getSharepointFiles(
 	server: string,
 	serverRelativePath: string,
 	recurse: boolean,
-) {
+): Promise<IOdspDriveItem[]> {
 	const fileInfo = await resolveDriveItemByServerRelativePath(
 		server,
 		serverRelativePath,
@@ -177,7 +177,11 @@ export async function getSharepointFiles(
 	return files;
 }
 
-export async function getSingleSharePointFile(server: string, drive: string, item: string) {
+export async function getSingleSharePointFile(
+	server: string,
+	drive: string,
+	item: string,
+): Promise<IOdspDriveItem> {
 	return resolveWrapper<IOdspDriveItem>(
 		// eslint-disable-next-line @typescript-eslint/promise-function-async
 		(authRequestInfo) => getDriveItemFromDriveAndItem(server, drive, item, authRequestInfo),

package/src/fluidFetchSnapshot.ts

@@ -66,7 +66,7 @@ function fetchBlobs(
 	tree: ISnapshotTree,
 	storage: IDocumentStorageService,
 	blobIdMap: Map<string, number>,
-) {
+): IFetchedBlob[] {
 	const result: IFetchedBlob[] = [];
 	for (const [item, blobId] of Object.entries(tree.blobs)) {
 		const treePath = `${prefix}${item}`;
@@ -150,12 +150,15 @@ async function fetchBlobsFromSnapshotTree(
 	return result;
 }
 
-function getDumpFetchedData(fetchedData: IFetchedData[]) {
+function getDumpFetchedData(fetchedData: IFetchedData[]): IFetchedData[] {
 	const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));
 	return sorted.filter((item) => !isFetchedTree(item) || !item.patched);
 }
 
-async function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {
+async function dumpSnapshotTreeVerbose(
+	name: string,
+	fetchedData: IFetchedData[],
+): Promise<void> {
 	let size = 0;
 	const sorted = getDumpFetchedData(fetchedData);
 
@@ -214,7 +217,11 @@ async function dumpSnapshotTree(
 	return { blobCountNew, blobCount: sorted.length, size, sizeNew };
 }
 
-async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {
+async function saveSnapshot(
+	name: string,
+	fetchedData: IFetchedData[],
+	saveDir: string,
+): Promise<void> {
 	const outDir = `${saveDir}/${name}/`;
 	const mkdir = util.promisify(fs.mkdir);
 
@@ -238,7 +245,9 @@ async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir:
 					if (!paramActualFormatting) {
 						decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);
 					}
-				} catch (e) {}
+				} catch (e) {
+					// TODO: document why we are ignoring the error here
+				}
 				fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`, decoded);
 			} else {
 				// Write out same data for tree decoded or not, except for formatting
@@ -255,7 +264,10 @@ async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir:
 	);
 }
 
-async function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {
+async function fetchBlobsFromVersion(
+	storage: IDocumentStorageService,
+	version: IVersion,
+): Promise<IFetchedData[]> {
 	const tree = await reportErrors(
 		`getSnapshotTree ${version.id}`,
 		storage.getSnapshotTree(version),
@@ -266,7 +278,7 @@ async function fetchBlobsFromVersion(storage: IDocumentStorageService, version:
 	return fetchBlobsFromSnapshotTree(storage, tree);
 }
 
-async function reportErrors<T>(message: string, res: Promise<T>) {
+async function reportErrors<T>(message: string, res: Promise<T>): Promise<T> {
 	try {
 		return await res;
 	} catch (error) {
@@ -278,7 +290,7 @@ async function reportErrors<T>(message: string, res: Promise<T>) {
 export async function fluidFetchSnapshot(
 	documentService?: IDocumentService,
 	saveDir?: string,
-) {
+): Promise<void> {
 	if (
 		!dumpSnapshotStats &&
 		!dumpSnapshotTrees &&
@@ -373,6 +385,7 @@ export async function fluidFetchSnapshot(
 		if (version === undefined) {
 			console.log("No snapshot tree");
 		} else {
+			// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions, @typescript-eslint/prefer-nullish-coalescing -- intentional behavior
 			if (blobsToDump === undefined) {
 				blobsToDump = await fetchBlobsFromVersion(storage, version);
 			}

package/.eslintrc.cjs

@@ -1,27 +0,0 @@
-/*!
- * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
- * Licensed under the MIT License.
- */
-
-module.exports = {
-	extends: [
-		require.resolve("@fluidframework/eslint-config-fluid/minimal-deprecated"),
-		"prettier",
-	],
-	rules: {
-		// This library is used in the browser, so we don't want dependencies on most node libraries.
-		"import-x/no-nodejs-modules": ["error", { allow: ["child_process", "fs", "util"] }],
-
-		// #region TODO: remove these once this config has been updated to use our "recommended" base instead of our deprecated minimal one.
-		"@typescript-eslint/consistent-type-exports": [
-			"error",
-			{ fixMixedExportsWithInlineTypeSpecifier: true },
-		],
-		"@typescript-eslint/consistent-type-imports": [
-			"error",
-			{ fixStyle: "inline-type-imports" },
-		],
-		"@typescript-eslint/no-import-type-side-effects": "error",
-		// #endregion
-	},
-};