@fluid-tools/fetch-tool 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

package/src/fluidFetch.ts

@@ -13,115 +13,120 @@ import { getSharepointFiles, getSingleSharePointFile } from "./fluidFetchSharePo
 import { fluidFetchSnapshot } from "./fluidFetchSnapshot";
 
 async function fluidFetchOneFile(urlStr: string, name?: string) {
-    const documentService = await fluidFetchInit(urlStr);
-    const saveDir = paramSaveDir !== undefined
-        ? (name !== undefined
-            ? `${paramSaveDir}/${name}`
-            : paramSaveDir)
-        : undefined;
-    if (saveDir !== undefined) {
-        const mkdir = util.promisify(fs.mkdir);
-        const writeFile = util.promisify(fs.writeFile);
-        await mkdir(saveDir, { recursive: true });
-        const info = {
-            creationDate: new Date().toString(),
-            connectionInfo,
-            url: urlStr,
-        };
-        await writeFile(`${saveDir}/info.json`, JSON.stringify(info, undefined, 2));
-    }
+	const documentService = await fluidFetchInit(urlStr);
+	const saveDir =
+		paramSaveDir !== undefined
+			? name !== undefined
+				? `${paramSaveDir}/${name}`
+				: paramSaveDir
+			: undefined;
+	if (saveDir !== undefined) {
+		const mkdir = util.promisify(fs.mkdir);
+		const writeFile = util.promisify(fs.writeFile);
+		await mkdir(saveDir, { recursive: true });
+		const info = {
+			creationDate: new Date().toString(),
+			connectionInfo,
+			url: urlStr,
+		};
+		await writeFile(`${saveDir}/info.json`, JSON.stringify(info, undefined, 2));
+	}
 
-    await fluidFetchSnapshot(documentService, saveDir);
-    await fluidFetchMessages(documentService, saveDir);
+	await fluidFetchSnapshot(documentService, saveDir);
+	await fluidFetchMessages(documentService, saveDir);
 }
 
 async function tryFluidFetchOneSharePointFile(server: string, driveItem: IOdspDriveItem) {
-    const { path, name, driveId, itemId } = driveItem;
-    console.log(`File: ${path}/${name}`);
-    await fluidFetchOneFile(`https://${server}/_api/v2.1/drives/${driveId}/items/${itemId}`, name);
+	const { path, name, driveId, itemId } = driveItem;
+	console.log(`File: ${path}/${name}`);
+	await fluidFetchOneFile(`https://${server}/_api/v2.1/drives/${driveId}/items/${itemId}`, name);
 }
 
-function getSharePointSpecificDriveItem(url: URL): { driveId: string; itemId: string; } | undefined {
-    if (url.searchParams.has("driveId") && url.searchParams.has("itemId")) {
-        return {
-            driveId: url.searchParams.get("driveId") as string,
-            itemId: url.searchParams.get("itemId") as string,
-        };
-    }
+function getSharePointSpecificDriveItem(url: URL): { driveId: string; itemId: string } | undefined {
+	if (url.searchParams.has("driveId") && url.searchParams.has("itemId")) {
+		return {
+			driveId: url.searchParams.get("driveId") as string,
+			itemId: url.searchParams.get("itemId") as string,
+		};
+	}
 }
 
 function getSharepointServerRelativePathFromURL(url: URL) {
-    if (url.pathname.startsWith("/_api/v2.1/drives/")) {
-        return undefined;
-    }
+	if (url.pathname.startsWith("/_api/v2.1/drives/")) {
+		return undefined;
+	}
 
-    const hostnameParts = url.hostname.split(".");
-    const suffix = hostnameParts[0].endsWith("-my") ? "/_layouts/15/onedrive.aspx" : "/forms/allitems.aspx";
+	const hostnameParts = url.hostname.split(".");
+	const suffix = hostnameParts[0].endsWith("-my")
+		? "/_layouts/15/onedrive.aspx"
+		: "/forms/allitems.aspx";
 
-    let sitePath = url.pathname;
-    if (url.searchParams.has("id")) {
-        sitePath = url.searchParams.get("id") as string;
-    } else if (url.searchParams.has("RootFolder")) {
-        sitePath = url.searchParams.get("RootFolder") as string;
-    } else if (url.pathname.toLowerCase().endsWith(suffix)) {
-        sitePath = sitePath.substr(0, url.pathname.length - suffix.length);
-    }
+	let sitePath = url.pathname;
+	if (url.searchParams.has("id")) {
+		sitePath = url.searchParams.get("id") as string;
+	} else if (url.searchParams.has("RootFolder")) {
+		sitePath = url.searchParams.get("RootFolder") as string;
+	} else if (url.pathname.toLowerCase().endsWith(suffix)) {
+		sitePath = sitePath.substr(0, url.pathname.length - suffix.length);
+	}
 
-    return decodeURI(sitePath);
+	return decodeURI(sitePath);
 }
 
 async function fluidFetchMain() {
-    if (paramURL === undefined) {
-        return;
-    }
+	if (paramURL === undefined) {
+		return;
+	}
 
-    const url = new URL(paramURL);
-    const server = url.hostname;
-    if (isOdspHostname(server)) {
-        // See if the url already has the specific item
-        const driveItem = getSharePointSpecificDriveItem(url);
-        if (driveItem) {
-            const file = await getSingleSharePointFile(server, driveItem.driveId, driveItem.itemId);
-            await tryFluidFetchOneSharePointFile(server, file);
-            return;
-        }
+	const url = new URL(paramURL);
+	const server = url.hostname;
+	if (isOdspHostname(server)) {
+		// See if the url already has the specific item
+		const driveItem = getSharePointSpecificDriveItem(url);
+		if (driveItem) {
+			const file = await getSingleSharePointFile(server, driveItem.driveId, driveItem.itemId);
+			await tryFluidFetchOneSharePointFile(server, file);
+			return;
+		}
 
-        // See if the url given represent a sharepoint directory
-        const serverRelativePath = getSharepointServerRelativePathFromURL(url);
-        if (serverRelativePath !== undefined) {
-            const files = await getSharepointFiles(server, serverRelativePath, false);
-            for (const file of files) {
-                if (file.name.endsWith(".b") || file.name.endsWith(".fluid")) {
-                    await tryFluidFetchOneSharePointFile(server, file);
-                }
-            }
-            return;
-        }
-    }
+		// See if the url given represent a sharepoint directory
+		const serverRelativePath = getSharepointServerRelativePathFromURL(url);
+		if (serverRelativePath !== undefined) {
+			const files = await getSharepointFiles(server, serverRelativePath, false);
+			for (const file of files) {
+				if (file.name.endsWith(".b") || file.name.endsWith(".fluid")) {
+					await tryFluidFetchOneSharePointFile(server, file);
+				}
+			}
+			return;
+		}
+	}
 
-    return fluidFetchOneFile(paramURL);
+	return fluidFetchOneFile(paramURL);
 }
 
 parseArguments();
 
 // eslint-disable-next-line @typescript-eslint/no-floating-promises
 fluidFetchMain()
-    .catch((error: Error) => {
-        if (error instanceof Error) {
-            let extraMsg = "";
-            for (const key of Object.keys(error)) {
-                // error[key] might have circular structure
-                try {
-                    if (key !== "message" && key !== "stack") {
-                        extraMsg += `\n${key}: ${JSON.stringify(error[key], undefined, 2)}`;
-                    }
-                } catch (_) { }
-            }
-            console.error(`ERROR: ${error.stack}${extraMsg}`);
-        } else if (typeof error === "object") {
-            console.error(`ERROR: Unknown exception object\n${JSON.stringify(error, undefined, 2)}`);
-        } else {
-            console.error(`ERROR: ${error}`);
-        }
-    })
-    .then(() => process.exit(0));
+	.catch((error: Error) => {
+		if (error instanceof Error) {
+			let extraMsg = "";
+			for (const key of Object.keys(error)) {
+				// error[key] might have circular structure
+				try {
+					if (key !== "message" && key !== "stack") {
+						extraMsg += `\n${key}: ${JSON.stringify(error[key], undefined, 2)}`;
+					}
+				} catch (_) {}
+			}
+			console.error(`ERROR: ${error.stack}${extraMsg}`);
+		} else if (typeof error === "object") {
+			console.error(
+				`ERROR: Unknown exception object\n${JSON.stringify(error, undefined, 2)}`,
+			);
+		} else {
+			console.error(`ERROR: ${error}`);
+		}
+	})
+	.then(() => process.exit(0));

package/src/fluidFetchArgs.ts

@@ -14,16 +14,15 @@ export let dumpSnapshotVersions = false;
 export let overWrite = false;
 export let paramSnapshotVersionIndex: number | undefined;
 export let paramNumSnapshotVersions = 10;
-export let paramUnpackAggregatedBlobs = true;
 export let paramActualFormatting = false;
 
 let paramForceTokenReauth = false;
 
 // Only return true once, to reauth on first call.
 export function getForceTokenReauth() {
-    const result = paramForceTokenReauth;
-    paramForceTokenReauth = false;
-    return result;
+	const result = paramForceTokenReauth;
+	paramForceTokenReauth = false;
+	return result;
 }
 
 export let paramSaveDir: string | undefined;
@@ -36,184 +35,184 @@ export let connectToWebSocket = false;
 
 export let localDataOnly = false;
 
-export let paramSite: string | undefined;
-
-const optionsArray =
-    [
-        ["--dump:rawmessage", "dump all messages"],
-        ["--dump:snapshotVersion", "dump a list of snapshot version"],
-        ["--dump:snapshotTree", "dump the snapshot trees"],
-        ["--forceTokenReauth", "Force reauthorize token (SPO only)"],
-        ["--stat:message", "show message type, channel type, data type statistics"],
-        ["--stat:snapshot", "show a table of snapshot path and blob size"],
-        ["--stat", "Show both messages & snapshot stats"],
-        ["--filter:messageType <type>", "filter message by <type>"],
-        ["--jwt <token>", "token to be used for routerlicious URLs"],
-        ["--numSnapshotVersions <number>", "Number of versions to load (default:10)"],
-        ["--noUnpack", "Do not unpack aggregated blobs"],
-        ["--actualPayload", "Do not format json payloads nicely, preserve actual bytes / formatting in storage"],
-        ["--saveDir <outdir>", "Save data of the snapshots and messages"],
-        ["--snapshotVersionIndex <number>", "Index of the version to dump"],
-        ["--websocket", "Connect to web socket to download initial messages"],
-        ["--local", "Do not connect to storage, use earlier downloaded data. Requires --saveDir."],
-    ];
-
-export function printUsage() {
-    console.log("Usage: fluid-fetch [options] URL");
-    console.log("URL: <ODSP URL>|<Routerlicious URL>");
-    console.log("Options:");
-    for (const i of optionsArray) {
-        console.log(`  ${i[0].padEnd(32)}: ${i[1]}`);
-    }
+const optionsArray = [
+	["--dump:rawmessage", "dump all messages"],
+	["--dump:snapshotVersion", "dump a list of snapshot version"],
+	["--dump:snapshotTree", "dump the snapshot trees"],
+	["--forceTokenReauth", "Force reauthorize token (SPO only)"],
+	["--stat:message", "show message type, channel type, data type statistics"],
+	["--stat:snapshot", "show a table of snapshot path and blob size"],
+	["--stat", "Show both messages & snapshot stats"],
+	["--filter:messageType <type>", "filter message by <type>"],
+	["--jwt <token>", "token to be used for routerlicious URLs"],
+	["--numSnapshotVersions <number>", "Number of versions to load (default:10)"],
+	[
+		"--actualPayload",
+		"Do not format json payloads nicely, preserve actual bytes / formatting in storage",
+	],
+	["--saveDir <outdir>", "Save data of the snapshots and messages"],
+	["--snapshotVersionIndex <number>", "Index of the version to dump"],
+	["--websocket", "Connect to web socket to download initial messages"],
+	["--local", "Do not connect to storage, use earlier downloaded data. Requires --saveDir."],
+];
+
+function printUsage() {
+	console.log("Usage: fluid-fetch [options] URL");
+	console.log("URL: <ODSP URL>|<Routerlicious URL>");
+	console.log("Options:");
+	for (const i of optionsArray) {
+		console.log(`  ${i[0].padEnd(32)}: ${i[1]}`);
+	}
 }
 
 // Can be used in unit test to pass in customized argument values
 // More argument options can be added when needed
 export function setArguments(values: {
-    saveDir: string;
-    paramURL: string;
-    dumpMessages?: boolean;
-    dumpMessageStats?: boolean;
-    dumpSnapshotStats?: boolean;
-    dumpSnapshotTrees?: boolean;
-    overWrite?: boolean; }) {
-    paramSaveDir = values.saveDir;
-    paramURL = values.paramURL;
-    dumpMessages = values.dumpMessages ?? dumpMessages;
-    dumpMessageStats = values.dumpMessageStats ?? dumpMessageStats;
-    dumpSnapshotStats = values.dumpSnapshotStats ?? dumpSnapshotStats;
-    dumpSnapshotTrees = values.dumpSnapshotTrees ?? dumpSnapshotTrees;
-    overWrite = values.overWrite ?? overWrite;
+	saveDir: string;
+	paramURL: string;
+	dumpMessages?: boolean;
+	dumpMessageStats?: boolean;
+	dumpSnapshotStats?: boolean;
+	dumpSnapshotTrees?: boolean;
+	overWrite?: boolean;
+}) {
+	paramSaveDir = values.saveDir;
+	paramURL = values.paramURL;
+	dumpMessages = values.dumpMessages ?? dumpMessages;
+	dumpMessageStats = values.dumpMessageStats ?? dumpMessageStats;
+	dumpSnapshotStats = values.dumpSnapshotStats ?? dumpSnapshotStats;
+	dumpSnapshotTrees = values.dumpSnapshotTrees ?? dumpSnapshotTrees;
+	overWrite = values.overWrite ?? overWrite;
 }
 
 export function parseArguments() {
-    for (let i = 2; i < process.argv.length; i++) {
-        const arg = process.argv[i];
-        switch (arg) {
-            case "--dump:rawmessage":
-                dumpMessages = true;
-                break;
-            case "--dump:rawmessage:overwrite":
-                dumpMessages = true;
-                overWrite = true;
-                break;
-            case "--stat:message":
-                dumpMessageStats = true;
-                break;
-            case "--stat":
-                dumpMessageStats = true;
-                dumpSnapshotStats = true;
-                break;
-            case "--filter:messageType":
-                messageTypeFilter.add(parseStrArg(i++, "type name for messageType filter"));
-                break;
-            case "--stat:snapshot":
-                dumpSnapshotStats = true;
-                break;
-            case "--dump:snapshotVersion":
-                dumpSnapshotVersions = true;
-                break;
-            case "--dump:snapshotTree":
-                dumpSnapshotTrees = true;
-                break;
-            case "--help":
-                printUsage();
-                process.exit(0);
-            case "--jwt":
-                paramJWT = parseStrArg(i++, "jwt token");
-                break;
-            case "--forceTokenReauth":
-                paramForceTokenReauth = true;
-                break;
-            case "--snapshotVersionIndex":
-                paramSnapshotVersionIndex = parseIntArg(i++, "version index", true);
-                break;
-            case "--numSnapshotVersions":
-                paramNumSnapshotVersions = parseIntArg(i++, "number of versions", false);
-                break;
-            case "--noUnpack":
-                paramUnpackAggregatedBlobs = false;
-                break;
-            case "--actualPayload":
-                paramActualFormatting = true;
-                break;
-            case "--saveDir":
-                paramSaveDir = parseStrArg(i++, "save data path");
-                break;
-            case "--websocket":
-                connectToWebSocket = true;
-                break;
-            case "--local":
-                localDataOnly = true;
-                break;
-            default:
-                try {
-                    const url = new URL(arg);
-                    if (url.protocol === "https:") {
-                        paramURL = arg;
-                        break;
-                    }
-                    if (url.protocol === "http:" && url.hostname === "localhost") {
-                        paramURL = arg;
-                        break;
-                    }
-                } catch (e) {
-                    console.error(e);
-                }
-
-                console.error(`ERROR: Invalid argument ${arg}`);
-                printUsage();
-                process.exit(-1);
-                break;
-        }
-    }
-    checkArgs();
+	for (let i = 2; i < process.argv.length; i++) {
+		const arg = process.argv[i];
+		switch (arg) {
+			case "--dump:rawmessage":
+				dumpMessages = true;
+				break;
+			case "--dump:rawmessage:overwrite":
+				dumpMessages = true;
+				overWrite = true;
+				break;
+			case "--stat:message":
+				dumpMessageStats = true;
+				break;
+			case "--stat":
+				dumpMessageStats = true;
+				dumpSnapshotStats = true;
+				break;
+			case "--filter:messageType":
+				messageTypeFilter.add(parseStrArg(i++, "type name for messageType filter"));
+				break;
+			case "--stat:snapshot":
+				dumpSnapshotStats = true;
+				break;
+			case "--dump:snapshotVersion":
+				dumpSnapshotVersions = true;
+				break;
+			case "--dump:snapshotTree":
+				dumpSnapshotTrees = true;
+				break;
+			case "--help":
+				printUsage();
+				process.exit(0);
+			case "--jwt":
+				paramJWT = parseStrArg(i++, "jwt token");
+				break;
+			case "--forceTokenReauth":
+				paramForceTokenReauth = true;
+				break;
+			case "--snapshotVersionIndex":
+				paramSnapshotVersionIndex = parseIntArg(i++, "version index", true);
+				break;
+			case "--numSnapshotVersions":
+				paramNumSnapshotVersions = parseIntArg(i++, "number of versions", false);
+				break;
+			case "--actualPayload":
+				paramActualFormatting = true;
+				break;
+			case "--saveDir":
+				paramSaveDir = parseStrArg(i++, "save data path");
+				break;
+			case "--websocket":
+				connectToWebSocket = true;
+				break;
+			case "--local":
+				localDataOnly = true;
+				break;
+			default:
+				try {
+					const url = new URL(arg);
+					if (url.protocol === "https:") {
+						paramURL = arg;
+						break;
+					}
+					if (url.protocol === "http:" && url.hostname === "localhost") {
+						paramURL = arg;
+						break;
+					}
+				} catch (e) {
+					console.error(e);
+				}
+
+				console.error(`ERROR: Invalid argument ${arg}`);
+				printUsage();
+				process.exit(-1);
+				break;
+		}
+	}
+	checkArgs();
 }
 
 function parseStrArg(i: number, name: string) {
-    if (i + 1 >= process.argv.length) {
-        console.error(`ERROR: Missing ${name}`);
-        printUsage();
-        process.exit(-1);
-    }
-    return process.argv[i + 1];
+	if (i + 1 >= process.argv.length) {
+		console.error(`ERROR: Missing ${name}`);
+		printUsage();
+		process.exit(-1);
+	}
+	return process.argv[i + 1];
 }
 function parseIntArg(i: number, name: string, allowZero: boolean) {
-    if (i + 1 >= process.argv.length) {
-        console.error(`ERROR: Missing ${name}`);
-        printUsage();
-        process.exit(-1);
-    }
-    const numStr = process.argv[i + 1];
-    const paramNumber = parseInt(numStr, 10);
-    if (isNaN(paramNumber) || (allowZero ? paramNumber < 0 : paramNumber <= 0)) {
-        console.error(`ERROR: Invalid ${name} ${numStr}`);
-        printUsage();
-        process.exit(-1);
-    }
-    return paramNumber;
+	if (i + 1 >= process.argv.length) {
+		console.error(`ERROR: Missing ${name}`);
+		printUsage();
+		process.exit(-1);
+	}
+	const numStr = process.argv[i + 1];
+	const paramNumber = parseInt(numStr, 10);
+	if (isNaN(paramNumber) || (allowZero ? paramNumber < 0 : paramNumber <= 0)) {
+		console.error(`ERROR: Invalid ${name} ${numStr}`);
+		printUsage();
+		process.exit(-1);
+	}
+	return paramNumber;
 }
 
 function checkArgs() {
-    if (paramSnapshotVersionIndex !== undefined) {
-        paramNumSnapshotVersions = Math.max(paramSnapshotVersionIndex + 1, paramNumSnapshotVersions);
-    }
-
-    if (paramURL === undefined) {
-        if (paramSaveDir !== undefined) {
-            const file = `${paramSaveDir}/info.json`;
-            if (fs.existsSync(file)) {
-                const info = JSON.parse(fs.readFileSync(file, { encoding: "utf-8" }));
-                paramURL = info.url;
-            } else {
-                console.log(`Can't find file ${file}`);
-            }
-        }
-
-        if (paramURL === undefined) {
-            console.error("ERROR: Missing URL");
-            printUsage();
-            process.exit(-1);
-        }
-    }
+	if (paramSnapshotVersionIndex !== undefined) {
+		paramNumSnapshotVersions = Math.max(
+			paramSnapshotVersionIndex + 1,
+			paramNumSnapshotVersions,
+		);
+	}
+
+	if (paramURL === undefined) {
+		if (paramSaveDir !== undefined) {
+			const file = `${paramSaveDir}/info.json`;
+			if (fs.existsSync(file)) {
+				const info = JSON.parse(fs.readFileSync(file, { encoding: "utf-8" }));
+				paramURL = info.url;
+			} else {
+				console.log(`Can't find file ${file}`);
+			}
+		}
+
+		if (paramURL === undefined) {
+			console.error("ERROR: Missing URL");
+			printUsage();
+			process.exit(-1);
+		}
+	}
 }