@fluid-tools/fetch-tool 0.53.0-46105

package/src/fluidFetch.ts

@@ -0,0 +1,123 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import fs from "fs";
+import util from "util";
+import { isOdspHostname, IOdspDriveItem } from "@fluidframework/odsp-doclib-utils";
+import { paramSaveDir, paramURL, parseArguments } from "./fluidFetchArgs";
+import { connectionInfo, fluidFetchInit } from "./fluidFetchInit";
+import { fluidFetchMessages } from "./fluidFetchMessages";
+import { getSharepointFiles, getSingleSharePointFile } from "./fluidFetchSharePoint";
+import { fluidFetchSnapshot } from "./fluidFetchSnapshot";
+
+async function fluidFetchOneFile(urlStr: string, name?: string) {
+    const documentService = await fluidFetchInit(urlStr);
+    const saveDir = paramSaveDir ? (name ? `${paramSaveDir}/${name}` : paramSaveDir) : undefined;
+    if (saveDir !== undefined) {
+        const mkdir = util.promisify(fs.mkdir);
+        const writeFile = util.promisify(fs.writeFile);
+        await mkdir(saveDir, { recursive: true });
+        const info = {
+            creationDate: new Date().toString(),
+            connectionInfo,
+            url: urlStr,
+        };
+        await writeFile(`${saveDir}/info.json`, JSON.stringify(info, undefined, 2));
+    }
+
+    await fluidFetchSnapshot(documentService, saveDir);
+    await fluidFetchMessages(documentService, saveDir);
+}
+
+async function tryFluidFetchOneSharePointFile(server: string, driveItem: IOdspDriveItem) {
+    const { path, name, driveId, itemId } = driveItem;
+    console.log(`File: ${path}/${name}`);
+    await fluidFetchOneFile(`https://${server}/_api/v2.1/drives/${driveId}/items/${itemId}`, name);
+}
+
+function getSharePointSpecificDriveItem(url: URL): { driveId: string; itemId: string } | undefined {
+    if (url.searchParams.has("driveId") && url.searchParams.has("itemId")) {
+        return {
+            driveId: url.searchParams.get("driveId") as string,
+            itemId: url.searchParams.get("itemId") as string,
+        };
+    }
+}
+
+function getSharepointServerRelativePathFromURL(url: URL) {
+    if (url.pathname.startsWith("/_api/v2.1/drives/")) {
+        return undefined;
+    }
+
+    const hostnameParts = url.hostname.split(".");
+    const suffix = hostnameParts[0].endsWith("-my") ? "/_layouts/15/onedrive.aspx" : "/forms/allitems.aspx";
+
+    let sitePath = url.pathname;
+    if (url.searchParams.has("id")) {
+        sitePath = url.searchParams.get("id") as string;
+    } else if (url.searchParams.has("RootFolder")) {
+        sitePath = url.searchParams.get("RootFolder") as string;
+    } else if (url.pathname.toLowerCase().endsWith(suffix)) {
+        sitePath = sitePath.substr(0, url.pathname.length - suffix.length);
+    }
+
+    return decodeURI(sitePath);
+}
+
+async function fluidFetchMain() {
+    if (!paramURL) {
+        return;
+    }
+
+    const url = new URL(paramURL);
+    const server = url.hostname;
+    if (isOdspHostname(server)) {
+        // See if the url already has the specific item
+        const driveItem = getSharePointSpecificDriveItem(url);
+        if (driveItem) {
+            const file = await getSingleSharePointFile(server, driveItem.driveId, driveItem.itemId);
+            await tryFluidFetchOneSharePointFile(server, file);
+            return;
+        }
+
+        // See if the url given represent a sharepoint directory
+        const serverRelativePath = getSharepointServerRelativePathFromURL(url);
+        if (serverRelativePath) {
+            const files = await getSharepointFiles(server, serverRelativePath, false);
+            for (const file of files) {
+                if (file.name.endsWith(".b") || file.name.endsWith(".fluid")) {
+                    await tryFluidFetchOneSharePointFile(server, file);
+                }
+            }
+            return;
+        }
+    }
+
+    return fluidFetchOneFile(paramURL);
+}
+
+parseArguments();
+
+// eslint-disable-next-line @typescript-eslint/no-floating-promises
+fluidFetchMain()
+    .catch((error: Error) => {
+        if (error instanceof Error) {
+            let extraMsg = "";
+            for (const key of Object.keys(error)) {
+                // error[key] might have circular structure
+                try {
+                    if (key !== "message" && key !== "stack") {
+                        extraMsg += `\n${key}: ${JSON.stringify(error[key], undefined, 2)}`;
+                    }
+                } catch (_) {}
+            }
+            console.error(`ERROR: ${error.stack}${extraMsg}`);
+        } else if (typeof error === "object") {
+            console.error(`ERROR: Unknown exception object\n${JSON.stringify(error, undefined, 2)}`);
+        } else {
+            console.error(`ERROR: ${error}`);
+        }
+    })
+    .then(() => process.exit(0));

package/src/fluidFetchArgs.ts

@@ -0,0 +1,224 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import fs from "fs";
+import { URL } from "url";
+
+export let dumpMessages = false;
+export let dumpMessageStats = false;
+export let dumpSnapshotStats = false;
+export let dumpSnapshotTrees = false;
+export let dumpSnapshotVersions = false;
+export let overWrite = false;
+export let paramSnapshotVersionIndex: number | undefined;
+export let paramNumSnapshotVersions = 10;
+export let paramUnpackAggregatedBlobs = true;
+export let paramActualFormatting = false;
+
+let paramForceTokenReauth = false;
+
+// Only return true once, to reauth on first call.
+export function getForceTokenReauth() {
+    const result = paramForceTokenReauth;
+    paramForceTokenReauth = false;
+    return result;
+}
+
+export let paramSaveDir: string | undefined;
+export const messageTypeFilter = new Set<string>();
+
+export let paramURL: string | undefined;
+export let paramJWT: string;
+export let paramAzureKey: string;
+
+export let connectToWebSocket = false;
+
+export let localDataOnly = false;
+
+export let paramSite: string | undefined;
+
+const optionsArray =
+    [
+        ["--dump:rawmessage", "dump all messages"],
+        ["--dump:snapshotVersion", "dump a list of snapshot version"],
+        ["--dump:snapshotTree", "dump the snapshot trees"],
+        ["--forceTokenReauth", "Force reauthorize token (SPO only)"],
+        ["--stat:message", "show message type, channel type, data type statistics"],
+        ["--stat:snapshot", "show a table of snapshot path and blob size"],
+        ["--stat", "Show both messages & snapshot stats"],
+        ["--filter:messageType <type>", "filter message by <type>"],
+        ["--jwt <token>", "token to be used for routerlicious URLs"],
+        ["--azureKey <key>", "secret key for Azure Fluid Relay instance"],
+        ["--numSnapshotVersions <number>", "Number of versions to load (default:10)"],
+        ["--noUnpack", "Do not unpack aggregated blobs"],
+        ["--actualPayload", "Do not format json payloads nicely, preserve actual bytes / formatting in storage"],
+        ["--saveDir <outdir>", "Save data of the snapshots and messages"],
+        ["--snapshotVersionIndex <number>", "Index of the version to dump"],
+        ["--websocket", "Connect to web socket to download initial messages"],
+        ["--local", "Do not connect to storage, use earlier downloaded data. Requires --saveDir."],
+    ];
+
+export function printUsage() {
+    console.log("Usage: fluid-fetch [options] URL");
+    console.log("URL: <ODSP URL>|<Routerlicious URL>");
+    console.log("Options:");
+    for (const i of optionsArray) {
+        console.log(`  ${i[0].padEnd(32)}: ${i[1]}`);
+    }
+}
+
+// Can be used in unit test to pass in customized argument values
+// More argument options can be added when needed
+export function setArguments(values: {
+    saveDir: string,
+    paramURL: string
+    dumpMessages?: boolean,
+    dumpMessageStats?: boolean,
+    dumpSnapshotStats?: boolean,
+    dumpSnapshotTrees?: boolean,
+    overWrite?: boolean }) {
+    paramSaveDir = values.saveDir;
+    paramURL = values.paramURL;
+    dumpMessages = values.dumpMessages ?? dumpMessages;
+    dumpMessageStats = values.dumpMessageStats ?? dumpMessageStats;
+    dumpSnapshotStats = values.dumpSnapshotStats ?? dumpSnapshotStats;
+    dumpSnapshotTrees = values.dumpSnapshotTrees ?? dumpSnapshotTrees;
+    overWrite = values.overWrite ?? overWrite;
+}
+
+export function parseArguments() {
+    for (let i = 2; i < process.argv.length; i++) {
+        const arg = process.argv[i];
+        switch (arg) {
+            case "--dump:rawmessage":
+                dumpMessages = true;
+                break;
+            case "--dump:rawmessage:overwrite":
+                dumpMessages = true;
+                overWrite = true;
+                break;
+            case "--stat:message":
+                dumpMessageStats = true;
+                break;
+            case "--stat":
+                dumpMessageStats = true;
+                dumpSnapshotStats = true;
+                break;
+            case "--filter:messageType":
+                messageTypeFilter.add(parseStrArg(i++, "type name for messageType filter"));
+                break;
+            case "--stat:snapshot":
+                dumpSnapshotStats = true;
+                break;
+            case "--dump:snapshotVersion":
+                dumpSnapshotVersions = true;
+                break;
+            case "--dump:snapshotTree":
+                dumpSnapshotTrees = true;
+                break;
+            case "--help":
+                printUsage();
+                process.exit(0);
+            case "--jwt":
+                paramJWT = parseStrArg(i++, "jwt token");
+                break;
+            case "--azureKey":
+                paramAzureKey = parseStrArg(i++, "Azure Fluid Relay key");
+                break;
+            case "--forceTokenReauth":
+                paramForceTokenReauth = true;
+                break;
+            case "--snapshotVersionIndex":
+                paramSnapshotVersionIndex = parseIntArg(i++, "version index", true);
+                break;
+            case "--numSnapshotVersions":
+                paramNumSnapshotVersions = parseIntArg(i++, "number of versions", false);
+                break;
+            case "--noUnpack":
+                paramUnpackAggregatedBlobs = false;
+                break;
+            case "--actualPayload":
+                paramActualFormatting = true;
+                break;
+            case "--saveDir":
+                paramSaveDir = parseStrArg(i++, "save data path");
+                break;
+            case "--websocket":
+                connectToWebSocket = true;
+                break;
+            case "--local":
+                localDataOnly = true;
+                break;
+            default:
+                try {
+                    const url = new URL(arg);
+                    if (url.protocol === "https:") {
+                        paramURL = arg;
+                        break;
+                    }
+                    if (url.protocol === "http:" && url.hostname === "localhost") {
+                        paramURL = arg;
+                        break;
+                    }
+                } catch (e) {
+                    console.error(e);
+                }
+
+                console.error(`ERROR: Invalid argument ${arg}`);
+                printUsage();
+                process.exit(-1);
+                break;
+        }
+    }
+    checkArgs();
+}
+
+function parseStrArg(i: number, name: string) {
+    if (i + 1 >= process.argv.length) {
+        console.error(`ERROR: Missing ${name}`);
+        printUsage();
+        process.exit(-1);
+    }
+    return process.argv[i + 1];
+}
+function parseIntArg(i: number, name: string, allowZero: boolean) {
+    if (i + 1 >= process.argv.length) {
+        console.error(`ERROR: Missing ${name}`);
+        printUsage();
+        process.exit(-1);
+    }
+    const numStr = process.argv[i + 1];
+    const paramNumber = parseInt(numStr, 10);
+    if (isNaN(paramNumber) || (allowZero ? paramNumber < 0 : paramNumber <= 0)) {
+        console.error(`ERROR: Invalid ${name} ${numStr}`);
+        printUsage();
+        process.exit(-1);
+    }
+    return paramNumber;
+}
+
+function checkArgs() {
+    if (paramSnapshotVersionIndex !== undefined) {
+        paramNumSnapshotVersions = Math.max(paramSnapshotVersionIndex + 1, paramNumSnapshotVersions);
+    }
+
+    if (!paramURL) {
+        if (paramSaveDir) {
+            const file = `${paramSaveDir}/info.json`;
+            if (fs.existsSync(file)) {
+                const info = JSON.parse(fs.readFileSync(file, { encoding: "utf-8" }));
+                paramURL = info.url;
+            } else {
+                console.log(`Can't find file ${file}`);
+            }
+        }
+
+        if (!paramURL) {
+            console.error("ERROR: Missing URL");
+            printUsage();
+            process.exit(-1);
+        }
+    }
+}

package/src/fluidFetchInit.ts

@@ -0,0 +1,168 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+
+import { URL } from "url";
+import child_process from "child_process";
+import { AzureUrlResolver } from "@fluidframework/azure-client/dist/AzureUrlResolver";
+import {
+    generateTestUser,
+    InsecureTokenProvider,
+} from "@fluidframework/test-client-utils";
+import { IFluidResolvedUrl, IResolvedUrl, IUrlResolver } from "@fluidframework/driver-definitions";
+import { configurableUrlResolver } from "@fluidframework/driver-utils";
+import { FluidAppOdspUrlResolver } from "@fluid-tools/fluidapp-odsp-urlresolver";
+import { IClientConfig, IOdspAuthRequestInfo } from "@fluidframework/odsp-doclib-utils";
+import * as odsp from "@fluidframework/odsp-driver";
+import { IOdspResolvedUrl, OdspResourceTokenFetchOptions } from "@fluidframework/odsp-driver-definitions";
+import { OdspUrlResolver } from "@fluidframework/odsp-urlresolver";
+import * as r11s from "@fluidframework/routerlicious-driver";
+import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver";
+import { getMicrosoftConfiguration } from "@fluidframework/tool-utils";
+import { localDataOnly, paramAzureKey, paramJWT } from "./fluidFetchArgs";
+import { resolveWrapper } from "./fluidFetchSharePoint";
+
+export let latestVersionsId: string = "";
+export let connectionInfo: any;
+
+export const fluidFetchWebNavigator = (url: string) => {
+    let message = "Please open browser and navigate to this URL:";
+    if (process.platform === "win32") {
+        child_process.exec(`start "fluid-fetch" /B "${url}"`);
+        message = "Opening browser to get authorization code.  If that doesn't open, please go to this URL manually";
+    }
+    console.log(`${message}\n  ${url}`);
+};
+
+async function initializeODSPCore(
+    odspResolvedUrl: IOdspResolvedUrl,
+    server: string,
+    clientConfig: IClientConfig,
+) {
+    const { driveId, itemId } = odspResolvedUrl;
+
+    connectionInfo = {
+        server,
+        drive: driveId,
+        item: itemId,
+    };
+
+    if (localDataOnly) {
+        return;
+    }
+
+    const docId = await odsp.getHashedDocumentId(driveId, itemId);
+
+    console.log(`Connecting to ODSP:
+  server: ${server}
+  drive:  ${driveId}
+  item:   ${itemId}
+  docId:  ${docId}`);
+
+    const getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {
+        return resolveWrapper(
+            async (authRequestInfo: IOdspAuthRequestInfo) => {
+                if ((options.refresh || !authRequestInfo.accessToken) && authRequestInfo.refreshTokenFn) {
+                    return authRequestInfo.refreshTokenFn();
+                }
+                return authRequestInfo.accessToken;
+            },
+            server,
+            clientConfig,
+            undefined,
+            true,
+        );
+    };
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) => Promise.resolve("");
+    const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(
+        getStorageTokenStub,
+        getWebsocketTokenStub,
+        undefined,
+        {
+            opsBatchSize: 20000,
+            concurrentOpsBatches: 4,
+        });
+    return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);
+}
+
+async function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IFluidResolvedUrl) {
+    const path = pathname.split("/");
+    let tenantId: string;
+    let documentId: string;
+    if (server === "localhost" && path.length < 4) {
+        tenantId = "fluid";
+        documentId = path[2];
+    } else {
+        tenantId = path[2];
+        documentId = path[3];
+    }
+
+    // Latest version id is the documentId for r11s
+    latestVersionsId = documentId;
+
+    connectionInfo = {
+        server,
+        tenantId,
+        id: documentId,
+    };
+
+    if (localDataOnly) {
+        return;
+    }
+
+    console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
+    const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);
+    const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
+    return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);
+}
+
+async function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string) {
+    connectionInfo = {
+        server: resolvedUrl.endpoints.ordererUrl,
+        tenantId,
+        id: resolvedUrl.id,
+    };
+    console.log(`Connecting to Azure Fluid Relay: tenantId=${tenantId} id:${resolvedUrl.id}`);
+    const user = generateTestUser();
+    const tokenProvider = new InsecureTokenProvider(paramAzureKey, user);
+    const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
+    return r11sDocumentServiceFactory.createDocumentService(resolvedUrl);
+}
+
+async function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {
+    const resolversList: IUrlResolver[] = [
+        new OdspUrlResolver(),
+        new FluidAppOdspUrlResolver(),
+        // eslint-disable-next-line @typescript-eslint/promise-function-async
+        new RouterliciousUrlResolver(undefined, () => Promise.resolve(paramJWT), ""),
+        new AzureUrlResolver(),
+    ];
+    const resolved = await configurableUrlResolver(resolversList, { url });
+    return resolved;
+}
+
+export async function fluidFetchInit(urlStr: string) {
+    const resolvedUrl = await resolveUrl(urlStr) as IFluidResolvedUrl;
+    if (!resolvedUrl) {
+        return Promise.reject(new Error(`Unknown URL ${urlStr}`));
+    }
+    const protocol = new URL(resolvedUrl.url).protocol;
+    if (protocol === "fluid-odsp:") {
+        const odspResolvedUrl = resolvedUrl as IOdspResolvedUrl;
+        return initializeODSPCore(odspResolvedUrl, new URL(odspResolvedUrl.siteUrl).host, getMicrosoftConfiguration());
+    } else if (protocol === "fluid:") {
+        const url = new URL(urlStr);
+        const server = url.hostname.toLowerCase();
+        return initializeR11s(server, url.pathname, resolvedUrl);
+    } else if (resolvedUrl.url.includes("fluidrelay.azure.com")) {
+        const url = new URL(urlStr);
+        const tenantId = url.searchParams.get("tenantId");
+        if (tenantId === null) {
+            throw new Error("Azure URL did not contain tenantId");
+        }
+        return initializeAzure(resolvedUrl, tenantId);
+    }
+    return Promise.reject(new Error(`Unknown resolved protocol ${protocol}`));
+}