@fluid-tools/fetch-tool 0.59.2001 → 0.59.3000

package/src/fluidAnalyzeMessages.ts

@@ -226,7 +226,7 @@ class DataStructureAnalyzer implements IMessageAnalyzer {
     private readonly dataType = new Map<string, string>();
     private readonly dataTypeStats = new Map<string, [number, number]>();
     private readonly objectStats = new Map<string, [number, number]>();
-    private readonly chunkMap = new Map<string, {chunks: string[], totalSize: number}>();
+    private readonly chunkMap = new Map<string, { chunks: string[]; totalSize: number; }>();
 
     public processOp(message: ISequencedDocumentMessage, msgSize: number, skipMessage: boolean): void {
         if (!skipMessage) {
@@ -479,7 +479,7 @@ function processOp(
     msgSize: number,
     dataTypeStats: Map<string, [number, number]>,
     messageTypeStats: Map<string, [number, number]>,
-    chunkMap: Map<string, {chunks: string[], totalSize: number}>) {
+    chunkMap: Map<string, { chunks: string[]; totalSize: number; }>) {
     let type = message.type;
     let recorded = false;
     let totalMsgSize = msgSize;
@@ -500,7 +500,11 @@ function processOp(
             case RuntimeMessage.ChunkedOp: {
                 const chunk = runtimeMessage.contents as IChunkedOp;
                 if (!chunkMap.has(runtimeMessage.clientId)) {
-                    chunkMap.set(runtimeMessage.clientId, {chunks: new Array<string>(chunk.totalChunks), totalSize:0});
+                    chunkMap.set(
+                        runtimeMessage.clientId, {
+                        chunks: new Array<string>(chunk.totalChunks),
+                        totalSize: 0,
+                    });
                 }
                 const value = chunkMap.get(runtimeMessage.clientId);
                 assert(value !== undefined, 0x2b8 /* "Chunk should be set in map" */);
@@ -528,68 +532,68 @@ function processOp(
             case RuntimeMessage.Alias:
             case RuntimeMessage.Rejoin:
             case RuntimeMessage.Operation:
-            {
-                let envelope = runtimeMessage.contents as IEnvelope;
-                // TODO: Legacy?
-                if (envelope && typeof envelope === "string") {
-                    envelope = JSON.parse(envelope);
-                }
-                const innerContent = envelope.contents as {
-                    content: any;
-                    type: string;
-                };
-                const address = envelope.address;
-                type = `${type}/${innerContent.type}`;
-                switch (innerContent.type) {
-                    case DataStoreMessageType.Attach: {
-                        const attachMessage = innerContent.content as IAttachMessage;
-                        let objectType = attachMessage.type;
-                        if (objectType.startsWith(objectTypePrefix)) {
-                            objectType = objectType.substring(objectTypePrefix.length);
-                        }
-                        dataType.set(getObjectId(address, attachMessage.id), objectType);
-                        break;
+                {
+                    let envelope = runtimeMessage.contents as IEnvelope;
+                    // TODO: Legacy?
+                    if (envelope && typeof envelope === "string") {
+                        envelope = JSON.parse(envelope);
                     }
-                    case DataStoreMessageType.ChannelOp:
-                    default: {
-                        const innerEnvelope = innerContent.content as IEnvelope;
-                        const innerContent2 = innerEnvelope.contents as {
-                            type?: string;
-                            value?: any;
-                        };
-
-                        const objectId = getObjectId(address, innerEnvelope.address);
-                        incr(objectStats, objectId, totalMsgSize, opCount);
-                        let objectType = dataType.get(objectId);
-                        if (objectType === undefined) {
-                            // Somehow we do not have data...
-                            dataType.set(objectId, objectId);
-                            objectType = objectId;
-                        }
-                        incr(dataTypeStats, objectType, totalMsgSize, opCount);
-                        recorded = true;
-
-                        let subType = innerContent2.type;
-                        if (innerContent2.type === "set" &&
-                            typeof innerContent2.value === "object" &&
-                            innerContent2.value !== null) {
-                            type = `${type}/${subType}`;
-                            subType = innerContent2.value.type;
-                        } else if (objectType === "mergeTree" && subType !== undefined) {
-                            const types = ["insert", "remove", "annotate", "group"];
-                            if (types[subType]) {
-                                subType = types[subType];
+                    const innerContent = envelope.contents as {
+                        content: any;
+                        type: string;
+                    };
+                    const address = envelope.address;
+                    type = `${type}/${innerContent.type}`;
+                    switch (innerContent.type) {
+                        case DataStoreMessageType.Attach: {
+                            const attachMessage = innerContent.content as IAttachMessage;
+                            let objectType = attachMessage.type;
+                            if (objectType.startsWith(objectTypePrefix)) {
+                                objectType = objectType.substring(objectTypePrefix.length);
                             }
+                            dataType.set(getObjectId(address, attachMessage.id), objectType);
+                            break;
                         }
-                        if (subType !== undefined) {
-                            type = `${type}/${subType}`;
-                        }
+                        case DataStoreMessageType.ChannelOp:
+                        default: {
+                            const innerEnvelope = innerContent.content as IEnvelope;
+                            const innerContent2 = innerEnvelope.contents as {
+                                type?: string;
+                                value?: any;
+                            };
+
+                            const objectId = getObjectId(address, innerEnvelope.address);
+                            incr(objectStats, objectId, totalMsgSize, opCount);
+                            let objectType = dataType.get(objectId);
+                            if (objectType === undefined) {
+                                // Somehow we do not have data...
+                                dataType.set(objectId, objectId);
+                                objectType = objectId;
+                            }
+                            incr(dataTypeStats, objectType, totalMsgSize, opCount);
+                            recorded = true;
+
+                            let subType = innerContent2.type;
+                            if (innerContent2.type === "set" &&
+                                typeof innerContent2.value === "object" &&
+                                innerContent2.value !== null) {
+                                type = `${type}/${subType}`;
+                                subType = innerContent2.value.type;
+                            } else if (objectType === "mergeTree" && subType !== undefined) {
+                                const types = ["insert", "remove", "annotate", "group"];
+                                if (types[subType]) {
+                                    subType = types[subType];
+                                }
+                            }
+                            if (subType !== undefined) {
+                                type = `${type}/${subType}`;
+                            }
 
-                        type = `${type} (${objectType})`;
+                            type = `${type} (${objectType})`;
+                        }
                     }
+                    break;
                 }
-                break;
-            }
             default:
                 unreachableCase(messageType, "Message type not recognized!");
         }

package/src/fluidFetch.ts

@@ -37,7 +37,7 @@ async function tryFluidFetchOneSharePointFile(server: string, driveItem: IOdspDr
     await fluidFetchOneFile(`https://${server}/_api/v2.1/drives/${driveId}/items/${itemId}`, name);
 }
 
-function getSharePointSpecificDriveItem(url: URL): { driveId: string; itemId: string } | undefined {
+function getSharePointSpecificDriveItem(url: URL): { driveId: string; itemId: string; } | undefined {
     if (url.searchParams.has("driveId") && url.searchParams.has("itemId")) {
         return {
             driveId: url.searchParams.get("driveId") as string,

package/src/fluidFetchArgs.ts

@@ -31,7 +31,6 @@ export const messageTypeFilter = new Set<string>();
 
 export let paramURL: string | undefined;
 export let paramJWT: string;
-export let paramAzureKey: string;
 
 export let connectToWebSocket = false;
 
@@ -50,7 +49,6 @@ const optionsArray =
         ["--stat", "Show both messages & snapshot stats"],
         ["--filter:messageType <type>", "filter message by <type>"],
         ["--jwt <token>", "token to be used for routerlicious URLs"],
-        ["--azureKey <key>", "secret key for Azure Fluid Relay instance"],
         ["--numSnapshotVersions <number>", "Number of versions to load (default:10)"],
         ["--noUnpack", "Do not unpack aggregated blobs"],
         ["--actualPayload", "Do not format json payloads nicely, preserve actual bytes / formatting in storage"],
@@ -72,13 +70,13 @@ export function printUsage() {
 // Can be used in unit test to pass in customized argument values
 // More argument options can be added when needed
 export function setArguments(values: {
-    saveDir: string,
-    paramURL: string
-    dumpMessages?: boolean,
-    dumpMessageStats?: boolean,
-    dumpSnapshotStats?: boolean,
-    dumpSnapshotTrees?: boolean,
-    overWrite?: boolean }) {
+    saveDir: string;
+    paramURL: string;
+    dumpMessages?: boolean;
+    dumpMessageStats?: boolean;
+    dumpSnapshotStats?: boolean;
+    dumpSnapshotTrees?: boolean;
+    overWrite?: boolean; }) {
     paramSaveDir = values.saveDir;
     paramURL = values.paramURL;
     dumpMessages = values.dumpMessages ?? dumpMessages;
@@ -121,13 +119,9 @@ export function parseArguments() {
             case "--help":
                 printUsage();
                 process.exit(0);
-            // fallthrough
             case "--jwt":
                 paramJWT = parseStrArg(i++, "jwt token");
                 break;
-            case "--azureKey":
-                paramAzureKey = parseStrArg(i++, "Azure Fluid Relay key");
-                break;
             case "--forceTokenReauth":
                 paramForceTokenReauth = true;
                 break;

package/src/fluidFetchInit.ts

@@ -5,11 +5,6 @@
 
 import { URL } from "url";
 import child_process from "child_process";
-import { AzureUrlResolver } from "@fluidframework/azure-client/dist/AzureUrlResolver";
-import {
-    generateTestUser,
-    InsecureTokenProvider,
-} from "@fluidframework/test-client-utils";
 import { IFluidResolvedUrl, IResolvedUrl, IUrlResolver } from "@fluidframework/driver-definitions";
 import { configurableUrlResolver } from "@fluidframework/driver-utils";
 import { FluidAppOdspUrlResolver } from "@fluid-tools/fluidapp-odsp-urlresolver";
@@ -20,7 +15,7 @@ import { OdspUrlResolver } from "@fluidframework/odsp-urlresolver";
 import * as r11s from "@fluidframework/routerlicious-driver";
 import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver";
 import { getMicrosoftConfiguration } from "@fluidframework/tool-utils";
-import { localDataOnly, paramAzureKey, paramJWT } from "./fluidFetchArgs";
+import { localDataOnly, paramJWT } from "./fluidFetchArgs";
 import { resolveWrapper } from "./fluidFetchSharePoint";
 
 export let latestVersionsId: string = "";
@@ -84,11 +79,7 @@ async function initializeODSPCore(
             opsBatchSize: 20000,
             concurrentOpsBatches: 4,
         });
-    return odspDocumentServiceFactory.createDocumentService(
-        odspResolvedUrl,
-        undefined, // logger
-         false, // clientIsSummrizer
-    );
+    return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);
 }
 
 async function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IFluidResolvedUrl) {
@@ -119,28 +110,7 @@ async function initializeR11s(server: string, pathname: string, r11sResolvedUrl:
     console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
     const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);
     const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
-    return r11sDocumentServiceFactory.createDocumentService(
-        r11sResolvedUrl,
-        undefined, // logger
-        false, // clientIsSummarizer
-    );
-}
-
-async function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string) {
-    connectionInfo = {
-        server: resolvedUrl.endpoints.ordererUrl,
-        tenantId,
-        id: resolvedUrl.id,
-    };
-    console.log(`Connecting to Azure Fluid Relay: tenantId=${tenantId} id:${resolvedUrl.id}`);
-    const user = generateTestUser();
-    const tokenProvider = new InsecureTokenProvider(paramAzureKey, user);
-    const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
-    return r11sDocumentServiceFactory.createDocumentService(
-        resolvedUrl,
-        undefined, // logger
-        false, // clientIsSummarizer
-    );
+    return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);
 }
 
 async function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {
@@ -149,7 +119,6 @@ async function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {
         new FluidAppOdspUrlResolver(),
         // eslint-disable-next-line @typescript-eslint/promise-function-async
         new RouterliciousUrlResolver(undefined, () => Promise.resolve(paramJWT), ""),
-        new AzureUrlResolver(),
     ];
     const resolved = await configurableUrlResolver(resolversList, { url });
     return resolved;
@@ -168,13 +137,6 @@ export async function fluidFetchInit(urlStr: string) {
         const url = new URL(urlStr);
         const server = url.hostname.toLowerCase();
         return initializeR11s(server, url.pathname, resolvedUrl);
-    } else if (resolvedUrl.url.includes("fluidrelay.azure.com")) {
-        const url = new URL(urlStr);
-        const tenantId = url.searchParams.get("tenantId");
-        if (tenantId === null) {
-            throw new Error("Azure URL did not contain tenantId");
-        }
-        return initializeAzure(resolvedUrl, tenantId);
     }
     return Promise.reject(new Error(`Unknown resolved protocol ${protocol}`));
 }

package/src/fluidFetchSharePoint.ts

@@ -97,7 +97,7 @@ export async function getSharepointFiles(server: string, serverRelativePath: str
 
     const fileInfo = await resolveDriveItemByServerRelativePath(server, serverRelativePath, clientConfig);
     console.log(fileInfo);
-    const pendingFolder: { path: string, folder: IOdspDriveItem }[] = [];
+    const pendingFolder: { path: string; folder: IOdspDriveItem; }[] = [];
     const files: IOdspDriveItem[] = [];
     if (fileInfo.isFolder) {
         pendingFolder.push({ path: serverRelativePath, folder: fileInfo });

package/src/fluidFetchSnapshot.ts

@@ -104,7 +104,7 @@ function fetchBlobs(prefix: string,
 
 function createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {
     const id = tree.id ?? "original";
-    const blob = stringToBuffer(JSON.stringify(tree),"utf8");
+    const blob = stringToBuffer(JSON.stringify(tree), "utf8");
     const filename = patched ? "tree" : `tree-${id}`;
     const treePath = `${prefix}${filename}`;
     return { treePath, blobId: "original tree $id", filename, blob, patched, reused: false };
@@ -172,7 +172,7 @@ async function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]
         if (buffer === undefined) {
             continue;
         }
-        const blob = bufferToString(buffer,"utf8");
+        const blob = bufferToString(buffer, "utf8");
         // eslint-disable-next-line max-len
         console.log(`${item.treePath.padEnd(nameLength)} |    ${item.reused ? "X" : " "}   | ${formatNumber(blob.length).padStart(10)}`);
         size += blob.length;
@@ -222,7 +222,7 @@ async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir:
 
             // we assume that the buffer is utf8 here, which currently is true for
             // all of our snapshot blobs.  It doesn't necessary be true in the future
-            let decoded = bufferToString(buffer,"utf8");
+            let decoded = bufferToString(buffer, "utf8");
             try {
                 if (!paramActualFormatting) {
                     decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);
@@ -233,7 +233,7 @@ async function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir:
                 `${outDir}/decoded/${item.filename}.json`, decoded);
         } else {
             // Write out same data for tree decoded or not, except for formatting
-            const treeString = bufferToString(buffer,"utf8");
+            const treeString = bufferToString(buffer, "utf8");
             fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);
             fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,
                 paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));