@fluid-tools/fetch-tool 2.0.0-dev-rc.2.0.0.246488 → 2.0.0-dev-rc.3.0.0.253463

package/lib/fluidFetchInit.js

@@ -0,0 +1,117 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import * as odsp from "@fluidframework/odsp-driver/internal";
+import { FluidAppOdspUrlResolver, OdspUrlResolver, } from "@fluidframework/odsp-urlresolver/internal";
+import * as r11s from "@fluidframework/routerlicious-driver/internal";
+import { RouterliciousUrlResolver } from "@fluidframework/routerlicious-urlresolver/internal";
+import { getMicrosoftConfiguration } from "@fluidframework/tool-utils/internal";
+import { localDataOnly, paramJWT } from "./fluidFetchArgs.js";
+import { resolveWrapper } from "./fluidFetchSharePoint.js";
+export let latestVersionsId = "";
+export let connectionInfo;
+async function initializeODSPCore(odspResolvedUrl, server, clientConfig) {
+    const { driveId, itemId } = odspResolvedUrl;
+    connectionInfo = {
+        server,
+        drive: driveId,
+        item: itemId,
+    };
+    if (localDataOnly) {
+        return;
+    }
+    const docId = await odsp.getHashedDocumentId(driveId, itemId);
+    console.log(`Connecting to ODSP:
+  server: ${server}
+  drive:  ${driveId}
+  item:   ${itemId}
+  docId:  ${docId}`);
+    const getStorageTokenStub = async (options) => {
+        return resolveWrapper(async (authRequestInfo) => {
+            if ((options.refresh || !authRequestInfo.accessToken) &&
+                authRequestInfo.refreshTokenFn) {
+                return authRequestInfo.refreshTokenFn();
+            }
+            return authRequestInfo.accessToken;
+        }, server, clientConfig, undefined, true);
+    };
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    const getWebsocketTokenStub = (_options) => Promise.resolve("");
+    const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(getStorageTokenStub, getWebsocketTokenStub, undefined, {
+        opsBatchSize: 20000,
+        concurrentOpsBatches: 4,
+    });
+    return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);
+}
+async function initializeR11s(server, pathname, r11sResolvedUrl) {
+    const path = pathname.split("/");
+    let tenantId;
+    let documentId;
+    if (server === "localhost" && path.length < 4) {
+        tenantId = "fluid";
+        documentId = path[2];
+    }
+    else {
+        tenantId = path[2];
+        documentId = path[3];
+    }
+    // Latest version id is the documentId for r11s
+    latestVersionsId = documentId;
+    connectionInfo = {
+        server,
+        tenantId,
+        id: documentId,
+    };
+    if (localDataOnly) {
+        return;
+    }
+    console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
+    const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);
+    const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
+    return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);
+}
+async function resolveUrl(url) {
+    const request = { url };
+    let maybeResolvedUrl;
+    // Try each url resolver in turn to figure out which one the request is compatible with.
+    maybeResolvedUrl = await new OdspUrlResolver().resolve(request);
+    if (maybeResolvedUrl !== undefined) {
+        return {
+            resolvedUrl: maybeResolvedUrl,
+            serviceType: "odsp",
+        };
+    }
+    maybeResolvedUrl = await new FluidAppOdspUrlResolver().resolve(request);
+    if (maybeResolvedUrl !== undefined) {
+        return {
+            resolvedUrl: maybeResolvedUrl,
+            serviceType: "odsp",
+        };
+    }
+    maybeResolvedUrl = await new RouterliciousUrlResolver(undefined, async () => Promise.resolve(paramJWT), "").resolve(request);
+    if (maybeResolvedUrl !== undefined) {
+        return {
+            resolvedUrl: maybeResolvedUrl,
+            serviceType: "r11s",
+        };
+    }
+    return undefined;
+}
+export async function fluidFetchInit(urlStr) {
+    const resolvedInfo = await resolveUrl(urlStr);
+    if (resolvedInfo === undefined) {
+        throw new Error(`Unknown URL ${urlStr}`);
+    }
+    const fluidResolvedUrl = resolvedInfo.resolvedUrl;
+    if (resolvedInfo.serviceType === "odsp") {
+        const odspResolvedUrl = fluidResolvedUrl;
+        return initializeODSPCore(odspResolvedUrl, new URL(odspResolvedUrl.siteUrl).host, getMicrosoftConfiguration());
+    }
+    else if (resolvedInfo.serviceType === "r11s") {
+        const url = new URL(urlStr);
+        const server = url.hostname.toLowerCase();
+        return initializeR11s(server, url.pathname, fluidResolvedUrl);
+    }
+}
+//# sourceMappingURL=fluidFetchInit.js.map

package/lib/fluidFetchInit.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchInit.js","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,KAAK,IAAI,MAAM,sCAAsC,CAAC;AAK7D,OAAO,EACN,uBAAuB,EACvB,eAAe,GACf,MAAM,2CAA2C,CAAC;AACnD,OAAO,KAAK,IAAI,MAAM,+CAA+C,CAAC;AACtE,OAAO,EAAE,wBAAwB,EAAE,MAAM,oDAAoD,CAAC;AAC9F,OAAO,EAAE,yBAAyB,EAAE,MAAM,qCAAqC,CAAC;AAEhF,OAAO,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,qBAAqB,CAAC;AAC9D,OAAO,EAAE,cAAc,EAAE,MAAM,2BAA2B,CAAC;AAE3D,MAAM,CAAC,IAAI,gBAAgB,GAAW,EAAE,CAAC;AACzC,MAAM,CAAC,IAAI,cAAmB,CAAC;AAE/B,KAAK,UAAU,kBAAkB,CAChC,eAAiC,EACjC,MAAc,EACd,YAA2B;IAE3B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;IAE5C,cAAc,GAAG;QAChB,MAAM;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACZ,CAAC;IAEF,IAAI,aAAa,EAAE;QAClB,OAAO;KACP;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,CAAC;YACD,MAAM;YACN,OAAO;YACP,MAAM;YACN,KAAK,EAAE,CAAC,CAAC;IAEpB,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAsC,EAAE,EAAE;QAC5E,OAAO,cAAc,CACpB,KAAK,EAAE,eAAqC,EAAE,EAAE;YAC/C,IACC,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC;gBACjD,eAAe,CAAC,cAAc,EAC7B;gBACD,OAAO,eAAe,CAAC,cAAc,EAAE,CAAC;aACxC;YACD,OAAO,eAAe,CAAC,WAAW,CAAC;QACpC,CAAC,EACD,MAAM,EACN,YAAY,EACZ,SAAS,EACT,IAAI,CACJ,CAAC;IACH,CAAC,CAAC;IACF,qEAAqE;IACrE,MAAM,qBAAqB,GAAG,CAAC,QAAuC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,0BAA0B,CACrE,mBAAmB,EACnB,qBAAqB,EACrB,SAAS,EACT;QACC,YAAY,EAAE,KAAK;QACnB,oBAAoB,EAAE,CAAC;KACvB,CACD,CAAC;IACF,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,eAA6B;IAC5F,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;QAC9C,QAAQ,GAAG,OAAO,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACrB;SAAM;QACN,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACrB;IAED,+CAA+C;IAC/C,gBAAgB,GAAG,UAAU,CAAC;IAE9B,cAAc,GAAG;QAChB,MAAM;QACN,QAAQ;QACR,EAAE,EAAE,UAAU;KACd,CAAC;IAEF,IAAI,aAAa,EAAE;QAClB,OAAO;KACP;IAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,OAAO,UAAU,EAAE,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,oBAAoB,CAAC,QAAQ,CAAC,CAAC;IAC9D,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC1E,CAAC;AAMD,KAAK,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,OAAO,GAAa,EAAE,GAAG,EAAE,CAAC;IAClC,IAAI,gBAA0C,CAAC;IAE/C,wFAAwF;IACxF,gBAAgB,GAAG,MAAM,IAAI,eAAe,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChE,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,gBAAgB,GAAG,MAAM,IAAI,uBAAuB,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,gBAAgB,GAAG,MAAM,IAAI,wBAAwB,CACpD,SAAS,EACT,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,EACrC,EAAE,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,cAAc,CAAC,MAAc;IAClD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,YAAY,KAAK,SAAS,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,eAAe,MAAM,EAAE,CAAC,CAAC;KACzC;IACD,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC;IAClD,IAAI,YAAY,CAAC,WAAW,KAAK,MAAM,EAAE;QACxC,MAAM,eAAe,GAAG,gBAAoC,CAAC;QAC7D,OAAO,kBAAkB,CACxB,eAAe,EACf,IAAI,GAAG,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EACrC,yBAAyB,EAAE,CAC3B,CAAC;KACF;SAAM,IAAI,YAAY,CAAC,WAAW,KAAK,MAAM,EAAE;QAC/C,MAAM,GAAG,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;KAC9D;AACF,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IRequest } from \"@fluidframework/core-interfaces\";\nimport { IResolvedUrl } from \"@fluidframework/driver-definitions/internal\";\nimport { IClientConfig, IOdspAuthRequestInfo } from \"@fluidframework/odsp-doclib-utils/internal\";\nimport * as odsp from \"@fluidframework/odsp-driver/internal\";\nimport {\n\tIOdspResolvedUrl,\n\tOdspResourceTokenFetchOptions,\n} from \"@fluidframework/odsp-driver-definitions/internal\";\nimport {\n\tFluidAppOdspUrlResolver,\n\tOdspUrlResolver,\n} from \"@fluidframework/odsp-urlresolver/internal\";\nimport * as r11s from \"@fluidframework/routerlicious-driver/internal\";\nimport { RouterliciousUrlResolver } from \"@fluidframework/routerlicious-urlresolver/internal\";\nimport { getMicrosoftConfiguration } from \"@fluidframework/tool-utils/internal\";\n\nimport { localDataOnly, paramJWT } from \"./fluidFetchArgs.js\";\nimport { resolveWrapper } from \"./fluidFetchSharePoint.js\";\n\nexport let latestVersionsId: string = \"\";\nexport let connectionInfo: any;\n\nasync function initializeODSPCore(\n\todspResolvedUrl: IOdspResolvedUrl,\n\tserver: string,\n\tclientConfig: IClientConfig,\n) {\n\tconst { driveId, itemId } = odspResolvedUrl;\n\n\tconnectionInfo = {\n\t\tserver,\n\t\tdrive: driveId,\n\t\titem: itemId,\n\t};\n\n\tif (localDataOnly) {\n\t\treturn;\n\t}\n\n\tconst docId = await odsp.getHashedDocumentId(driveId, itemId);\n\n\tconsole.log(`Connecting to ODSP:\n  server: ${server}\n  drive:  ${driveId}\n  item:   ${itemId}\n  docId:  ${docId}`);\n\n\tconst getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {\n\t\treturn resolveWrapper(\n\t\t\tasync (authRequestInfo: IOdspAuthRequestInfo) => {\n\t\t\t\tif (\n\t\t\t\t\t(options.refresh || !authRequestInfo.accessToken) &&\n\t\t\t\t\tauthRequestInfo.refreshTokenFn\n\t\t\t\t) {\n\t\t\t\t\treturn authRequestInfo.refreshTokenFn();\n\t\t\t\t}\n\t\t\t\treturn authRequestInfo.accessToken;\n\t\t\t},\n\t\t\tserver,\n\t\t\tclientConfig,\n\t\t\tundefined,\n\t\t\ttrue,\n\t\t);\n\t};\n\t// eslint-disable-next-line @typescript-eslint/promise-function-async\n\tconst getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) => Promise.resolve(\"\");\n\tconst odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(\n\t\tgetStorageTokenStub,\n\t\tgetWebsocketTokenStub,\n\t\tundefined,\n\t\t{\n\t\t\topsBatchSize: 20000,\n\t\t\tconcurrentOpsBatches: 4,\n\t\t},\n\t);\n\treturn odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);\n}\n\nasync function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IResolvedUrl) {\n\tconst path = pathname.split(\"/\");\n\tlet tenantId: string;\n\tlet documentId: string;\n\tif (server === \"localhost\" && path.length < 4) {\n\t\ttenantId = \"fluid\";\n\t\tdocumentId = path[2];\n\t} else {\n\t\ttenantId = path[2];\n\t\tdocumentId = path[3];\n\t}\n\n\t// Latest version id is the documentId for r11s\n\tlatestVersionsId = documentId;\n\n\tconnectionInfo = {\n\t\tserver,\n\t\ttenantId,\n\t\tid: documentId,\n\t};\n\n\tif (localDataOnly) {\n\t\treturn;\n\t}\n\n\tconsole.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);\n\tconst tokenProvider = new r11s.DefaultTokenProvider(paramJWT);\n\tconst r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n\treturn r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);\n}\n\ninterface IResolvedInfo {\n\tresolvedUrl: IResolvedUrl;\n\tserviceType: \"odsp\" | \"r11s\";\n}\nasync function resolveUrl(url: string): Promise<IResolvedInfo | undefined> {\n\tconst request: IRequest = { url };\n\tlet maybeResolvedUrl: IResolvedUrl | undefined;\n\n\t// Try each url resolver in turn to figure out which one the request is compatible with.\n\tmaybeResolvedUrl = await new OdspUrlResolver().resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"odsp\",\n\t\t};\n\t}\n\n\tmaybeResolvedUrl = await new FluidAppOdspUrlResolver().resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"odsp\",\n\t\t};\n\t}\n\n\tmaybeResolvedUrl = await new RouterliciousUrlResolver(\n\t\tundefined,\n\t\tasync () => Promise.resolve(paramJWT),\n\t\t\"\",\n\t).resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"r11s\",\n\t\t};\n\t}\n\n\treturn undefined;\n}\n\nexport async function fluidFetchInit(urlStr: string) {\n\tconst resolvedInfo = await resolveUrl(urlStr);\n\tif (resolvedInfo === undefined) {\n\t\tthrow new Error(`Unknown URL ${urlStr}`);\n\t}\n\tconst fluidResolvedUrl = resolvedInfo.resolvedUrl;\n\tif (resolvedInfo.serviceType === \"odsp\") {\n\t\tconst odspResolvedUrl = fluidResolvedUrl as IOdspResolvedUrl;\n\t\treturn initializeODSPCore(\n\t\t\todspResolvedUrl,\n\t\t\tnew URL(odspResolvedUrl.siteUrl).host,\n\t\t\tgetMicrosoftConfiguration(),\n\t\t);\n\t} else if (resolvedInfo.serviceType === \"r11s\") {\n\t\tconst url = new URL(urlStr);\n\t\tconst server = url.hostname.toLowerCase();\n\t\treturn initializeR11s(server, url.pathname, fluidResolvedUrl);\n\t}\n}\n"]}

package/lib/fluidFetchMessages.d.ts

@@ -0,0 +1,7 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { IDocumentService } from "@fluidframework/driver-definitions/internal";
+export declare function fluidFetchMessages(documentService?: IDocumentService, saveDir?: string): Promise<void>;
+//# sourceMappingURL=fluidFetchMessages.d.ts.map

package/lib/fluidFetchMessages.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchMessages.d.ts","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EAAE,gBAAgB,EAAE,MAAM,6CAA6C,CAAC;AAsQ/E,wBAAsB,kBAAkB,CAAC,eAAe,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,MAAM,iBAgC5F"}

package/lib/fluidFetchMessages.js

@@ -0,0 +1,225 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import fs from "fs";
+import { assert } from "@fluidframework/core-utils/internal";
+import { MessageType, ScopeType, } from "@fluidframework/protocol-definitions";
+import { printMessageStats } from "./fluidAnalyzeMessages.js";
+import { connectToWebSocket, dumpMessageStats, dumpMessages, messageTypeFilter, overWrite, paramActualFormatting, } from "./fluidFetchArgs.js";
+function filenameFromIndex(index) {
+    return index === 0 ? "" : index.toString(); // support old tools...
+}
+let firstAvailableDelta = 1;
+async function* loadAllSequencedMessages(documentService, dir, files) {
+    let lastSeq = 0;
+    // flag for mismatch between last sequence number read and new one to be read
+    let seqNumMismatch = false;
+    // If we have local save, read ops from there first
+    if (files !== undefined) {
+        for (let i = 0; i < files.length; i++) {
+            const file = filenameFromIndex(i);
+            try {
+                console.log(`reading messages${file}.json`);
+                const fileContent = fs.readFileSync(`${dir}/messages${file}.json`, {
+                    encoding: "utf-8",
+                });
+                const messages = JSON.parse(fileContent);
+                // check if there is mismatch
+                seqNumMismatch = messages[0].sequenceNumber !== lastSeq + 1;
+                assert(!seqNumMismatch, 0x1b9 /* "Unexpected value for sequence number of first message in file" */);
+                lastSeq = messages[messages.length - 1].sequenceNumber;
+                yield messages;
+            }
+            catch (e) {
+                if (seqNumMismatch) {
+                    if (overWrite) {
+                        // with overWrite option on, we will delete all exisintg message.json files
+                        for (let index = 0; index < files.length; index++) {
+                            const name = filenameFromIndex(index);
+                            fs.unlinkSync(`${dir}/messages${name}.json`);
+                        }
+                        break;
+                    }
+                    // prompt user to back up and delete existing files
+                    console.error("There are deleted ops in the document being requested," +
+                        " please back up the existing messages.json file and delete it from its directory." +
+                        " Then try fetch tool again.");
+                    console.error(e);
+                    return;
+                }
+                else {
+                    console.error(`Error reading / parsing messages from ${files}`);
+                    console.error(e);
+                    return;
+                }
+            }
+        }
+        if (lastSeq !== 0) {
+            console.log(`Read ${lastSeq} ops from local cache`);
+        }
+    }
+    if (!documentService) {
+        return;
+    }
+    const deltaStorage = await documentService.connectToDeltaStorage();
+    let timeStart = Date.now();
+    let requests = 0;
+    let opsStorage = 0;
+    // reading only 1 op to test if there is mismatch
+    const teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);
+    let statusCode;
+    let innerMostErrorCode;
+    let response;
+    try {
+        await teststream.read();
+    }
+    catch (error) {
+        statusCode = error.getTelemetryProperties().statusCode;
+        innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
+        // if there is gap between ops, catch the error and check it is the error we need
+        if (statusCode !== 410 || innerMostErrorCode !== "fluidDeltaDataNotAvailable") {
+            throw error;
+        }
+        // get firstAvailableDelta from the error response, and set current sequence number to that
+        response = JSON.parse(error.getTelemetryProperties().response);
+        firstAvailableDelta = response.error.firstAvailableDelta;
+        lastSeq = firstAvailableDelta - 1;
+    }
+    // continue reading rest of the ops
+    const stream = deltaStorage.fetchMessages(lastSeq + 1, // inclusive left
+    undefined);
+    while (true) {
+        const result = await stream.read();
+        if (result.done) {
+            break;
+        }
+        requests++;
+        const messages = result.value;
+        // Empty buckets should never be returned
+        assert(messages.length !== 0, 0x1ba /* "should not return empty buckets" */);
+        // console.log(`Loaded ops at ${messages[0].sequenceNumber}`);
+        // This parsing of message contents happens in delta manager. But when we analyze messages
+        // for message stats, we skip that path. So parsing of json contents needs to happen here.
+        for (const message of messages) {
+            if (typeof message.contents === "string" &&
+                message.contents !== "" &&
+                message.type !== MessageType.ClientLeave) {
+                message.contents = JSON.parse(message.contents);
+            }
+        }
+        opsStorage += messages.length;
+        lastSeq = messages[messages.length - 1].sequenceNumber;
+        yield messages;
+    }
+    console.log(`\n${Math.floor((Date.now() - timeStart) / 1000)} seconds to retrieve ${opsStorage} ops in ${requests} requests`);
+    if (connectToWebSocket) {
+        let logMsg = "";
+        const client = {
+            mode: "write",
+            permission: [],
+            scopes: [ScopeType.DocRead, ScopeType.DocWrite, ScopeType.SummaryWrite],
+            details: {
+                capabilities: { interactive: true },
+            },
+            user: { id: "blah" },
+        };
+        console.log("Retrieving messages from web socket");
+        timeStart = Date.now();
+        const deltaStream = await documentService.connectToDeltaStream(client);
+        const initialMessages = deltaStream.initialMessages;
+        deltaStream.dispose();
+        console.log(`${Math.floor((Date.now() - timeStart) / 1000)} seconds to connect to web socket`);
+        if (initialMessages !== undefined) {
+            const lastSequenceNumber = lastSeq;
+            const filtered = initialMessages.filter((a) => a.sequenceNumber > lastSequenceNumber);
+            const sorted = filtered.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
+            lastSeq = sorted[sorted.length - 1].sequenceNumber;
+            logMsg = ` (${opsStorage} delta storage, ${initialMessages.length} initial ws messages, ${initialMessages.length - sorted.length} dup)`;
+            yield sorted;
+        }
+        console.log(`${lastSeq} total messages${logMsg}`);
+    }
+}
+async function* saveOps(gen, // AsyncGenerator<ISequencedDocumentMessage[]>,
+dir, files) {
+    // Split into 100K ops
+    const chunk = 100 * 1000;
+    let sequencedMessages = [];
+    // Figure out first file we want to write to
+    let index = 0;
+    let curr = 1;
+    if (files.length !== 0) {
+        index = files.length - 1;
+        const name = filenameFromIndex(index);
+        const fileContent = fs.readFileSync(`${dir}/messages${name}.json`, { encoding: "utf-8" });
+        const messages = JSON.parse(fileContent);
+        curr = messages[0].sequenceNumber;
+    }
+    while (true) {
+        const result = await gen.next();
+        if (files.length === 0) {
+            curr = firstAvailableDelta;
+        }
+        if (result.done !== true) {
+            let messages = result.value;
+            yield messages;
+            if (messages[messages.length - 1].sequenceNumber < curr) {
+                // Nothing interesting.
+                continue;
+            }
+            if (messages[0].sequenceNumber < curr) {
+                messages = messages.filter((msg) => msg.sequenceNumber >= curr);
+            }
+            sequencedMessages = sequencedMessages.concat(messages);
+            assert(sequencedMessages[0].sequenceNumber === curr, 0x1bb /* "Unexpected sequence number on first of messages to save" */);
+            assert(sequencedMessages[sequencedMessages.length - 1].sequenceNumber ===
+                curr + sequencedMessages.length - 1, 0x1bc /* "Unexpected sequence number on last of messages to save" */);
+        }
+        // Time to write it out?
+        while (sequencedMessages.length >= chunk ||
+            (result.done === true && sequencedMessages.length !== 0)) {
+            const name = filenameFromIndex(index);
+            const write = sequencedMessages.splice(0, chunk);
+            console.log(`writing messages${name}.json`);
+            fs.writeFileSync(`${dir}/messages${name}.json`, JSON.stringify(write, undefined, paramActualFormatting ? 0 : 2));
+            // increment curr by chunk
+            curr += chunk;
+            assert(sequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr, 0x1bd /* "Stopped writing at unexpected sequence number" */);
+            index++;
+        }
+        if (result.done === true) {
+            break;
+        }
+    }
+}
+export async function fluidFetchMessages(documentService, saveDir) {
+    const messageStats = dumpMessageStats || dumpMessages;
+    if (!messageStats && (saveDir === undefined || documentService === undefined)) {
+        return;
+    }
+    const files = saveDir === undefined
+        ? undefined
+        : fs
+            .readdirSync(saveDir)
+            .filter((file) => {
+            if (!file.startsWith("messages")) {
+                return false;
+            }
+            return true;
+        })
+            .sort((a, b) => a.localeCompare(b));
+    let generator = loadAllSequencedMessages(documentService, saveDir, files);
+    if (saveDir !== undefined && files !== undefined && documentService) {
+        generator = saveOps(generator, saveDir, files);
+    }
+    if (messageStats) {
+        return printMessageStats(generator, dumpMessageStats, dumpMessages, messageTypeFilter);
+    }
+    else {
+        let item;
+        for await (item of generator) {
+        }
+    }
+}
+//# sourceMappingURL=fluidFetchMessages.js.map

package/lib/fluidFetchMessages.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchMessages.js","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,MAAM,IAAI,CAAC;AAEpB,OAAO,EAAE,MAAM,EAAE,MAAM,qCAAqC,CAAC;AAE7D,OAAO,EAGN,WAAW,EACX,SAAS,GACT,MAAM,sCAAsC,CAAC;AAE9C,OAAO,EAAE,iBAAiB,EAAE,MAAM,2BAA2B,CAAC;AAC9D,OAAO,EACN,kBAAkB,EAClB,gBAAgB,EAChB,YAAY,EACZ,iBAAiB,EACjB,SAAS,EACT,qBAAqB,GACrB,MAAM,qBAAqB,CAAC;AAE7B,SAAS,iBAAiB,CAAC,KAAa;IACvC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,uBAAuB;AACpE,CAAC;AAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAC5B,KAAK,SAAS,CAAC,CAAC,wBAAwB,CACvC,eAAkC,EAClC,GAAY,EACZ,KAAgB;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,6EAA6E;IAC7E,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,mDAAmD;IACnD,IAAI,KAAK,KAAK,SAAS,EAAE;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI;gBACH,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE;oBAClE,QAAQ,EAAE,OAAO;iBACjB,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACtE,6BAA6B;gBAC7B,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,OAAO,GAAG,CAAC,CAAC;gBAC5D,MAAM,CACL,CAAC,cAAc,EACf,KAAK,CAAC,qEAAqE,CAC3E,CAAC;gBACF,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;gBACvD,MAAM,QAAQ,CAAC;aACf;YAAC,OAAO,CAAC,EAAE;gBACX,IAAI,cAAc,EAAE;oBACnB,IAAI,SAAS,EAAE;wBACd,2EAA2E;wBAC3E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;4BAClD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;4BACtC,EAAE,CAAC,UAAU,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,CAAC;yBAC7C;wBACD,MAAM;qBACN;oBACD,mDAAmD;oBACnD,OAAO,CAAC,KAAK,CACZ,wDAAwD;wBACvD,mFAAmF;wBACnF,6BAA6B,CAC9B,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACjB,OAAO;iBACP;qBAAM;oBACN,OAAO,CAAC,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;oBAChE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACjB,OAAO;iBACP;aACD;SACD;QACD,IAAI,OAAO,KAAK,CAAC,EAAE;YAClB,OAAO,CAAC,GAAG,CAAC,QAAQ,OAAO,uBAAuB,CAAC,CAAC;SACpD;KACD;IAED,IAAI,CAAC,eAAe,EAAE;QACrB,OAAO;KACP;IAED,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,qBAAqB,EAAE,CAAC;IAEnE,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,iDAAiD;IACjD,MAAM,UAAU,GAAG,YAAY,CAAC,aAAa,CAAC,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAExE,IAAI,UAAU,CAAC;IACf,IAAI,kBAAkB,CAAC;IACvB,IAAI,QAAQ,CAAC;IAEb,IAAI;QACH,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;KACxB;IAAC,OAAO,KAAU,EAAE;QACpB,UAAU,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,UAAU,CAAC;QACvD,kBAAkB,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,kBAAkB,CAAC;QACvE,iFAAiF;QACjF,IAAI,UAAU,KAAK,GAAG,IAAI,kBAAkB,KAAK,4BAA4B,EAAE;YAC9E,MAAM,KAAK,CAAC;SACZ;QACD,2FAA2F;QAC3F,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC/D,mBAAmB,GAAG,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC;QACzD,OAAO,GAAG,mBAAmB,GAAG,CAAC,CAAC;KAClC;IAED,mCAAmC;IACnC,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,CACxC,OAAO,GAAG,CAAC,EAAE,iBAAiB;IAC9B,SAAS,CACT,CAAC;IAEF,OAAO,IAAI,EAAE;QACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,IAAI,EAAE;YAChB,MAAM;SACN;QACD,QAAQ,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;QAE9B,yCAAyC;QACzC,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC7E,8DAA8D;QAE9D,0FAA0F;QAC1F,0FAA0F;QAC1F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC/B,IACC,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,OAAO,CAAC,QAAQ,KAAK,EAAE;gBACvB,OAAO,CAAC,IAAI,KAAK,WAAW,CAAC,WAAW,EACvC;gBACD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAChD;SACD;QAED,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC9B,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;QACvD,MAAM,QAAQ,CAAC;KACf;IAED,OAAO,CAAC,GAAG,CACV,KAAK,IAAI,CAAC,KAAK,CACd,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAC/B,wBAAwB,UAAU,WAAW,QAAQ,WAAW,CACjE,CAAC;IAEF,IAAI,kBAAkB,EAAE;QACvB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,MAAM,GAAY;YACvB,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,EAAE;YACd,MAAM,EAAE,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,QAAQ,EAAE,SAAS,CAAC,YAAY,CAAC;YACvE,OAAO,EAAE;gBACR,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;aACnC;YACD,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE;SACpB,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,eAAe,GAAG,WAAW,CAAC,eAAe,CAAC;QACpD,WAAW,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACV,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,mCAAmC,CACjF,CAAC;QAEF,IAAI,eAAe,KAAK,SAAS,EAAE;YAClC,MAAM,kBAAkB,GAAG,OAAO,CAAC;YACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,kBAAkB,CAAC,CAAC;YACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YAC5E,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;YACnD,MAAM,GAAG,KAAK,UAAU,mBACvB,eAAe,CAAC,MACjB,yBAAyB,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC;YACvE,MAAM,MAAM,CAAC;SACb;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,kBAAkB,MAAM,EAAE,CAAC,CAAC;KAClD;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,OAAO,CACtB,GAAG,EAAE,+CAA+C;AACpD,GAAW,EACX,KAAe;IAEf,sBAAsB;IACtB,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC;IAEzB,IAAI,iBAAiB,GAAgC,EAAE,CAAC;IAExD,4CAA4C;IAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAW,CAAC,CAAC;IACrB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACvB,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,EAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACtE,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;KAClC;IAED,OAAO,IAAI,EAAE;QACZ,MAAM,MAAM,GAAgD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,IAAI,GAAG,mBAAmB,CAAC;SAC3B;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;YACzB,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAC5B,MAAM,QAAQ,CAAC;YACf,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;gBACxD,uBAAuB;gBACvB,SAAS;aACT;YACD,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;gBACtC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;aAChE;YACD,iBAAiB,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvD,MAAM,CACL,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAC5C,KAAK,CAAC,+DAA+D,CACrE,CAAC;YACF,MAAM,CACL,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc;gBAC7D,IAAI,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,EACpC,KAAK,CAAC,8DAA8D,CACpE,CAAC;SACF;QAED,wBAAwB;QACxB,OACC,iBAAiB,CAAC,MAAM,IAAI,KAAK;YACjC,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,EACvD;YACD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;YAC5C,EAAE,CAAC,aAAa,CACf,GAAG,GAAG,YAAY,IAAI,OAAO,EAC7B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,qBAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,0BAA0B;YAC1B,IAAI,IAAI,KAAK,CAAC;YACd,MAAM,CACL,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAC9E,KAAK,CAAC,qDAAqD,CAC3D,CAAC;YACF,KAAK,EAAE,CAAC;SACR;QAED,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;YACzB,MAAM;SACN;KACD;AACF,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CAAC,eAAkC,EAAE,OAAgB;IAC5F,MAAM,YAAY,GAAG,gBAAgB,IAAI,YAAY,CAAC;IACtD,IAAI,CAAC,YAAY,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS,CAAC,EAAE;QAC9E,OAAO;KACP;IAED,MAAM,KAAK,GACV,OAAO,KAAK,SAAS;QACpB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,EAAE;aACD,WAAW,CAAC,OAAO,CAAC;aACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBACjC,OAAO,KAAK,CAAC;aACb;YACD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAExC,IAAI,SAAS,GAAG,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAE1E,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE;QACpE,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;KAC/C;IAED,IAAI,YAAY,EAAE;QACjB,OAAO,iBAAiB,CAAC,SAAS,EAAE,gBAAgB,EAAE,YAAY,EAAE,iBAAiB,CAAC,CAAC;KACvF;SAAM;QACN,IAAI,IAAI,CAAC;QACT,IAAI,KAAK,EAAE,IAAI,IAAI,SAAS,EAAE;SAC7B;KACD;AACF,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\n\nimport { assert } from \"@fluidframework/core-utils/internal\";\nimport { IDocumentService } from \"@fluidframework/driver-definitions/internal\";\nimport {\n\tIClient,\n\tISequencedDocumentMessage,\n\tMessageType,\n\tScopeType,\n} from \"@fluidframework/protocol-definitions\";\n\nimport { printMessageStats } from \"./fluidAnalyzeMessages.js\";\nimport {\n\tconnectToWebSocket,\n\tdumpMessageStats,\n\tdumpMessages,\n\tmessageTypeFilter,\n\toverWrite,\n\tparamActualFormatting,\n} from \"./fluidFetchArgs.js\";\n\nfunction filenameFromIndex(index: number): string {\n\treturn index === 0 ? \"\" : index.toString(); // support old tools...\n}\n\nlet firstAvailableDelta = 1;\nasync function* loadAllSequencedMessages(\n\tdocumentService?: IDocumentService,\n\tdir?: string,\n\tfiles?: string[],\n) {\n\tlet lastSeq = 0;\n\t// flag for mismatch between last sequence number read and new one to be read\n\tlet seqNumMismatch = false;\n\n\t// If we have local save, read ops from there first\n\tif (files !== undefined) {\n\t\tfor (let i = 0; i < files.length; i++) {\n\t\t\tconst file = filenameFromIndex(i);\n\t\t\ttry {\n\t\t\t\tconsole.log(`reading messages${file}.json`);\n\t\t\t\tconst fileContent = fs.readFileSync(`${dir}/messages${file}.json`, {\n\t\t\t\t\tencoding: \"utf-8\",\n\t\t\t\t});\n\t\t\t\tconst messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n\t\t\t\t// check if there is mismatch\n\t\t\t\tseqNumMismatch = messages[0].sequenceNumber !== lastSeq + 1;\n\t\t\t\tassert(\n\t\t\t\t\t!seqNumMismatch,\n\t\t\t\t\t0x1b9 /* \"Unexpected value for sequence number of first message in file\" */,\n\t\t\t\t);\n\t\t\t\tlastSeq = messages[messages.length - 1].sequenceNumber;\n\t\t\t\tyield messages;\n\t\t\t} catch (e) {\n\t\t\t\tif (seqNumMismatch) {\n\t\t\t\t\tif (overWrite) {\n\t\t\t\t\t\t// with overWrite option on, we will delete all exisintg message.json files\n\t\t\t\t\t\tfor (let index = 0; index < files.length; index++) {\n\t\t\t\t\t\t\tconst name = filenameFromIndex(index);\n\t\t\t\t\t\t\tfs.unlinkSync(`${dir}/messages${name}.json`);\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\t// prompt user to back up and delete existing files\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t\"There are deleted ops in the document being requested,\" +\n\t\t\t\t\t\t\t\" please back up the existing messages.json file and delete it from its directory.\" +\n\t\t\t\t\t\t\t\" Then try fetch tool again.\",\n\t\t\t\t\t);\n\t\t\t\t\tconsole.error(e);\n\t\t\t\t\treturn;\n\t\t\t\t} else {\n\t\t\t\t\tconsole.error(`Error reading / parsing messages from ${files}`);\n\t\t\t\t\tconsole.error(e);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif (lastSeq !== 0) {\n\t\t\tconsole.log(`Read ${lastSeq} ops from local cache`);\n\t\t}\n\t}\n\n\tif (!documentService) {\n\t\treturn;\n\t}\n\n\tconst deltaStorage = await documentService.connectToDeltaStorage();\n\n\tlet timeStart = Date.now();\n\tlet requests = 0;\n\tlet opsStorage = 0;\n\n\t// reading only 1 op to test if there is mismatch\n\tconst teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);\n\n\tlet statusCode;\n\tlet innerMostErrorCode;\n\tlet response;\n\n\ttry {\n\t\tawait teststream.read();\n\t} catch (error: any) {\n\t\tstatusCode = error.getTelemetryProperties().statusCode;\n\t\tinnerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;\n\t\t// if there is gap between ops, catch the error and check it is the error we need\n\t\tif (statusCode !== 410 || innerMostErrorCode !== \"fluidDeltaDataNotAvailable\") {\n\t\t\tthrow error;\n\t\t}\n\t\t// get firstAvailableDelta from the error response, and set current sequence number to that\n\t\tresponse = JSON.parse(error.getTelemetryProperties().response);\n\t\tfirstAvailableDelta = response.error.firstAvailableDelta;\n\t\tlastSeq = firstAvailableDelta - 1;\n\t}\n\n\t// continue reading rest of the ops\n\tconst stream = deltaStorage.fetchMessages(\n\t\tlastSeq + 1, // inclusive left\n\t\tundefined, // to\n\t);\n\n\twhile (true) {\n\t\tconst result = await stream.read();\n\t\tif (result.done) {\n\t\t\tbreak;\n\t\t}\n\t\trequests++;\n\t\tconst messages = result.value;\n\n\t\t// Empty buckets should never be returned\n\t\tassert(messages.length !== 0, 0x1ba /* \"should not return empty buckets\" */);\n\t\t// console.log(`Loaded ops at ${messages[0].sequenceNumber}`);\n\n\t\t// This parsing of message contents happens in delta manager. But when we analyze messages\n\t\t// for message stats, we skip that path. So parsing of json contents needs to happen here.\n\t\tfor (const message of messages) {\n\t\t\tif (\n\t\t\t\ttypeof message.contents === \"string\" &&\n\t\t\t\tmessage.contents !== \"\" &&\n\t\t\t\tmessage.type !== MessageType.ClientLeave\n\t\t\t) {\n\t\t\t\tmessage.contents = JSON.parse(message.contents);\n\t\t\t}\n\t\t}\n\n\t\topsStorage += messages.length;\n\t\tlastSeq = messages[messages.length - 1].sequenceNumber;\n\t\tyield messages;\n\t}\n\n\tconsole.log(\n\t\t`\\n${Math.floor(\n\t\t\t(Date.now() - timeStart) / 1000,\n\t\t)} seconds to retrieve ${opsStorage} ops in ${requests} requests`,\n\t);\n\n\tif (connectToWebSocket) {\n\t\tlet logMsg = \"\";\n\t\tconst client: IClient = {\n\t\t\tmode: \"write\",\n\t\t\tpermission: [],\n\t\t\tscopes: [ScopeType.DocRead, ScopeType.DocWrite, ScopeType.SummaryWrite],\n\t\t\tdetails: {\n\t\t\t\tcapabilities: { interactive: true },\n\t\t\t},\n\t\t\tuser: { id: \"blah\" },\n\t\t};\n\t\tconsole.log(\"Retrieving messages from web socket\");\n\t\ttimeStart = Date.now();\n\t\tconst deltaStream = await documentService.connectToDeltaStream(client);\n\t\tconst initialMessages = deltaStream.initialMessages;\n\t\tdeltaStream.dispose();\n\t\tconsole.log(\n\t\t\t`${Math.floor((Date.now() - timeStart) / 1000)} seconds to connect to web socket`,\n\t\t);\n\n\t\tif (initialMessages !== undefined) {\n\t\t\tconst lastSequenceNumber = lastSeq;\n\t\t\tconst filtered = initialMessages.filter((a) => a.sequenceNumber > lastSequenceNumber);\n\t\t\tconst sorted = filtered.sort((a, b) => a.sequenceNumber - b.sequenceNumber);\n\t\t\tlastSeq = sorted[sorted.length - 1].sequenceNumber;\n\t\t\tlogMsg = ` (${opsStorage} delta storage, ${\n\t\t\t\tinitialMessages.length\n\t\t\t} initial ws messages, ${initialMessages.length - sorted.length} dup)`;\n\t\t\tyield sorted;\n\t\t}\n\t\tconsole.log(`${lastSeq} total messages${logMsg}`);\n\t}\n}\n\nasync function* saveOps(\n\tgen, // AsyncGenerator<ISequencedDocumentMessage[]>,\n\tdir: string,\n\tfiles: string[],\n) {\n\t// Split into 100K ops\n\tconst chunk = 100 * 1000;\n\n\tlet sequencedMessages: ISequencedDocumentMessage[] = [];\n\n\t// Figure out first file we want to write to\n\tlet index = 0;\n\tlet curr: number = 1;\n\tif (files.length !== 0) {\n\t\tindex = files.length - 1;\n\t\tconst name = filenameFromIndex(index);\n\t\tconst fileContent = fs.readFileSync(`${dir}/messages${name}.json`, { encoding: \"utf-8\" });\n\t\tconst messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n\t\tcurr = messages[0].sequenceNumber;\n\t}\n\n\twhile (true) {\n\t\tconst result: IteratorResult<ISequencedDocumentMessage[]> = await gen.next();\n\t\tif (files.length === 0) {\n\t\t\tcurr = firstAvailableDelta;\n\t\t}\n\t\tif (result.done !== true) {\n\t\t\tlet messages = result.value;\n\t\t\tyield messages;\n\t\t\tif (messages[messages.length - 1].sequenceNumber < curr) {\n\t\t\t\t// Nothing interesting.\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (messages[0].sequenceNumber < curr) {\n\t\t\t\tmessages = messages.filter((msg) => msg.sequenceNumber >= curr);\n\t\t\t}\n\t\t\tsequencedMessages = sequencedMessages.concat(messages);\n\t\t\tassert(\n\t\t\t\tsequencedMessages[0].sequenceNumber === curr,\n\t\t\t\t0x1bb /* \"Unexpected sequence number on first of messages to save\" */,\n\t\t\t);\n\t\t\tassert(\n\t\t\t\tsequencedMessages[sequencedMessages.length - 1].sequenceNumber ===\n\t\t\t\t\tcurr + sequencedMessages.length - 1,\n\t\t\t\t0x1bc /* \"Unexpected sequence number on last of messages to save\" */,\n\t\t\t);\n\t\t}\n\n\t\t// Time to write it out?\n\t\twhile (\n\t\t\tsequencedMessages.length >= chunk ||\n\t\t\t(result.done === true && sequencedMessages.length !== 0)\n\t\t) {\n\t\t\tconst name = filenameFromIndex(index);\n\t\t\tconst write = sequencedMessages.splice(0, chunk);\n\t\t\tconsole.log(`writing messages${name}.json`);\n\t\t\tfs.writeFileSync(\n\t\t\t\t`${dir}/messages${name}.json`,\n\t\t\t\tJSON.stringify(write, undefined, paramActualFormatting ? 0 : 2),\n\t\t\t);\n\t\t\t// increment curr by chunk\n\t\t\tcurr += chunk;\n\t\t\tassert(\n\t\t\t\tsequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr,\n\t\t\t\t0x1bd /* \"Stopped writing at unexpected sequence number\" */,\n\t\t\t);\n\t\t\tindex++;\n\t\t}\n\n\t\tif (result.done === true) {\n\t\t\tbreak;\n\t\t}\n\t}\n}\n\nexport async function fluidFetchMessages(documentService?: IDocumentService, saveDir?: string) {\n\tconst messageStats = dumpMessageStats || dumpMessages;\n\tif (!messageStats && (saveDir === undefined || documentService === undefined)) {\n\t\treturn;\n\t}\n\n\tconst files =\n\t\tsaveDir === undefined\n\t\t\t? undefined\n\t\t\t: fs\n\t\t\t\t\t.readdirSync(saveDir)\n\t\t\t\t\t.filter((file) => {\n\t\t\t\t\t\tif (!file.startsWith(\"messages\")) {\n\t\t\t\t\t\t\treturn false;\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn true;\n\t\t\t\t\t})\n\t\t\t\t\t.sort((a, b) => a.localeCompare(b));\n\n\tlet generator = loadAllSequencedMessages(documentService, saveDir, files);\n\n\tif (saveDir !== undefined && files !== undefined && documentService) {\n\t\tgenerator = saveOps(generator, saveDir, files);\n\t}\n\n\tif (messageStats) {\n\t\treturn printMessageStats(generator, dumpMessageStats, dumpMessages, messageTypeFilter);\n\t} else {\n\t\tlet item;\n\t\tfor await (item of generator) {\n\t\t}\n\t}\n}\n"]}

package/lib/fluidFetchSharePoint.d.ts

@@ -0,0 +1,9 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { IClientConfig, IOdspAuthRequestInfo, IOdspDriveItem } from "@fluidframework/odsp-doclib-utils/internal";
+export declare function resolveWrapper<T>(callback: (authRequestInfo: IOdspAuthRequestInfo) => Promise<T>, server: string, clientConfig: IClientConfig, forceTokenReauth?: boolean, forToken?: boolean): Promise<T>;
+export declare function getSharepointFiles(server: string, serverRelativePath: string, recurse: boolean): Promise<IOdspDriveItem[]>;
+export declare function getSingleSharePointFile(server: string, drive: string, item: string): Promise<IOdspDriveItem>;
+//# sourceMappingURL=fluidFetchSharePoint.d.ts.map

package/lib/fluidFetchSharePoint.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSharePoint.d.ts","sourceRoot":"","sources":["../src/fluidFetchSharePoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACN,aAAa,EACb,oBAAoB,EACpB,cAAc,EAKd,MAAM,4CAA4C,CAAC;AAWpD,wBAAsB,cAAc,CAAC,CAAC,EACrC,QAAQ,EAAE,CAAC,eAAe,EAAE,oBAAoB,KAAK,OAAO,CAAC,CAAC,CAAC,EAC/D,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,aAAa,EAC3B,gBAAgB,UAAQ,EACxB,QAAQ,UAAQ,GACd,OAAO,CAAC,CAAC,CAAC,CAoCZ;AA6BD,wBAAsB,kBAAkB,CACvC,MAAM,EAAE,MAAM,EACd,kBAAkB,EAAE,MAAM,EAC1B,OAAO,EAAE,OAAO,6BAsChB;AAED,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,2BASxF"}

package/lib/fluidFetchSharePoint.js

@@ -0,0 +1,100 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import child_process from "child_process";
+import { DriverErrorTypes } from "@fluidframework/driver-definitions";
+import { getChildrenByDriveItem, getDriveItemByServerRelativePath, getDriveItemFromDriveAndItem, getOdspRefreshTokenFn, } from "@fluidframework/odsp-doclib-utils/internal";
+import { OdspTokenManager, getMicrosoftConfiguration, odspTokensCache, } from "@fluidframework/tool-utils/internal";
+import { getForceTokenReauth } from "./fluidFetchArgs.js";
+export async function resolveWrapper(callback, server, clientConfig, forceTokenReauth = false, forToken = false) {
+    try {
+        const odspTokenManager = new OdspTokenManager(odspTokensCache);
+        const tokenConfig = {
+            type: "browserLogin",
+            navigator: fluidFetchWebNavigator,
+        };
+        const tokens = await odspTokenManager.getOdspTokens(server, clientConfig, tokenConfig, undefined /* forceRefresh */, forceTokenReauth || getForceTokenReauth());
+        const result = await callback({
+            accessToken: tokens.accessToken,
+            refreshTokenFn: getOdspRefreshTokenFn(server, clientConfig, tokens),
+        });
+        // If this is used for getting a token, then refresh the cache with new token.
+        if (forToken) {
+            const key = { isPush: false, userOrServer: server };
+            await odspTokenManager.updateTokensCache(key, {
+                accessToken: result,
+                refreshToken: tokens.refreshToken,
+            });
+            return result;
+        }
+        return result;
+    }
+    catch (e) {
+        if (e.errorType === DriverErrorTypes.authorizationError && !forceTokenReauth) {
+            // Re-auth
+            return resolveWrapper(callback, server, clientConfig, true, forToken);
+        }
+        throw e;
+    }
+}
+async function resolveDriveItemByServerRelativePath(server, serverRelativePath, clientConfig) {
+    return resolveWrapper(
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    (authRequestInfo) => getDriveItemByServerRelativePath(server, serverRelativePath, authRequestInfo, false), server, clientConfig);
+}
+async function resolveChildrenByDriveItem(server, folderDriveItem, clientConfig) {
+    return resolveWrapper(
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    (authRequestInfo) => getChildrenByDriveItem(folderDriveItem, server, authRequestInfo), server, clientConfig);
+}
+export async function getSharepointFiles(server, serverRelativePath, recurse) {
+    const clientConfig = getMicrosoftConfiguration();
+    const fileInfo = await resolveDriveItemByServerRelativePath(server, serverRelativePath, clientConfig);
+    console.log(fileInfo);
+    const pendingFolder = [];
+    const files = [];
+    if (fileInfo.isFolder) {
+        pendingFolder.push({ path: serverRelativePath, folder: fileInfo });
+    }
+    else {
+        files.push(fileInfo);
+    }
+    // eslint-disable-next-line no-constant-condition
+    while (true) {
+        const folderInfo = pendingFolder.shift();
+        if (!folderInfo) {
+            break;
+        }
+        const { path, folder } = folderInfo;
+        const children = await resolveChildrenByDriveItem(server, folder, clientConfig);
+        for (const child of children) {
+            const childPath = `${path}/${child.name}`;
+            if (child.isFolder) {
+                if (recurse) {
+                    pendingFolder.push({ path: childPath, folder: child });
+                }
+            }
+            else {
+                files.push(child);
+            }
+        }
+    }
+    return files;
+}
+export async function getSingleSharePointFile(server, drive, item) {
+    const clientConfig = getMicrosoftConfiguration();
+    return resolveWrapper(
+    // eslint-disable-next-line @typescript-eslint/promise-function-async
+    (authRequestInfo) => getDriveItemFromDriveAndItem(server, drive, item, authRequestInfo), server, clientConfig);
+}
+const fluidFetchWebNavigator = (url) => {
+    let message = "Please open browser and navigate to this URL:";
+    if (process.platform === "win32") {
+        child_process.exec(`start "fluid-fetch" /B "${url}"`);
+        message =
+            "Opening browser to get authorization code.  If that doesn't open, please go to this URL manually";
+    }
+    console.log(`${message}\n  ${url}`);
+};
+//# sourceMappingURL=fluidFetchSharePoint.js.map

package/lib/fluidFetchSharePoint.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSharePoint.js","sourceRoot":"","sources":["../src/fluidFetchSharePoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,aAAa,MAAM,eAAe,CAAC;AAE1C,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AACtE,OAAO,EAIN,sBAAsB,EACtB,gCAAgC,EAChC,4BAA4B,EAC5B,qBAAqB,GACrB,MAAM,4CAA4C,CAAC;AACpD,OAAO,EAGN,gBAAgB,EAChB,yBAAyB,EACzB,eAAe,GACf,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAAE,mBAAmB,EAAE,MAAM,qBAAqB,CAAC;AAE1D,MAAM,CAAC,KAAK,UAAU,cAAc,CACnC,QAA+D,EAC/D,MAAc,EACd,YAA2B,EAC3B,gBAAgB,GAAG,KAAK,EACxB,QAAQ,GAAG,KAAK;IAEhB,IAAI;QACH,MAAM,gBAAgB,GAAG,IAAI,gBAAgB,CAAC,eAAe,CAAC,CAAC;QAC/D,MAAM,WAAW,GAAoB;YACpC,IAAI,EAAE,cAAc;YACpB,SAAS,EAAE,sBAAsB;SACjC,CAAC;QACF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,aAAa,CAClD,MAAM,EACN,YAAY,EACZ,WAAW,EACX,SAAS,CAAC,kBAAkB,EAC5B,gBAAgB,IAAI,mBAAmB,EAAE,CACzC,CAAC;QAEF,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC;YAC7B,WAAW,EAAE,MAAM,CAAC,WAAW;YAC/B,cAAc,EAAE,qBAAqB,CAAC,MAAM,EAAE,YAAY,EAAE,MAAM,CAAC;SACnE,CAAC,CAAC;QACH,8EAA8E;QAC9E,IAAI,QAAQ,EAAE;YACb,MAAM,GAAG,GAA8B,EAAE,MAAM,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,CAAC;YAC/E,MAAM,gBAAgB,CAAC,iBAAiB,CAAC,GAAG,EAAE;gBAC7C,WAAW,EAAE,MAAuB;gBACpC,YAAY,EAAE,MAAM,CAAC,YAAY;aACjC,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;SACd;QACD,OAAO,MAAM,CAAC;KACd;IAAC,OAAO,CAAM,EAAE;QAChB,IAAI,CAAC,CAAC,SAAS,KAAK,gBAAgB,CAAC,kBAAkB,IAAI,CAAC,gBAAgB,EAAE;YAC7E,UAAU;YACV,OAAO,cAAc,CAAI,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,IAAI,EAAE,QAAQ,CAAC,CAAC;SACzE;QACD,MAAM,CAAC,CAAC;KACR;AACF,CAAC;AAED,KAAK,UAAU,oCAAoC,CAClD,MAAc,EACd,kBAA0B,EAC1B,YAA2B;IAE3B,OAAO,cAAc;IACpB,qEAAqE;IACrE,CAAC,eAAe,EAAE,EAAE,CACnB,gCAAgC,CAAC,MAAM,EAAE,kBAAkB,EAAE,eAAe,EAAE,KAAK,CAAC,EACrF,MAAM,EACN,YAAY,CACZ,CAAC;AACH,CAAC;AAED,KAAK,UAAU,0BAA0B,CACxC,MAAc,EACd,eAA+B,EAC/B,YAA2B;IAE3B,OAAO,cAAc;IACpB,qEAAqE;IACrE,CAAC,eAAe,EAAE,EAAE,CAAC,sBAAsB,CAAC,eAAe,EAAE,MAAM,EAAE,eAAe,CAAC,EACrF,MAAM,EACN,YAAY,CACZ,CAAC;AACH,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,kBAAkB,CACvC,MAAc,EACd,kBAA0B,EAC1B,OAAgB;IAEhB,MAAM,YAAY,GAAG,yBAAyB,EAAE,CAAC;IAEjD,MAAM,QAAQ,GAAG,MAAM,oCAAoC,CAC1D,MAAM,EACN,kBAAkB,EAClB,YAAY,CACZ,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACtB,MAAM,aAAa,GAA+C,EAAE,CAAC;IACrE,MAAM,KAAK,GAAqB,EAAE,CAAC;IACnC,IAAI,QAAQ,CAAC,QAAQ,EAAE;QACtB,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,kBAAkB,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;KACnE;SAAM;QACN,KAAK,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KACrB;IAED,iDAAiD;IACjD,OAAO,IAAI,EAAE;QACZ,MAAM,UAAU,GAAG,aAAa,CAAC,KAAK,EAAE,CAAC;QACzC,IAAI,CAAC,UAAU,EAAE;YAChB,MAAM;SACN;QACD,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,UAAU,CAAC;QACpC,MAAM,QAAQ,GAAG,MAAM,0BAA0B,CAAC,MAAM,EAAE,MAAM,EAAE,YAAY,CAAC,CAAC;QAChF,KAAK,MAAM,KAAK,IAAI,QAAQ,EAAE;YAC7B,MAAM,SAAS,GAAG,GAAG,IAAI,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YAC1C,IAAI,KAAK,CAAC,QAAQ,EAAE;gBACnB,IAAI,OAAO,EAAE;oBACZ,aAAa,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC,CAAC;iBACvD;aACD;iBAAM;gBACN,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;aAClB;SACD;KACD;IACD,OAAO,KAAK,CAAC;AACd,CAAC;AAED,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAAC,MAAc,EAAE,KAAa,EAAE,IAAY;IACxF,MAAM,YAAY,GAAG,yBAAyB,EAAE,CAAC;IAEjD,OAAO,cAAc;IACpB,qEAAqE;IACrE,CAAC,eAAe,EAAE,EAAE,CAAC,4BAA4B,CAAC,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,eAAe,CAAC,EACvF,MAAM,EACN,YAAY,CACZ,CAAC;AACH,CAAC;AAED,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAE,EAAE;IAC9C,IAAI,OAAO,GAAG,+CAA+C,CAAC;IAC9D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QACjC,aAAa,CAAC,IAAI,CAAC,2BAA2B,GAAG,GAAG,CAAC,CAAC;QACtD,OAAO;YACN,kGAAkG,CAAC;KACpG;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,OAAO,GAAG,EAAE,CAAC,CAAC;AACrC,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport child_process from \"child_process\";\n\nimport { DriverErrorTypes } from \"@fluidframework/driver-definitions\";\nimport {\n\tIClientConfig,\n\tIOdspAuthRequestInfo,\n\tIOdspDriveItem,\n\tgetChildrenByDriveItem,\n\tgetDriveItemByServerRelativePath,\n\tgetDriveItemFromDriveAndItem,\n\tgetOdspRefreshTokenFn,\n} from \"@fluidframework/odsp-doclib-utils/internal\";\nimport {\n\tIOdspTokenManagerCacheKey,\n\tOdspTokenConfig,\n\tOdspTokenManager,\n\tgetMicrosoftConfiguration,\n\todspTokensCache,\n} from \"@fluidframework/tool-utils/internal\";\n\nimport { getForceTokenReauth } from \"./fluidFetchArgs.js\";\n\nexport async function resolveWrapper<T>(\n\tcallback: (authRequestInfo: IOdspAuthRequestInfo) => Promise<T>,\n\tserver: string,\n\tclientConfig: IClientConfig,\n\tforceTokenReauth = false,\n\tforToken = false,\n): Promise<T> {\n\ttry {\n\t\tconst odspTokenManager = new OdspTokenManager(odspTokensCache);\n\t\tconst tokenConfig: OdspTokenConfig = {\n\t\t\ttype: \"browserLogin\",\n\t\t\tnavigator: fluidFetchWebNavigator,\n\t\t};\n\t\tconst tokens = await odspTokenManager.getOdspTokens(\n\t\t\tserver,\n\t\t\tclientConfig,\n\t\t\ttokenConfig,\n\t\t\tundefined /* forceRefresh */,\n\t\t\tforceTokenReauth || getForceTokenReauth(),\n\t\t);\n\n\t\tconst result = await callback({\n\t\t\taccessToken: tokens.accessToken,\n\t\t\trefreshTokenFn: getOdspRefreshTokenFn(server, clientConfig, tokens),\n\t\t});\n\t\t// If this is used for getting a token, then refresh the cache with new token.\n\t\tif (forToken) {\n\t\t\tconst key: IOdspTokenManagerCacheKey = { isPush: false, userOrServer: server };\n\t\t\tawait odspTokenManager.updateTokensCache(key, {\n\t\t\t\taccessToken: result as any as string,\n\t\t\t\trefreshToken: tokens.refreshToken,\n\t\t\t});\n\t\t\treturn result;\n\t\t}\n\t\treturn result;\n\t} catch (e: any) {\n\t\tif (e.errorType === DriverErrorTypes.authorizationError && !forceTokenReauth) {\n\t\t\t// Re-auth\n\t\t\treturn resolveWrapper<T>(callback, server, clientConfig, true, forToken);\n\t\t}\n\t\tthrow e;\n\t}\n}\n\nasync function resolveDriveItemByServerRelativePath(\n\tserver: string,\n\tserverRelativePath: string,\n\tclientConfig: IClientConfig,\n) {\n\treturn resolveWrapper<IOdspDriveItem>(\n\t\t// eslint-disable-next-line @typescript-eslint/promise-function-async\n\t\t(authRequestInfo) =>\n\t\t\tgetDriveItemByServerRelativePath(server, serverRelativePath, authRequestInfo, false),\n\t\tserver,\n\t\tclientConfig,\n\t);\n}\n\nasync function resolveChildrenByDriveItem(\n\tserver: string,\n\tfolderDriveItem: IOdspDriveItem,\n\tclientConfig: IClientConfig,\n) {\n\treturn resolveWrapper<IOdspDriveItem[]>(\n\t\t// eslint-disable-next-line @typescript-eslint/promise-function-async\n\t\t(authRequestInfo) => getChildrenByDriveItem(folderDriveItem, server, authRequestInfo),\n\t\tserver,\n\t\tclientConfig,\n\t);\n}\n\nexport async function getSharepointFiles(\n\tserver: string,\n\tserverRelativePath: string,\n\trecurse: boolean,\n) {\n\tconst clientConfig = getMicrosoftConfiguration();\n\n\tconst fileInfo = await resolveDriveItemByServerRelativePath(\n\t\tserver,\n\t\tserverRelativePath,\n\t\tclientConfig,\n\t);\n\tconsole.log(fileInfo);\n\tconst pendingFolder: { path: string; folder: IOdspDriveItem }[] = [];\n\tconst files: IOdspDriveItem[] = [];\n\tif (fileInfo.isFolder) {\n\t\tpendingFolder.push({ path: serverRelativePath, folder: fileInfo });\n\t} else {\n\t\tfiles.push(fileInfo);\n\t}\n\n\t// eslint-disable-next-line no-constant-condition\n\twhile (true) {\n\t\tconst folderInfo = pendingFolder.shift();\n\t\tif (!folderInfo) {\n\t\t\tbreak;\n\t\t}\n\t\tconst { path, folder } = folderInfo;\n\t\tconst children = await resolveChildrenByDriveItem(server, folder, clientConfig);\n\t\tfor (const child of children) {\n\t\t\tconst childPath = `${path}/${child.name}`;\n\t\t\tif (child.isFolder) {\n\t\t\t\tif (recurse) {\n\t\t\t\t\tpendingFolder.push({ path: childPath, folder: child });\n\t\t\t\t}\n\t\t\t} else {\n\t\t\t\tfiles.push(child);\n\t\t\t}\n\t\t}\n\t}\n\treturn files;\n}\n\nexport async function getSingleSharePointFile(server: string, drive: string, item: string) {\n\tconst clientConfig = getMicrosoftConfiguration();\n\n\treturn resolveWrapper<IOdspDriveItem>(\n\t\t// eslint-disable-next-line @typescript-eslint/promise-function-async\n\t\t(authRequestInfo) => getDriveItemFromDriveAndItem(server, drive, item, authRequestInfo),\n\t\tserver,\n\t\tclientConfig,\n\t);\n}\n\nconst fluidFetchWebNavigator = (url: string) => {\n\tlet message = \"Please open browser and navigate to this URL:\";\n\tif (process.platform === \"win32\") {\n\t\tchild_process.exec(`start \"fluid-fetch\" /B \"${url}\"`);\n\t\tmessage =\n\t\t\t\"Opening browser to get authorization code.  If that doesn't open, please go to this URL manually\";\n\t}\n\tconsole.log(`${message}\\n  ${url}`);\n};\n"]}

package/lib/fluidFetchSnapshot.d.ts

@@ -0,0 +1,7 @@
+/*!
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
+ * Licensed under the MIT License.
+ */
+import { IDocumentService } from "@fluidframework/driver-definitions/internal";
+export declare function fluidFetchSnapshot(documentService?: IDocumentService, saveDir?: string): Promise<void>;
+//# sourceMappingURL=fluidFetchSnapshot.d.ts.map

package/lib/fluidFetchSnapshot.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAMH,OAAO,EACN,gBAAgB,EAEhB,MAAM,6CAA6C,CAAC;AAqQrD,wBAAsB,kBAAkB,CAAC,eAAe,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,MAAM,iBAsG5F"}