@fluid-tools/fetch-tool 0.58.2002 → 0.59.1000
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/fluidFetchInit.d.ts.map +1 -1
- package/dist/fluidFetchInit.js +6 -3
- package/dist/fluidFetchInit.js.map +1 -1
- package/dist/fluidFetchSnapshot.d.ts.map +1 -1
- package/dist/fluidFetchSnapshot.js +9 -28
- package/dist/fluidFetchSnapshot.js.map +1 -1
- package/package.json +19 -19
- package/src/fluidFetchInit.ts +15 -3
- package/src/fluidFetchSnapshot.ts +10 -38
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchInit.d.ts","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsBH,eAAO,IAAI,gBAAgB,EAAE,MAAW,CAAC;AACzC,eAAO,IAAI,cAAc,EAAE,GAAG,CAAC;AAE/B,eAAO,MAAM,sBAAsB,QAAS,MAAM,SAOjD,CAAC;
|
|
1
|
+
{"version":3,"file":"fluidFetchInit.d.ts","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAsBH,eAAO,IAAI,gBAAgB,EAAE,MAAW,CAAC;AACzC,eAAO,IAAI,cAAc,EAAE,GAAG,CAAC;AAE/B,eAAO,MAAM,sBAAsB,QAAS,MAAM,SAOjD,CAAC;AA0HF,wBAAsB,cAAc,CAAC,MAAM,EAAE,MAAM,sFAsBlD"}
|
package/dist/fluidFetchInit.js
CHANGED
|
@@ -80,7 +80,8 @@ async function initializeODSPCore(odspResolvedUrl, server, clientConfig) {
|
|
|
80
80
|
opsBatchSize: 20000,
|
|
81
81
|
concurrentOpsBatches: 4,
|
|
82
82
|
});
|
|
83
|
-
return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl
|
|
83
|
+
return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl, undefined, // logger
|
|
84
|
+
false);
|
|
84
85
|
}
|
|
85
86
|
async function initializeR11s(server, pathname, r11sResolvedUrl) {
|
|
86
87
|
const path = pathname.split("/");
|
|
@@ -107,7 +108,8 @@ async function initializeR11s(server, pathname, r11sResolvedUrl) {
|
|
|
107
108
|
console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
|
|
108
109
|
const tokenProvider = new r11s.DefaultTokenProvider(fluidFetchArgs_1.paramJWT);
|
|
109
110
|
const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
|
|
110
|
-
return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl
|
|
111
|
+
return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl, undefined, // logger
|
|
112
|
+
false);
|
|
111
113
|
}
|
|
112
114
|
async function initializeAzure(resolvedUrl, tenantId) {
|
|
113
115
|
exports.connectionInfo = {
|
|
@@ -119,7 +121,8 @@ async function initializeAzure(resolvedUrl, tenantId) {
|
|
|
119
121
|
const user = test_client_utils_1.generateTestUser();
|
|
120
122
|
const tokenProvider = new test_client_utils_1.InsecureTokenProvider(fluidFetchArgs_1.paramAzureKey, user);
|
|
121
123
|
const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
|
|
122
|
-
return r11sDocumentServiceFactory.createDocumentService(resolvedUrl
|
|
124
|
+
return r11sDocumentServiceFactory.createDocumentService(resolvedUrl, undefined, // logger
|
|
125
|
+
false);
|
|
123
126
|
}
|
|
124
127
|
async function resolveUrl(url) {
|
|
125
128
|
const resolversList = [
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchInit.js","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,6BAA0B;AAC1B,kEAA0C;AAC1C,yFAAsF;AACtF,yEAG2C;AAE3C,+DAAuE;AACvE,sFAAiF;AAEjF,kEAAoD;AAEpD,uEAAmE;AACnE,2EAA6D;AAC7D,yFAAqF;AACrF,2DAAuE;AACvE,qDAA0E;AAC1E,iEAAwD;AAE7C,QAAA,gBAAgB,GAAW,EAAE,CAAC;AAGlC,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAE,EAAE;IAClD,IAAI,OAAO,GAAG,+CAA+C,CAAC;IAC9D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAC9B,uBAAa,CAAC,IAAI,CAAC,2BAA2B,GAAG,GAAG,CAAC,CAAC;QACtD,OAAO,GAAG,kGAAkG,CAAC;KAChH;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,OAAO,GAAG,EAAE,CAAC,CAAC;AACxC,CAAC,CAAC;AAPW,QAAA,sBAAsB,0BAOjC;AAEF,KAAK,UAAU,kBAAkB,CAC7B,eAAiC,EACjC,MAAc,EACd,YAA2B;IAE3B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;IAE5C,sBAAc,GAAG;QACb,MAAM;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACf,CAAC;IAEF,IAAI,8BAAa,EAAE;QACf,OAAO;KACV;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,CAAC;YACJ,MAAM;YACN,OAAO;YACP,MAAM;YACN,KAAK,EAAE,CAAC,CAAC;IAEjB,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAsC,EAAE,EAAE;QACzE,OAAO,qCAAc,CACjB,KAAK,EAAE,eAAqC,EAAE,EAAE;YAC5C,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,eAAe,CAAC,cAAc,EAAE;gBACrF,OAAO,eAAe,CAAC,cAAc,EAAE,CAAC;aAC3C;YACD,OAAO,eAAe,CAAC,WAAW,CAAC;QACvC,CAAC,EACD,MAAM,EACN,YAAY,EACZ,SAAS,EACT,IAAI,CACP,CAAC;IACN,CAAC,CAAC;IACF,qEAAqE;IACrE,MAAM,qBAAqB,GAAG,CAAC,QAAuC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,0BAA0B,CAClE,mBAAmB,EACnB,qBAAqB,EACrB,SAAS,EACT;QACI,YAAY,EAAE,KAAK;QACnB,oBAAoB,EAAE,CAAC;KAC1B,CAAC,CAAC;IACP,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC7E,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,eAAkC;IAC9F,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3C,QAAQ,GAAG,OAAO,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;SAAM;QACH,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;IAED,+CAA+C;IAC/C,wBAAgB,GAAG,UAAU,CAAC;IAE9B,sBAAc,GAAG;QACb,MAAM;QACN,QAAQ;QACR,EAAE,EAAE,UAAU;KACjB,CAAC;IAEF,IAAI,8BAAa,EAAE;QACf,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,OAAO,UAAU,EAAE,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,oBAAoB,CAAC,yBAAQ,CAAC,CAAC;IAC9D,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC7E,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,WAA8B,EAAE,QAAgB;IAC3E,sBAAc,GAAG;QACb,MAAM,EAAE,WAAW,CAAC,SAAS,CAAC,UAAU;QACxC,QAAQ;QACR,EAAE,EAAE,WAAW,CAAC,EAAE;KACrB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,6CAA6C,QAAQ,OAAO,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,oCAAgB,EAAE,CAAC;IAChC,MAAM,aAAa,GAAG,IAAI,yCAAqB,CAAC,8BAAa,EAAE,IAAI,CAAC,CAAC;IACrE,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,WAAW,CAAC,CAAC;AACzE,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,GAAW;IACjC,MAAM,aAAa,GAAmB;QAClC,IAAI,kCAAe,EAAE;QACrB,IAAI,mDAAuB,EAAE;QAC7B,qEAAqE;QACrE,IAAI,oDAAwB,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAQ,CAAC,EAAE,EAAE,CAAC;QAC5E,IAAI,mCAAgB,EAAE;KACzB,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,sCAAuB,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACvE,OAAO,QAAQ,CAAC;AACpB,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IAC/C,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CAAsB,CAAC;IAClE,IAAI,CAAC,WAAW,EAAE;QACd,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,MAAM,EAAE,CAAC,CAAC,CAAC;KAC7D;IACD,MAAM,QAAQ,GAAG,IAAI,SAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACnD,IAAI,QAAQ,KAAK,aAAa,EAAE;QAC5B,MAAM,eAAe,GAAG,WAA+B,CAAC;QACxD,OAAO,kBAAkB,CAAC,eAAe,EAAE,IAAI,SAAG,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,sCAAyB,EAAE,CAAC,CAAC;KAClH;SAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE;QAC9B,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;KAC5D;SAAM,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE;QACzD,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,QAAQ,KAAK,IAAI,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;SACzD;QACD,OAAO,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;KACjD;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC,CAAC;AAC9E,CAAC;AAtBD,wCAsBC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { URL } from \"url\";\nimport child_process from \"child_process\";\nimport { AzureUrlResolver } from \"@fluidframework/azure-client/dist/AzureUrlResolver\";\nimport {\n generateTestUser,\n InsecureTokenProvider,\n} from \"@fluidframework/test-client-utils\";\nimport { IFluidResolvedUrl, IResolvedUrl, IUrlResolver } from \"@fluidframework/driver-definitions\";\nimport { configurableUrlResolver } from \"@fluidframework/driver-utils\";\nimport { FluidAppOdspUrlResolver } from \"@fluid-tools/fluidapp-odsp-urlresolver\";\nimport { IClientConfig, IOdspAuthRequestInfo } from \"@fluidframework/odsp-doclib-utils\";\nimport * as odsp from \"@fluidframework/odsp-driver\";\nimport { IOdspResolvedUrl, OdspResourceTokenFetchOptions } from \"@fluidframework/odsp-driver-definitions\";\nimport { OdspUrlResolver } from \"@fluidframework/odsp-urlresolver\";\nimport * as r11s from \"@fluidframework/routerlicious-driver\";\nimport { RouterliciousUrlResolver } from \"@fluidframework/routerlicious-urlresolver\";\nimport { getMicrosoftConfiguration } from \"@fluidframework/tool-utils\";\nimport { localDataOnly, paramAzureKey, paramJWT } from \"./fluidFetchArgs\";\nimport { resolveWrapper } from \"./fluidFetchSharePoint\";\n\nexport let latestVersionsId: string = \"\";\nexport let connectionInfo: any;\n\nexport const fluidFetchWebNavigator = (url: string) => {\n let message = \"Please open browser and navigate to this URL:\";\n if (process.platform === \"win32\") {\n child_process.exec(`start \"fluid-fetch\" /B \"${url}\"`);\n message = \"Opening browser to get authorization code. If that doesn't open, please go to this URL manually\";\n }\n console.log(`${message}\\n ${url}`);\n};\n\nasync function initializeODSPCore(\n odspResolvedUrl: IOdspResolvedUrl,\n server: string,\n clientConfig: IClientConfig,\n) {\n const { driveId, itemId } = odspResolvedUrl;\n\n connectionInfo = {\n server,\n drive: driveId,\n item: itemId,\n };\n\n if (localDataOnly) {\n return;\n }\n\n const docId = await odsp.getHashedDocumentId(driveId, itemId);\n\n console.log(`Connecting to ODSP:\n server: ${server}\n drive: ${driveId}\n item: ${itemId}\n docId: ${docId}`);\n\n const getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {\n return resolveWrapper(\n async (authRequestInfo: IOdspAuthRequestInfo) => {\n if ((options.refresh || !authRequestInfo.accessToken) && authRequestInfo.refreshTokenFn) {\n return authRequestInfo.refreshTokenFn();\n }\n return authRequestInfo.accessToken;\n },\n server,\n clientConfig,\n undefined,\n true,\n );\n };\n // eslint-disable-next-line @typescript-eslint/promise-function-async\n const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) => Promise.resolve(\"\");\n const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(\n getStorageTokenStub,\n getWebsocketTokenStub,\n undefined,\n {\n opsBatchSize: 20000,\n concurrentOpsBatches: 4,\n });\n return odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);\n}\n\nasync function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IFluidResolvedUrl) {\n const path = pathname.split(\"/\");\n let tenantId: string;\n let documentId: string;\n if (server === \"localhost\" && path.length < 4) {\n tenantId = \"fluid\";\n documentId = path[2];\n } else {\n tenantId = path[2];\n documentId = path[3];\n }\n\n // Latest version id is the documentId for r11s\n latestVersionsId = documentId;\n\n connectionInfo = {\n server,\n tenantId,\n id: documentId,\n };\n\n if (localDataOnly) {\n return;\n }\n\n console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);\n const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);\n const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n return r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);\n}\n\nasync function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string) {\n connectionInfo = {\n server: resolvedUrl.endpoints.ordererUrl,\n tenantId,\n id: resolvedUrl.id,\n };\n console.log(`Connecting to Azure Fluid Relay: tenantId=${tenantId} id:${resolvedUrl.id}`);\n const user = generateTestUser();\n const tokenProvider = new InsecureTokenProvider(paramAzureKey, user);\n const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n return r11sDocumentServiceFactory.createDocumentService(resolvedUrl);\n}\n\nasync function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {\n const resolversList: IUrlResolver[] = [\n new OdspUrlResolver(),\n new FluidAppOdspUrlResolver(),\n // eslint-disable-next-line @typescript-eslint/promise-function-async\n new RouterliciousUrlResolver(undefined, () => Promise.resolve(paramJWT), \"\"),\n new AzureUrlResolver(),\n ];\n const resolved = await configurableUrlResolver(resolversList, { url });\n return resolved;\n}\n\nexport async function fluidFetchInit(urlStr: string) {\n const resolvedUrl = await resolveUrl(urlStr) as IFluidResolvedUrl;\n if (!resolvedUrl) {\n return Promise.reject(new Error(`Unknown URL ${urlStr}`));\n }\n const protocol = new URL(resolvedUrl.url).protocol;\n if (protocol === \"fluid-odsp:\") {\n const odspResolvedUrl = resolvedUrl as IOdspResolvedUrl;\n return initializeODSPCore(odspResolvedUrl, new URL(odspResolvedUrl.siteUrl).host, getMicrosoftConfiguration());\n } else if (protocol === \"fluid:\") {\n const url = new URL(urlStr);\n const server = url.hostname.toLowerCase();\n return initializeR11s(server, url.pathname, resolvedUrl);\n } else if (resolvedUrl.url.includes(\"fluidrelay.azure.com\")) {\n const url = new URL(urlStr);\n const tenantId = url.searchParams.get(\"tenantId\");\n if (tenantId === null) {\n throw new Error(\"Azure URL did not contain tenantId\");\n }\n return initializeAzure(resolvedUrl, tenantId);\n }\n return Promise.reject(new Error(`Unknown resolved protocol ${protocol}`));\n}\n"]}
|
|
1
|
+
{"version":3,"file":"fluidFetchInit.js","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,6BAA0B;AAC1B,kEAA0C;AAC1C,yFAAsF;AACtF,yEAG2C;AAE3C,+DAAuE;AACvE,sFAAiF;AAEjF,kEAAoD;AAEpD,uEAAmE;AACnE,2EAA6D;AAC7D,yFAAqF;AACrF,2DAAuE;AACvE,qDAA0E;AAC1E,iEAAwD;AAE7C,QAAA,gBAAgB,GAAW,EAAE,CAAC;AAGlC,MAAM,sBAAsB,GAAG,CAAC,GAAW,EAAE,EAAE;IAClD,IAAI,OAAO,GAAG,+CAA+C,CAAC;IAC9D,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,EAAE;QAC9B,uBAAa,CAAC,IAAI,CAAC,2BAA2B,GAAG,GAAG,CAAC,CAAC;QACtD,OAAO,GAAG,kGAAkG,CAAC;KAChH;IACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,OAAO,GAAG,EAAE,CAAC,CAAC;AACxC,CAAC,CAAC;AAPW,QAAA,sBAAsB,0BAOjC;AAEF,KAAK,UAAU,kBAAkB,CAC7B,eAAiC,EACjC,MAAc,EACd,YAA2B;IAE3B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;IAE5C,sBAAc,GAAG;QACb,MAAM;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACf,CAAC;IAEF,IAAI,8BAAa,EAAE;QACf,OAAO;KACV;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,CAAC;YACJ,MAAM;YACN,OAAO;YACP,MAAM;YACN,KAAK,EAAE,CAAC,CAAC;IAEjB,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAsC,EAAE,EAAE;QACzE,OAAO,qCAAc,CACjB,KAAK,EAAE,eAAqC,EAAE,EAAE;YAC5C,IAAI,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC,IAAI,eAAe,CAAC,cAAc,EAAE;gBACrF,OAAO,eAAe,CAAC,cAAc,EAAE,CAAC;aAC3C;YACD,OAAO,eAAe,CAAC,WAAW,CAAC;QACvC,CAAC,EACD,MAAM,EACN,YAAY,EACZ,SAAS,EACT,IAAI,CACP,CAAC;IACN,CAAC,CAAC;IACF,qEAAqE;IACrE,MAAM,qBAAqB,GAAG,CAAC,QAAuC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,0BAA0B,CAClE,mBAAmB,EACnB,qBAAqB,EACrB,SAAS,EACT;QACI,YAAY,EAAE,KAAK;QACnB,oBAAoB,EAAE,CAAC;KAC1B,CAAC,CAAC;IACP,OAAO,0BAA0B,CAAC,qBAAqB,CACnD,eAAe,EACf,SAAS,EAAE,SAAS;IACnB,KAAK,CACT,CAAC;AACN,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,eAAkC;IAC9F,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;QAC3C,QAAQ,GAAG,OAAO,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;SAAM;QACH,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACxB;IAED,+CAA+C;IAC/C,wBAAgB,GAAG,UAAU,CAAC;IAE9B,sBAAc,GAAG;QACb,MAAM;QACN,QAAQ;QACR,EAAE,EAAE,UAAU;KACjB,CAAC;IAEF,IAAI,8BAAa,EAAE;QACf,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,OAAO,UAAU,EAAE,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,oBAAoB,CAAC,yBAAQ,CAAC,CAAC;IAC9D,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CACnD,eAAe,EACf,SAAS,EAAE,SAAS;IACpB,KAAK,CACR,CAAC;AACN,CAAC;AAED,KAAK,UAAU,eAAe,CAAC,WAA8B,EAAE,QAAgB;IAC3E,sBAAc,GAAG;QACb,MAAM,EAAE,WAAW,CAAC,SAAS,CAAC,UAAU;QACxC,QAAQ;QACR,EAAE,EAAE,WAAW,CAAC,EAAE;KACrB,CAAC;IACF,OAAO,CAAC,GAAG,CAAC,6CAA6C,QAAQ,OAAO,WAAW,CAAC,EAAE,EAAE,CAAC,CAAC;IAC1F,MAAM,IAAI,GAAG,oCAAgB,EAAE,CAAC;IAChC,MAAM,aAAa,GAAG,IAAI,yCAAqB,CAAC,8BAAa,EAAE,IAAI,CAAC,CAAC;IACrE,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CACnD,WAAW,EACX,SAAS,EAAE,SAAS;IACpB,KAAK,CACR,CAAC;AACN,CAAC;AAED,KAAK,UAAU,UAAU,CAAC,GAAW;IACjC,MAAM,aAAa,GAAmB;QAClC,IAAI,kCAAe,EAAE;QACrB,IAAI,mDAAuB,EAAE;QAC7B,qEAAqE;QACrE,IAAI,oDAAwB,CAAC,SAAS,EAAE,GAAG,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAQ,CAAC,EAAE,EAAE,CAAC;QAC5E,IAAI,mCAAgB,EAAE;KACzB,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,sCAAuB,CAAC,aAAa,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC;IACvE,OAAO,QAAQ,CAAC;AACpB,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IAC/C,MAAM,WAAW,GAAG,MAAM,UAAU,CAAC,MAAM,CAAsB,CAAC;IAClE,IAAI,CAAC,WAAW,EAAE;QACd,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,eAAe,MAAM,EAAE,CAAC,CAAC,CAAC;KAC7D;IACD,MAAM,QAAQ,GAAG,IAAI,SAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC;IACnD,IAAI,QAAQ,KAAK,aAAa,EAAE;QAC5B,MAAM,eAAe,GAAG,WAA+B,CAAC;QACxD,OAAO,kBAAkB,CAAC,eAAe,EAAE,IAAI,SAAG,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EAAE,sCAAyB,EAAE,CAAC,CAAC;KAClH;SAAM,IAAI,QAAQ,KAAK,QAAQ,EAAE;QAC9B,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,WAAW,CAAC,CAAC;KAC5D;SAAM,IAAI,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,sBAAsB,CAAC,EAAE;QACzD,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,QAAQ,GAAG,GAAG,CAAC,YAAY,CAAC,GAAG,CAAC,UAAU,CAAC,CAAC;QAClD,IAAI,QAAQ,KAAK,IAAI,EAAE;YACnB,MAAM,IAAI,KAAK,CAAC,oCAAoC,CAAC,CAAC;SACzD;QACD,OAAO,eAAe,CAAC,WAAW,EAAE,QAAQ,CAAC,CAAC;KACjD;IACD,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,6BAA6B,QAAQ,EAAE,CAAC,CAAC,CAAC;AAC9E,CAAC;AAtBD,wCAsBC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { URL } from \"url\";\nimport child_process from \"child_process\";\nimport { AzureUrlResolver } from \"@fluidframework/azure-client/dist/AzureUrlResolver\";\nimport {\n generateTestUser,\n InsecureTokenProvider,\n} from \"@fluidframework/test-client-utils\";\nimport { IFluidResolvedUrl, IResolvedUrl, IUrlResolver } from \"@fluidframework/driver-definitions\";\nimport { configurableUrlResolver } from \"@fluidframework/driver-utils\";\nimport { FluidAppOdspUrlResolver } from \"@fluid-tools/fluidapp-odsp-urlresolver\";\nimport { IClientConfig, IOdspAuthRequestInfo } from \"@fluidframework/odsp-doclib-utils\";\nimport * as odsp from \"@fluidframework/odsp-driver\";\nimport { IOdspResolvedUrl, OdspResourceTokenFetchOptions } from \"@fluidframework/odsp-driver-definitions\";\nimport { OdspUrlResolver } from \"@fluidframework/odsp-urlresolver\";\nimport * as r11s from \"@fluidframework/routerlicious-driver\";\nimport { RouterliciousUrlResolver } from \"@fluidframework/routerlicious-urlresolver\";\nimport { getMicrosoftConfiguration } from \"@fluidframework/tool-utils\";\nimport { localDataOnly, paramAzureKey, paramJWT } from \"./fluidFetchArgs\";\nimport { resolveWrapper } from \"./fluidFetchSharePoint\";\n\nexport let latestVersionsId: string = \"\";\nexport let connectionInfo: any;\n\nexport const fluidFetchWebNavigator = (url: string) => {\n let message = \"Please open browser and navigate to this URL:\";\n if (process.platform === \"win32\") {\n child_process.exec(`start \"fluid-fetch\" /B \"${url}\"`);\n message = \"Opening browser to get authorization code. If that doesn't open, please go to this URL manually\";\n }\n console.log(`${message}\\n ${url}`);\n};\n\nasync function initializeODSPCore(\n odspResolvedUrl: IOdspResolvedUrl,\n server: string,\n clientConfig: IClientConfig,\n) {\n const { driveId, itemId } = odspResolvedUrl;\n\n connectionInfo = {\n server,\n drive: driveId,\n item: itemId,\n };\n\n if (localDataOnly) {\n return;\n }\n\n const docId = await odsp.getHashedDocumentId(driveId, itemId);\n\n console.log(`Connecting to ODSP:\n server: ${server}\n drive: ${driveId}\n item: ${itemId}\n docId: ${docId}`);\n\n const getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {\n return resolveWrapper(\n async (authRequestInfo: IOdspAuthRequestInfo) => {\n if ((options.refresh || !authRequestInfo.accessToken) && authRequestInfo.refreshTokenFn) {\n return authRequestInfo.refreshTokenFn();\n }\n return authRequestInfo.accessToken;\n },\n server,\n clientConfig,\n undefined,\n true,\n );\n };\n // eslint-disable-next-line @typescript-eslint/promise-function-async\n const getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) => Promise.resolve(\"\");\n const odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(\n getStorageTokenStub,\n getWebsocketTokenStub,\n undefined,\n {\n opsBatchSize: 20000,\n concurrentOpsBatches: 4,\n });\n return odspDocumentServiceFactory.createDocumentService(\n odspResolvedUrl,\n undefined, // logger\n false, // clientIsSummrizer\n );\n}\n\nasync function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IFluidResolvedUrl) {\n const path = pathname.split(\"/\");\n let tenantId: string;\n let documentId: string;\n if (server === \"localhost\" && path.length < 4) {\n tenantId = \"fluid\";\n documentId = path[2];\n } else {\n tenantId = path[2];\n documentId = path[3];\n }\n\n // Latest version id is the documentId for r11s\n latestVersionsId = documentId;\n\n connectionInfo = {\n server,\n tenantId,\n id: documentId,\n };\n\n if (localDataOnly) {\n return;\n }\n\n console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);\n const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);\n const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n return r11sDocumentServiceFactory.createDocumentService(\n r11sResolvedUrl,\n undefined, // logger\n false, // clientIsSummarizer\n );\n}\n\nasync function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string) {\n connectionInfo = {\n server: resolvedUrl.endpoints.ordererUrl,\n tenantId,\n id: resolvedUrl.id,\n };\n console.log(`Connecting to Azure Fluid Relay: tenantId=${tenantId} id:${resolvedUrl.id}`);\n const user = generateTestUser();\n const tokenProvider = new InsecureTokenProvider(paramAzureKey, user);\n const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n return r11sDocumentServiceFactory.createDocumentService(\n resolvedUrl,\n undefined, // logger\n false, // clientIsSummarizer\n );\n}\n\nasync function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {\n const resolversList: IUrlResolver[] = [\n new OdspUrlResolver(),\n new FluidAppOdspUrlResolver(),\n // eslint-disable-next-line @typescript-eslint/promise-function-async\n new RouterliciousUrlResolver(undefined, () => Promise.resolve(paramJWT), \"\"),\n new AzureUrlResolver(),\n ];\n const resolved = await configurableUrlResolver(resolversList, { url });\n return resolved;\n}\n\nexport async function fluidFetchInit(urlStr: string) {\n const resolvedUrl = await resolveUrl(urlStr) as IFluidResolvedUrl;\n if (!resolvedUrl) {\n return Promise.reject(new Error(`Unknown URL ${urlStr}`));\n }\n const protocol = new URL(resolvedUrl.url).protocol;\n if (protocol === \"fluid-odsp:\") {\n const odspResolvedUrl = resolvedUrl as IOdspResolvedUrl;\n return initializeODSPCore(odspResolvedUrl, new URL(odspResolvedUrl.siteUrl).host, getMicrosoftConfiguration());\n } else if (protocol === \"fluid:\") {\n const url = new URL(urlStr);\n const server = url.hostname.toLowerCase();\n return initializeR11s(server, url.pathname, resolvedUrl);\n } else if (resolvedUrl.url.includes(\"fluidrelay.azure.com\")) {\n const url = new URL(urlStr);\n const tenantId = url.searchParams.get(\"tenantId\");\n if (tenantId === null) {\n throw new Error(\"Azure URL did not contain tenantId\");\n }\n return initializeAzure(resolvedUrl, tenantId);\n }\n return Promise.reject(new Error(`Unknown resolved protocol ${protocol}`));\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAEnB,MAAM,oCAAoC,CAAC;
|
|
1
|
+
{"version":3,"file":"fluidFetchSnapshot.d.ts","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAKH,OAAO,EACH,gBAAgB,EAEnB,MAAM,oCAAoC,CAAC;AAyP5C,wBAAsB,kBAAkB,CACpC,eAAe,CAAC,EAAE,gBAAgB,EAClC,OAAO,CAAC,EAAE,MAAM,iBA4FnB"}
|
|
@@ -61,10 +61,9 @@ function createTreeBlob(tree, prefix, patched) {
|
|
|
61
61
|
const treePath = `${prefix}${filename}`;
|
|
62
62
|
return { treePath, blobId: "original tree $id", filename, blob, patched, reused: false };
|
|
63
63
|
}
|
|
64
|
-
async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/",
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
if (commit && fluidFetchArgs_1.dumpSnapshotTrees) {
|
|
64
|
+
async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", parentBlobIdMap) {
|
|
65
|
+
const isTopLevel = !parentBlobIdMap;
|
|
66
|
+
if (isTopLevel && fluidFetchArgs_1.dumpSnapshotTrees) {
|
|
68
67
|
console.log(tree);
|
|
69
68
|
}
|
|
70
69
|
if (prefix === "/") {
|
|
@@ -72,37 +71,19 @@ async function fetchBlobsFromSnapshotTree(storage, tree, prefix = "/", perCommit
|
|
|
72
71
|
blobCacheCurrent = new Map();
|
|
73
72
|
}
|
|
74
73
|
// Create the tree info before fetching blobs (which will modify it)
|
|
75
|
-
let
|
|
76
|
-
if (
|
|
77
|
-
|
|
74
|
+
let topLevelBlob;
|
|
75
|
+
if (isTopLevel) {
|
|
76
|
+
topLevelBlob = createTreeBlob(tree, prefix, false);
|
|
78
77
|
}
|
|
79
|
-
const blobIdMap =
|
|
78
|
+
const blobIdMap = parentBlobIdMap !== null && parentBlobIdMap !== void 0 ? parentBlobIdMap : new Map();
|
|
80
79
|
let result = fetchBlobs(prefix, tree, storage, blobIdMap);
|
|
81
|
-
for (const dataStore of Object.keys(tree.commits)) {
|
|
82
|
-
const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);
|
|
83
|
-
if (dataStoreVersions.length !== 1) {
|
|
84
|
-
console.error(`ERROR: Unable to get versions for ${dataStore}`);
|
|
85
|
-
continue;
|
|
86
|
-
}
|
|
87
|
-
const dataStoreSnapShotTree = await reportErrors(`getSnapshotTree ${dataStoreVersions[0].id}`, storage.getSnapshotTree(dataStoreVersions[0]));
|
|
88
|
-
if (dataStoreSnapShotTree === null) {
|
|
89
|
-
// eslint-disable-next-line max-len
|
|
90
|
-
console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);
|
|
91
|
-
continue;
|
|
92
|
-
}
|
|
93
|
-
common_utils_1.assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore], 0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);
|
|
94
|
-
common_utils_1.assert(tree.commits[dataStore] === dataStoreVersions[0].id, 0x1c0 /* "Mismatch between commit id and fetched tree id" */);
|
|
95
|
-
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, dataStoreSnapShotTree, `${prefix}[${dataStore}]/`);
|
|
96
|
-
result = result.concat(dataStoreBlobs);
|
|
97
|
-
}
|
|
98
80
|
for (const subtreeId of Object.keys(tree.trees)) {
|
|
99
81
|
const subtree = tree.trees[subtreeId];
|
|
100
|
-
common_utils_1.assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* "Unexpected subtree properties" */);
|
|
101
82
|
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(storage, subtree, `${prefix}${subtreeId}/`, blobIdMap);
|
|
102
83
|
result = result.concat(dataStoreBlobs);
|
|
103
84
|
}
|
|
104
|
-
if (
|
|
105
|
-
result.push(
|
|
85
|
+
if (topLevelBlob) {
|
|
86
|
+
result.push(topLevelBlob);
|
|
106
87
|
result.push(createTreeBlob(tree, prefix, true));
|
|
107
88
|
}
|
|
108
89
|
return result;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,gDAAwB;AACxB,+DAA2G;AAS3G,+DAAsE;AACtE,iEAAsD;AACtD,qDAQ0B;AAC1B,qDAAoD;AA8BpD,SAAS,aAAa,CAAC,WAAyB;IAC5C,OAAO,SAAS,IAAI,WAAW,CAAC;AACpC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAAC,MAAc,EAC9B,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACjB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACP,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACpB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC/B;aACJ;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE;gBACR,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAChC;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC/B;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;;IACzE,MAAM,EAAE,SAAG,IAAI,CAAC,EAAE,mCAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,6BAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACrC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,kBAAwC;IACxC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,GAAG,CAAC,EAC7D,KAAK,CAAC,sCAAsC,CAAC,CAAC;IAClD,MAAM,MAAM,GAAG,CAAC,kBAAkB,CAAC;IACnC,IAAI,MAAM,IAAI,kCAAiB,EAAE;QAC7B,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KACrB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QAChB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAClE;IAED,oEAAoE;IACpE,IAAI,UAAoC,CAAC;IACzC,IAAI,MAAM,EAAE;QACR,UAAU,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACpD;IAED,MAAM,SAAS,GAAG,kBAAkB,aAAlB,kBAAkB,cAAlB,kBAAkB,GAAI,IAAI,GAAG,EAAkB,CAAC;IAClE,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE;QAC/C,MAAM,iBAAiB,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,EAAE;YAChC,OAAO,CAAC,KAAK,CAAC,qCAAqC,SAAS,EAAE,CAAC,CAAC;YAChE,SAAS;SACZ;QACD,MAAM,qBAAqB,GAAG,MAAM,YAAY,CAC5C,mBAAmB,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,EAC5C,OAAO,CAAC,eAAe,CAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACnD,IAAI,qBAAqB,KAAK,IAAI,EAAE;YAChC,mCAAmC;YACnC,OAAO,CAAC,KAAK,CAAC,uCAAuC,SAAS,YAAY,MAAM,eAAe,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC1H,SAAS;SACZ;QACD,qBAAM,CAAC,qBAAqB,CAAC,EAAE,KAAK,SAAS,IAAI,qBAAqB,CAAC,EAAE,KAAK,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,EACjG,KAAK,CAAC,2DAA2D,CAAC,CAAC;QACvE,qBAAM,CAAC,IAAI,CAAC,OAAO,CAAC,SAAS,CAAC,KAAK,iBAAiB,CAAC,CAAC,CAAC,CAAC,EAAE,EACtD,KAAK,CAAC,sDAAsD,CAAC,CAAC;QAClE,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,qBAAqB,EACrB,GAAG,MAAM,IAAI,SAAS,IAAI,CAAC,CAAC;QAChC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,qBAAM,CAAC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC/F,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,IAAI,UAAU,EAAE;QACZ,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KAC3D;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;QAC3C,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,mCAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,mCAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SAClB;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IAClF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACV;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACtB,6BAA6B;YAC7B,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI;gBACA,IAAI,CAAC,sCAAqB,EAAE;oBACxB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC/D;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,YAAE,CAAC,aAAa,CACZ,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SAC3D;aAAM;YACH,qEAAqE;YACrE,MAAM,UAAU,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YACjD,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACtD,sCAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;SAClG;IACL,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACpF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;KACpE;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC3D,IAAI;QACA,OAAO,MAAM,GAAG,CAAC;KACpB;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACf;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACpC,eAAkC,EAClC,OAAgB;IAEhB,IAAI,CAAC,kCAAiB,IAAI,CAAC,kCAAiB,IAAI,CAAC,qCAAoB,IAAI,OAAO,KAAK,SAAS,EAAE;QAC5F,OAAO;KACV;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QAClB,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IACvD,IAAI,2CAA0B,EAAE;QAC5B,OAAO,GAAG,qCAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,kCAAmB,EAAE,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvG;IAED,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAC/B,eAAe,iCAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,iCAAgB,EAAE,yCAAwB,CAAC,CAAC,CAAC;IACrE,IAAI,qCAAoB,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACzB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,0CAAyB,KAAK,SAAS,EAAE;QACzC,OAAO,GAAG,QAAQ,CAAC,0CAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAAC,CAAC;YAChG,OAAO;SACV;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAClD;KACJ;SAAM;QACH,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC3C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,EAAE;oBACR,IAAI;wBACA,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBAC5C;oBAAC,OAAO,CAAC,EAAE;wBACR,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACX,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACvC;qBACJ;iBACJ;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,mCAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,mCAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,mCAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,mCAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CAAC,CAAC;gBAEpG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aAC5C;SACJ;KACJ;IAED,IAAI,kCAAiB,IAAI,kCAAiB,EAAE;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SACnC;aAAM;YACH,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC3B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC/D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SAC1D;KACJ;AACL,CAAC;AA9FD,gDA8FC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\nimport { assert, bufferToString, stringToBuffer, TelemetryNullLogger } from \"@fluidframework/common-utils\";\nimport {\n IDocumentService,\n IDocumentStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n ISnapshotTree,\n IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport { BlobAggregationStorage } from \"@fluidframework/driver-utils\";\nimport { formatNumber } from \"./fluidAnalyzeMessages\";\nimport {\n dumpSnapshotStats,\n dumpSnapshotTrees,\n dumpSnapshotVersions,\n paramActualFormatting,\n paramNumSnapshotVersions,\n paramSnapshotVersionIndex,\n paramUnpackAggregatedBlobs,\n} from \"./fluidFetchArgs\";\nimport { latestVersionsId } from \"./fluidFetchInit\";\n\ninterface ISnapshotInfo {\n blobCountNew: number;\n blobCount: number;\n size: number;\n sizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n treePath: string;\n filename: string;\n blobId: string;\n blob: Promise<ArrayBufferLike | undefined>;\n reused: boolean;\n}\n\ninterface IFetchedTree {\n treePath: string;\n blobId: string;\n filename: string;\n blob: ArrayBufferLike;\n\n reused: false;\n\n patched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n return \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(prefix: string,\n tree: ISnapshotTree,\n storage: IDocumentStorageService,\n blobIdMap: Map<string, number>,\n) {\n const result: IFetchedBlob[] = [];\n for (const item of Object.keys(tree.blobs)) {\n const treePath = `${prefix}${item}`;\n const blobId = tree.blobs[item];\n if (blobId !== null) {\n let reused = true;\n let blob = blobCachePrevious.get(blobId);\n if (!blob) {\n reused = false;\n blob = blobCache.get(blobId);\n if (blob === undefined) {\n blob = storage.readBlob(blobId);\n blobCache.set(blobId, blob);\n }\n }\n blobCacheCurrent.set(blobId, blob);\n\n // Use the blobIdMap to assign a number for each unique blob\n // and use it as a prefix for files to avoid case-insensitive fs\n let index = blobIdMap.get(blobId);\n if (!index) {\n index = blobIdMap.size;\n blobIdMap.set(blobId, index);\n }\n const filename = `${index}-${blobId}`;\n result.push({ treePath, blobId, blob, reused, filename });\n\n // patch the tree so that we can write it out to reference the file\n tree.blobs[item] = filename;\n }\n }\n return result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n const id = tree.id ?? \"original\";\n const blob = stringToBuffer(JSON.stringify(tree),\"utf8\");\n const filename = patched ? \"tree\" : `tree-${id}`;\n const treePath = `${prefix}${filename}`;\n return { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n storage: IDocumentStorageService,\n tree: ISnapshotTree,\n prefix: string = \"/\",\n perCommitBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {\n assert(Object.keys(tree.commits).length === 0 || (prefix === \"/\"),\n 0x1be /* \"Unexpected tree input to fetch\" */);\n const commit = !perCommitBlobIdMap;\n if (commit && dumpSnapshotTrees) {\n console.log(tree);\n }\n\n if (prefix === \"/\") {\n blobCachePrevious = blobCacheCurrent;\n blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n }\n\n // Create the tree info before fetching blobs (which will modify it)\n let commitBlob: IFetchedTree | undefined;\n if (commit) {\n commitBlob = createTreeBlob(tree, prefix, false);\n }\n\n const blobIdMap = perCommitBlobIdMap ?? new Map<string, number>();\n let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n for (const dataStore of Object.keys(tree.commits)) {\n const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);\n if (dataStoreVersions.length !== 1) {\n console.error(`ERROR: Unable to get versions for ${dataStore}`);\n continue;\n }\n const dataStoreSnapShotTree = await reportErrors(\n `getSnapshotTree ${dataStoreVersions[0].id}`,\n storage.getSnapshotTree(dataStoreVersions[0]));\n if (dataStoreSnapShotTree === null) {\n // eslint-disable-next-line max-len\n console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);\n continue;\n }\n assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore],\n 0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);\n assert(tree.commits[dataStore] === dataStoreVersions[0].id,\n 0x1c0 /* \"Mismatch between commit id and fetched tree id\" */);\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n dataStoreSnapShotTree,\n `${prefix}[${dataStore}]/`);\n result = result.concat(dataStoreBlobs);\n }\n\n for (const subtreeId of Object.keys(tree.trees)) {\n const subtree = tree.trees[subtreeId];\n assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* \"Unexpected subtree properties\" */);\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n subtree,\n `${prefix}${subtreeId}/`, blobIdMap);\n result = result.concat(dataStoreBlobs);\n }\n\n if (commitBlob) {\n result.push(commitBlob);\n result.push(createTreeBlob(tree, prefix, true));\n }\n return result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n return sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n let size = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n let nameLength = 10;\n for (const item of sorted) {\n nameLength = Math.max(nameLength, item.treePath.length);\n }\n\n console.log(\"\");\n console.log(`${\"Blob Path\".padEnd(nameLength)} | Reused | Bytes`);\n console.log(\"-\".repeat(nameLength + 26));\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer,\"utf8\");\n // eslint-disable-next-line max-len\n console.log(`${item.treePath.padEnd(nameLength)} | ${item.reused ? \"X\" : \" \"} | ${formatNumber(blob.length).padStart(10)}`);\n size += blob.length;\n }\n\n console.log(\"-\".repeat(nameLength + 26));\n console.log(`${\"Total snapshot size\".padEnd(nameLength)} | | ${formatNumber(size).padStart(10)}`);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n let size = 0;\n let sizeNew = 0;\n let blobCountNew = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer, \"utf8\");\n if (!item.reused) {\n sizeNew += blob.length;\n blobCountNew++;\n }\n size += blob.length;\n }\n\n return { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n const outDir = `${saveDir}/${name}/`;\n const mkdir = util.promisify(fs.mkdir);\n\n await mkdir(`${outDir}/decoded`, { recursive: true });\n await Promise.all(fetchedData.map(async (item) => {\n const buffer = await item.blob;\n if (buffer === undefined) {\n console.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n return;\n }\n\n if (!isFetchedTree(item)) {\n // Just write the data as is.\n fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n // we assume that the buffer is utf8 here, which currently is true for\n // all of our snapshot blobs. It doesn't necessary be true in the future\n let decoded = bufferToString(buffer,\"utf8\");\n try {\n if (!paramActualFormatting) {\n decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n }\n } catch (e) {\n }\n fs.writeFileSync(\n `${outDir}/decoded/${item.filename}.json`, decoded);\n } else {\n // Write out same data for tree decoded or not, except for formatting\n const treeString = bufferToString(buffer,\"utf8\");\n fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,\n paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));\n }\n }));\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));\n if (!tree) {\n return Promise.reject(new Error(\"Failed to load snapshot tree\"));\n }\n return fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n try {\n return await res;\n } catch (error) {\n console.error(`Error calling ${message}`);\n throw error;\n }\n}\n\nexport async function fluidFetchSnapshot(\n documentService?: IDocumentService,\n saveDir?: string,\n ) {\n if (!dumpSnapshotStats && !dumpSnapshotTrees && !dumpSnapshotVersions && saveDir === undefined) {\n return;\n }\n\n // --local mode - do not connect to storage.\n // For now, bail out early.\n // In future, separate download from analyzes parts and allow offline analyzes\n if (!documentService) {\n return;\n }\n\n console.log(\"\\n\");\n\n let storage = await documentService.connectToStorage();\n if (paramUnpackAggregatedBlobs) {\n storage = BlobAggregationStorage.wrap(storage, new TelemetryNullLogger(), false /* allowPacking */);\n }\n\n let version: IVersion | undefined;\n const versions = await reportErrors(\n `getVersions ${latestVersionsId}`,\n storage.getVersions(latestVersionsId, paramNumSnapshotVersions));\n if (dumpSnapshotVersions) {\n console.log(\"Snapshot versions\");\n console.log(versions);\n }\n\n let blobsToDump: IFetchedData[] | undefined;\n if (paramSnapshotVersionIndex !== undefined) {\n version = versions[paramSnapshotVersionIndex];\n if (version === undefined) {\n console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);\n return;\n }\n if (saveDir !== undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n const name = version.id;\n console.log(`Saving snapshot ${name}`);\n await saveSnapshot(name, blobsToDump, saveDir);\n }\n } else {\n version = versions[0];\n if (saveDir !== undefined && versions.length > 0) {\n console.log(\" Name | Date | Size | New Size | Blobs | New Blobs\");\n console.log(\"-\".repeat(86));\n\n // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n for (let i = versions.length - 1; i >= 0; i--) {\n const v = versions[i];\n const blobs = await fetchBlobsFromVersion(storage, v);\n blobsToDump = blobs;\n const name = `${i}-${v.id}`;\n const res = await dumpSnapshotTree(name, blobs);\n\n let date = \"\";\n if (v.date) {\n try {\n date = new Date(v.date).toLocaleString();\n } catch (e) {\n date = v.date.replace(\"T\", \" \");\n const index = date.lastIndexOf(\".\");\n if (index > 0) {\n date = `${date.substr(0, index)} Z`;\n }\n }\n }\n date = date.padStart(23);\n const size = formatNumber(res.size).padStart(10);\n const sizeNew = formatNumber(res.sizeNew).padStart(10);\n const blobCount = formatNumber(res.blobCount).padStart(6);\n const blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);\n\n await saveSnapshot(name, blobs, saveDir);\n }\n }\n }\n\n if (dumpSnapshotStats || dumpSnapshotTrees) {\n if (version === undefined) {\n console.log(\"No snapshot tree\");\n } else {\n if (blobsToDump === undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n }\n console.log(`\\n\\nSnapshot version ${version.id}`);\n await dumpSnapshotTreeVerbose(version.id, blobsToDump);\n }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"fluidFetchSnapshot.js","sourceRoot":"","sources":["../src/fluidFetchSnapshot.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,gDAAwB;AACxB,+DAAmG;AASnG,+DAAsE;AACtE,iEAAsD;AACtD,qDAQ0B;AAC1B,qDAAoD;AA8BpD,SAAS,aAAa,CAAC,WAAyB;IAC5C,OAAO,SAAS,IAAI,WAAW,CAAC;AACpC,CAAC;AAED,MAAM,SAAS,GAAG,IAAI,GAAG,EAAoC,CAAC;AAC9D,IAAI,iBAAiB,GAAG,IAAI,GAAG,EAAoC,CAAC;AACpE,IAAI,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;AAEnE,SAAS,UAAU,CAAC,MAAc,EAC9B,IAAmB,EACnB,OAAgC,EAChC,SAA8B;IAE9B,MAAM,MAAM,GAAmB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QACxC,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,IAAI,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAChC,IAAI,MAAM,KAAK,IAAI,EAAE;YACjB,IAAI,MAAM,GAAG,IAAI,CAAC;YAClB,IAAI,IAAI,GAAG,iBAAiB,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YACzC,IAAI,CAAC,IAAI,EAAE;gBACP,MAAM,GAAG,KAAK,CAAC;gBACf,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC7B,IAAI,IAAI,KAAK,SAAS,EAAE;oBACpB,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;oBAChC,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;iBAC/B;aACJ;YACD,gBAAgB,CAAC,GAAG,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;YAEnC,4DAA4D;YAC5D,gEAAgE;YAChE,IAAI,KAAK,GAAG,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAClC,IAAI,CAAC,KAAK,EAAE;gBACR,KAAK,GAAG,SAAS,CAAC,IAAI,CAAC;gBACvB,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;aAChC;YACD,MAAM,QAAQ,GAAG,GAAG,KAAK,IAAI,MAAM,EAAE,CAAC;YACtC,MAAM,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;YAE1D,mEAAmE;YACnE,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,QAAQ,CAAC;SAC/B;KACJ;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,cAAc,CAAC,IAAmB,EAAE,MAAc,EAAE,OAAgB;;IACzE,MAAM,EAAE,SAAG,IAAI,CAAC,EAAE,mCAAI,UAAU,CAAC;IACjC,MAAM,IAAI,GAAG,6BAAc,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,EAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC;IACjD,MAAM,QAAQ,GAAG,GAAG,MAAM,GAAG,QAAQ,EAAE,CAAC;IACxC,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,mBAAmB,EAAE,QAAQ,EAAE,IAAI,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,CAAC;AAC7F,CAAC;AAED,KAAK,UAAU,0BAA0B,CACrC,OAAgC,EAChC,IAAmB,EACnB,SAAiB,GAAG,EACpB,eAAqC;IACrC,MAAM,UAAU,GAAG,CAAC,eAAe,CAAC;IACpC,IAAI,UAAU,IAAI,kCAAiB,EAAE;QACjC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;KACrB;IAED,IAAI,MAAM,KAAK,GAAG,EAAE;QAChB,iBAAiB,GAAG,gBAAgB,CAAC;QACrC,gBAAgB,GAAG,IAAI,GAAG,EAAoC,CAAC;KAClE;IAED,oEAAoE;IACpE,IAAI,YAAsC,CAAC;IAC3C,IAAI,UAAU,EAAE;QACZ,YAAY,GAAG,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,KAAK,CAAC,CAAC;KACtD;IAED,MAAM,SAAS,GAAG,eAAe,aAAf,eAAe,cAAf,eAAe,GAAI,IAAI,GAAG,EAAkB,CAAC;IAC/D,IAAI,MAAM,GAAmB,UAAU,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAE1E,KAAK,MAAM,SAAS,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE;QAC7C,MAAM,OAAO,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,cAAc,GAAG,MAAM,0BAA0B,CACnD,OAAO,EACP,OAAO,EACP,GAAG,MAAM,GAAG,SAAS,GAAG,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;KAC1C;IAED,IAAI,YAAY,EAAE;QACd,MAAM,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;KACnD;IACD,OAAO,MAAM,CAAC;AAClB,CAAC;AAED,SAAS,kBAAkB,CAAC,WAA2B;IACnD,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC;IAChF,OAAO,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,uBAAuB,CAAC,IAAY,EAAE,WAA2B;IAC5E,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;KAC3D;IAED,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC;IAChB,OAAO,CAAC,GAAG,CAAC,GAAG,WAAW,CAAC,MAAM,CAAC,UAAU,CAAC,wBAAwB,CAAC,CAAC;IACvE,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;QAC3C,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,UAAU,CAAC,SAAS,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG,QAAQ,mCAAY,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;QACjI,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,UAAU,GAAG,EAAE,CAAC,CAAC,CAAC;IACzC,OAAO,CAAC,GAAG,CAAC,GAAG,qBAAqB,CAAC,MAAM,CAAC,UAAU,CAAC,eAAe,mCAAY,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,EAAE,CAAC,CAAC;AAC7G,CAAC;AAED,KAAK,UAAU,gBAAgB,CAAC,IAAY,EAAE,WAA2B;IACrE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,IAAI,YAAY,GAAG,CAAC,CAAC;IACrB,MAAM,MAAM,GAAG,kBAAkB,CAAC,WAAW,CAAC,CAAC;IAE/C,KAAK,MAAM,IAAI,IAAI,MAAM,EAAE;QACvB,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,SAAS;SACZ;QACD,MAAM,IAAI,GAAG,6BAAc,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;QAC5C,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE;YACd,OAAO,IAAI,IAAI,CAAC,MAAM,CAAC;YACvB,YAAY,EAAE,CAAC;SAClB;QACD,IAAI,IAAI,IAAI,CAAC,MAAM,CAAC;KACvB;IAED,OAAO,EAAE,YAAY,EAAE,SAAS,EAAE,MAAM,CAAC,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,YAAY,CAAC,IAAY,EAAE,WAA2B,EAAE,OAAe;IAClF,MAAM,MAAM,GAAG,GAAG,OAAO,IAAI,IAAI,GAAG,CAAC;IACrC,MAAM,KAAK,GAAG,cAAI,CAAC,SAAS,CAAC,YAAE,CAAC,KAAK,CAAC,CAAC;IAEvC,MAAM,KAAK,CAAC,GAAG,MAAM,UAAU,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACtD,MAAM,OAAO,CAAC,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;QAC7C,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,CAAC;QAC/B,IAAI,MAAM,KAAK,SAAS,EAAE;YACtB,OAAO,CAAC,KAAK,CAAC,sCAAsC,IAAI,CAAC,MAAM,EAAE,CAAC,CAAC;YACnE,OAAO;SACV;QAED,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,EAAE;YACtB,6BAA6B;YAC7B,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,EAAE,EAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC;YAEpE,sEAAsE;YACtE,yEAAyE;YACzE,IAAI,OAAO,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YAC5C,IAAI;gBACA,IAAI,CAAC,sCAAqB,EAAE;oBACxB,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC;iBAC/D;aACJ;YAAC,OAAO,CAAC,EAAE;aACX;YACD,YAAE,CAAC,aAAa,CACZ,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EAAE,OAAO,CAAC,CAAC;SAC3D;aAAM;YACH,qEAAqE;YACrE,MAAM,UAAU,GAAG,6BAAc,CAAC,MAAM,EAAC,MAAM,CAAC,CAAC;YACjD,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,IAAI,IAAI,CAAC,QAAQ,OAAO,EAAE,UAAU,CAAC,CAAC;YAChE,YAAE,CAAC,aAAa,CAAC,GAAG,MAAM,YAAY,IAAI,CAAC,QAAQ,OAAO,EACtD,sCAAqB,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC;SAClG;IACL,CAAC,CAAC,CAAC,CAAC;AACR,CAAC;AAED,KAAK,UAAU,qBAAqB,CAAC,OAAgC,EAAE,OAAiB;IACpF,MAAM,IAAI,GAAG,MAAM,YAAY,CAAC,mBAAmB,OAAO,CAAC,EAAE,EAAE,EAAE,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,CAAC;IACnG,IAAI,CAAC,IAAI,EAAE;QACP,OAAO,OAAO,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,8BAA8B,CAAC,CAAC,CAAC;KACpE;IACD,OAAO,0BAA0B,CAAC,OAAO,EAAE,IAAI,CAAC,CAAC;AACrD,CAAC;AAED,KAAK,UAAU,YAAY,CAAI,OAAe,EAAE,GAAe;IAC3D,IAAI;QACA,OAAO,MAAM,GAAG,CAAC;KACpB;IAAC,OAAO,KAAK,EAAE;QACZ,OAAO,CAAC,KAAK,CAAC,iBAAiB,OAAO,EAAE,CAAC,CAAC;QAC1C,MAAM,KAAK,CAAC;KACf;AACL,CAAC;AAEM,KAAK,UAAU,kBAAkB,CACpC,eAAkC,EAClC,OAAgB;IAEhB,IAAI,CAAC,kCAAiB,IAAI,CAAC,kCAAiB,IAAI,CAAC,qCAAoB,IAAI,OAAO,KAAK,SAAS,EAAE;QAC5F,OAAO;KACV;IAED,4CAA4C;IAC5C,2BAA2B;IAC3B,8EAA8E;IAC9E,IAAI,CAAC,eAAe,EAAE;QAClB,OAAO;KACV;IAED,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;IAElB,IAAI,OAAO,GAAG,MAAM,eAAe,CAAC,gBAAgB,EAAE,CAAC;IACvD,IAAI,2CAA0B,EAAE;QAC5B,OAAO,GAAG,qCAAsB,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,kCAAmB,EAAE,EAAE,KAAK,CAAC,kBAAkB,CAAC,CAAC;KACvG;IAED,IAAI,OAA6B,CAAC;IAClC,MAAM,QAAQ,GAAG,MAAM,YAAY,CAC/B,eAAe,iCAAgB,EAAE,EACjC,OAAO,CAAC,WAAW,CAAC,iCAAgB,EAAE,yCAAwB,CAAC,CAAC,CAAC;IACrE,IAAI,qCAAoB,EAAE;QACtB,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;QACjC,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACzB;IAED,IAAI,WAAuC,CAAC;IAC5C,IAAI,0CAAyB,KAAK,SAAS,EAAE;QACzC,OAAO,GAAG,QAAQ,CAAC,0CAAyB,CAAC,CAAC;QAC9C,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,QAAQ,CAAC,MAAM,iDAAiD,CAAC,CAAC;YAChG,OAAO;SACV;QACD,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;YAC5D,MAAM,IAAI,GAAG,OAAO,CAAC,EAAE,CAAC;YACxB,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,EAAE,CAAC,CAAC;YACvC,MAAM,YAAY,CAAC,IAAI,EAAE,WAAW,EAAE,OAAO,CAAC,CAAC;SAClD;KACJ;SAAM;QACH,OAAO,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;QACtB,IAAI,OAAO,KAAK,SAAS,IAAI,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE;YAC9C,OAAO,CAAC,GAAG,CAAC,0FAA0F,CAAC,CAAC;YACxG,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC;YAE5B,2FAA2F;YAC3F,KAAK,IAAI,CAAC,GAAG,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,EAAE;gBAC3C,MAAM,CAAC,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC;gBACtB,MAAM,KAAK,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;gBACtD,WAAW,GAAG,KAAK,CAAC;gBACpB,MAAM,IAAI,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,EAAE,EAAE,CAAC;gBAC5B,MAAM,GAAG,GAAG,MAAM,gBAAgB,CAAC,IAAI,EAAE,KAAK,CAAC,CAAC;gBAEhD,IAAI,IAAI,GAAG,EAAE,CAAC;gBACd,IAAI,CAAC,CAAC,IAAI,EAAE;oBACR,IAAI;wBACA,IAAI,GAAG,IAAI,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,cAAc,EAAE,CAAC;qBAC5C;oBAAC,OAAO,CAAC,EAAE;wBACR,IAAI,GAAG,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,CAAC,CAAC;wBAChC,MAAM,KAAK,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC;wBACpC,IAAI,KAAK,GAAG,CAAC,EAAE;4BACX,IAAI,GAAG,GAAG,IAAI,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,IAAI,CAAC;yBACvC;qBACJ;iBACJ;gBACD,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACzB,MAAM,IAAI,GAAG,mCAAY,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACjD,MAAM,OAAO,GAAG,mCAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;gBACvD,MAAM,SAAS,GAAG,mCAAY,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAC1D,MAAM,YAAY,GAAG,mCAAY,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;gBAEhE,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,IAAI,MAAM,IAAI,MAAM,OAAO,MAAM,SAAS,MAAM,YAAY,EAAE,CAAC,CAAC;gBAEpG,MAAM,YAAY,CAAC,IAAI,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;aAC5C;SACJ;KACJ;IAED,IAAI,kCAAiB,IAAI,kCAAiB,EAAE;QACxC,IAAI,OAAO,KAAK,SAAS,EAAE;YACvB,OAAO,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;SACnC;aAAM;YACH,IAAI,WAAW,KAAK,SAAS,EAAE;gBAC3B,WAAW,GAAG,MAAM,qBAAqB,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC;aAC/D;YACD,OAAO,CAAC,GAAG,CAAC,wBAAwB,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;YAClD,MAAM,uBAAuB,CAAC,OAAO,CAAC,EAAE,EAAE,WAAW,CAAC,CAAC;SAC1D;KACJ;AACL,CAAC;AA9FD,gDA8FC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport util from \"util\";\nimport { bufferToString, stringToBuffer, TelemetryNullLogger } from \"@fluidframework/common-utils\";\nimport {\n IDocumentService,\n IDocumentStorageService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n ISnapshotTree,\n IVersion,\n} from \"@fluidframework/protocol-definitions\";\nimport { BlobAggregationStorage } from \"@fluidframework/driver-utils\";\nimport { formatNumber } from \"./fluidAnalyzeMessages\";\nimport {\n dumpSnapshotStats,\n dumpSnapshotTrees,\n dumpSnapshotVersions,\n paramActualFormatting,\n paramNumSnapshotVersions,\n paramSnapshotVersionIndex,\n paramUnpackAggregatedBlobs,\n} from \"./fluidFetchArgs\";\nimport { latestVersionsId } from \"./fluidFetchInit\";\n\ninterface ISnapshotInfo {\n blobCountNew: number;\n blobCount: number;\n size: number;\n sizeNew: number;\n}\n\ntype IFetchedData = IFetchedBlob | IFetchedTree;\n\ninterface IFetchedBlob {\n treePath: string;\n filename: string;\n blobId: string;\n blob: Promise<ArrayBufferLike | undefined>;\n reused: boolean;\n}\n\ninterface IFetchedTree {\n treePath: string;\n blobId: string;\n filename: string;\n blob: ArrayBufferLike;\n\n reused: false;\n\n patched: boolean;\n}\n\nfunction isFetchedTree(fetchedData: IFetchedData): fetchedData is IFetchedTree {\n return \"patched\" in fetchedData;\n}\n\nconst blobCache = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCachePrevious = new Map<string, Promise<ArrayBufferLike>>();\nlet blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n\nfunction fetchBlobs(prefix: string,\n tree: ISnapshotTree,\n storage: IDocumentStorageService,\n blobIdMap: Map<string, number>,\n) {\n const result: IFetchedBlob[] = [];\n for (const item of Object.keys(tree.blobs)) {\n const treePath = `${prefix}${item}`;\n const blobId = tree.blobs[item];\n if (blobId !== null) {\n let reused = true;\n let blob = blobCachePrevious.get(blobId);\n if (!blob) {\n reused = false;\n blob = blobCache.get(blobId);\n if (blob === undefined) {\n blob = storage.readBlob(blobId);\n blobCache.set(blobId, blob);\n }\n }\n blobCacheCurrent.set(blobId, blob);\n\n // Use the blobIdMap to assign a number for each unique blob\n // and use it as a prefix for files to avoid case-insensitive fs\n let index = blobIdMap.get(blobId);\n if (!index) {\n index = blobIdMap.size;\n blobIdMap.set(blobId, index);\n }\n const filename = `${index}-${blobId}`;\n result.push({ treePath, blobId, blob, reused, filename });\n\n // patch the tree so that we can write it out to reference the file\n tree.blobs[item] = filename;\n }\n }\n return result;\n}\n\nfunction createTreeBlob(tree: ISnapshotTree, prefix: string, patched: boolean): IFetchedTree {\n const id = tree.id ?? \"original\";\n const blob = stringToBuffer(JSON.stringify(tree),\"utf8\");\n const filename = patched ? \"tree\" : `tree-${id}`;\n const treePath = `${prefix}${filename}`;\n return { treePath, blobId: \"original tree $id\", filename, blob, patched, reused: false };\n}\n\nasync function fetchBlobsFromSnapshotTree(\n storage: IDocumentStorageService,\n tree: ISnapshotTree,\n prefix: string = \"/\",\n parentBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {\n const isTopLevel = !parentBlobIdMap;\n if (isTopLevel && dumpSnapshotTrees) {\n console.log(tree);\n }\n\n if (prefix === \"/\") {\n blobCachePrevious = blobCacheCurrent;\n blobCacheCurrent = new Map<string, Promise<ArrayBufferLike>>();\n }\n\n // Create the tree info before fetching blobs (which will modify it)\n let topLevelBlob: IFetchedTree | undefined;\n if (isTopLevel) {\n topLevelBlob = createTreeBlob(tree, prefix, false);\n }\n\n const blobIdMap = parentBlobIdMap ?? new Map<string, number>();\n let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);\n\n for (const subtreeId of Object.keys(tree.trees)) {\n const subtree = tree.trees[subtreeId];\n const dataStoreBlobs = await fetchBlobsFromSnapshotTree(\n storage,\n subtree,\n `${prefix}${subtreeId}/`, blobIdMap);\n result = result.concat(dataStoreBlobs);\n }\n\n if (topLevelBlob) {\n result.push(topLevelBlob);\n result.push(createTreeBlob(tree, prefix, true));\n }\n return result;\n}\n\nfunction getDumpFetchedData(fetchedData: IFetchedData[]) {\n const sorted = fetchedData.sort((a, b) => a.treePath.localeCompare(b.treePath));\n return sorted.filter((item) => !isFetchedTree(item) || !item.patched);\n}\n\nasync function dumpSnapshotTreeVerbose(name: string, fetchedData: IFetchedData[]) {\n let size = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n let nameLength = 10;\n for (const item of sorted) {\n nameLength = Math.max(nameLength, item.treePath.length);\n }\n\n console.log(\"\");\n console.log(`${\"Blob Path\".padEnd(nameLength)} | Reused | Bytes`);\n console.log(\"-\".repeat(nameLength + 26));\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer,\"utf8\");\n // eslint-disable-next-line max-len\n console.log(`${item.treePath.padEnd(nameLength)} | ${item.reused ? \"X\" : \" \"} | ${formatNumber(blob.length).padStart(10)}`);\n size += blob.length;\n }\n\n console.log(\"-\".repeat(nameLength + 26));\n console.log(`${\"Total snapshot size\".padEnd(nameLength)} | | ${formatNumber(size).padStart(10)}`);\n}\n\nasync function dumpSnapshotTree(name: string, fetchedData: IFetchedData[]): Promise<ISnapshotInfo> {\n let size = 0;\n let sizeNew = 0;\n let blobCountNew = 0;\n const sorted = getDumpFetchedData(fetchedData);\n\n for (const item of sorted) {\n const buffer = await item.blob;\n if (buffer === undefined) {\n continue;\n }\n const blob = bufferToString(buffer, \"utf8\");\n if (!item.reused) {\n sizeNew += blob.length;\n blobCountNew++;\n }\n size += blob.length;\n }\n\n return { blobCountNew, blobCount: sorted.length, size, sizeNew };\n}\n\nasync function saveSnapshot(name: string, fetchedData: IFetchedData[], saveDir: string) {\n const outDir = `${saveDir}/${name}/`;\n const mkdir = util.promisify(fs.mkdir);\n\n await mkdir(`${outDir}/decoded`, { recursive: true });\n await Promise.all(fetchedData.map(async (item) => {\n const buffer = await item.blob;\n if (buffer === undefined) {\n console.error(`ERROR: Unable to get data for blob ${item.blobId}`);\n return;\n }\n\n if (!isFetchedTree(item)) {\n // Just write the data as is.\n fs.writeFileSync(`${outDir}/${item.filename}`, Buffer.from(buffer));\n\n // we assume that the buffer is utf8 here, which currently is true for\n // all of our snapshot blobs. It doesn't necessary be true in the future\n let decoded = bufferToString(buffer,\"utf8\");\n try {\n if (!paramActualFormatting) {\n decoded = JSON.stringify(JSON.parse(decoded), undefined, 2);\n }\n } catch (e) {\n }\n fs.writeFileSync(\n `${outDir}/decoded/${item.filename}.json`, decoded);\n } else {\n // Write out same data for tree decoded or not, except for formatting\n const treeString = bufferToString(buffer,\"utf8\");\n fs.writeFileSync(`${outDir}/${item.filename}.json`, treeString);\n fs.writeFileSync(`${outDir}/decoded/${item.filename}.json`,\n paramActualFormatting ? treeString : JSON.stringify(JSON.parse(treeString), undefined, 2));\n }\n }));\n}\n\nasync function fetchBlobsFromVersion(storage: IDocumentStorageService, version: IVersion) {\n const tree = await reportErrors(`getSnapshotTree ${version.id}`, storage.getSnapshotTree(version));\n if (!tree) {\n return Promise.reject(new Error(\"Failed to load snapshot tree\"));\n }\n return fetchBlobsFromSnapshotTree(storage, tree);\n}\n\nasync function reportErrors<T>(message: string, res: Promise<T>) {\n try {\n return await res;\n } catch (error) {\n console.error(`Error calling ${message}`);\n throw error;\n }\n}\n\nexport async function fluidFetchSnapshot(\n documentService?: IDocumentService,\n saveDir?: string,\n ) {\n if (!dumpSnapshotStats && !dumpSnapshotTrees && !dumpSnapshotVersions && saveDir === undefined) {\n return;\n }\n\n // --local mode - do not connect to storage.\n // For now, bail out early.\n // In future, separate download from analyzes parts and allow offline analyzes\n if (!documentService) {\n return;\n }\n\n console.log(\"\\n\");\n\n let storage = await documentService.connectToStorage();\n if (paramUnpackAggregatedBlobs) {\n storage = BlobAggregationStorage.wrap(storage, new TelemetryNullLogger(), false /* allowPacking */);\n }\n\n let version: IVersion | undefined;\n const versions = await reportErrors(\n `getVersions ${latestVersionsId}`,\n storage.getVersions(latestVersionsId, paramNumSnapshotVersions));\n if (dumpSnapshotVersions) {\n console.log(\"Snapshot versions\");\n console.log(versions);\n }\n\n let blobsToDump: IFetchedData[] | undefined;\n if (paramSnapshotVersionIndex !== undefined) {\n version = versions[paramSnapshotVersionIndex];\n if (version === undefined) {\n console.log(`There are only ${versions.length} snapshots, --snapshotVersionIndex is too large`);\n return;\n }\n if (saveDir !== undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n const name = version.id;\n console.log(`Saving snapshot ${name}`);\n await saveSnapshot(name, blobsToDump, saveDir);\n }\n } else {\n version = versions[0];\n if (saveDir !== undefined && versions.length > 0) {\n console.log(\" Name | Date | Size | New Size | Blobs | New Blobs\");\n console.log(\"-\".repeat(86));\n\n // Go in reverse order, to correctly calculate blob reuse - from oldest to newest snapshots\n for (let i = versions.length - 1; i >= 0; i--) {\n const v = versions[i];\n const blobs = await fetchBlobsFromVersion(storage, v);\n blobsToDump = blobs;\n const name = `${i}-${v.id}`;\n const res = await dumpSnapshotTree(name, blobs);\n\n let date = \"\";\n if (v.date) {\n try {\n date = new Date(v.date).toLocaleString();\n } catch (e) {\n date = v.date.replace(\"T\", \" \");\n const index = date.lastIndexOf(\".\");\n if (index > 0) {\n date = `${date.substr(0, index)} Z`;\n }\n }\n }\n date = date.padStart(23);\n const size = formatNumber(res.size).padStart(10);\n const sizeNew = formatNumber(res.sizeNew).padStart(10);\n const blobCount = formatNumber(res.blobCount).padStart(6);\n const blobCountNew = formatNumber(res.blobCountNew).padStart(9);\n\n console.log(`${name.padEnd(15)} | ${date} | ${size} | ${sizeNew} | ${blobCount} | ${blobCountNew}`);\n\n await saveSnapshot(name, blobs, saveDir);\n }\n }\n }\n\n if (dumpSnapshotStats || dumpSnapshotTrees) {\n if (version === undefined) {\n console.log(\"No snapshot tree\");\n } else {\n if (blobsToDump === undefined) {\n blobsToDump = await fetchBlobsFromVersion(storage, version);\n }\n console.log(`\\n\\nSnapshot version ${version.id}`);\n await dumpSnapshotTreeVerbose(version.id, blobsToDump);\n }\n }\n}\n"]}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluid-tools/fetch-tool",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.59.1000",
|
|
4
4
|
"description": "Console tool to fetch Fluid data from relay service",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": {
|
|
@@ -29,28 +29,28 @@
|
|
|
29
29
|
"tsfmt:fix": "tsfmt --replace"
|
|
30
30
|
},
|
|
31
31
|
"dependencies": {
|
|
32
|
-
"@fluid-tools/fluidapp-odsp-urlresolver": "^0.
|
|
33
|
-
"@fluidframework/azure-client": "^0.
|
|
32
|
+
"@fluid-tools/fluidapp-odsp-urlresolver": "^0.59.1000",
|
|
33
|
+
"@fluidframework/azure-client": "^0.59.1000",
|
|
34
34
|
"@fluidframework/common-utils": "^0.32.1",
|
|
35
|
-
"@fluidframework/container-runtime": "^0.
|
|
36
|
-
"@fluidframework/datastore": "^0.
|
|
37
|
-
"@fluidframework/driver-definitions": "^0.
|
|
38
|
-
"@fluidframework/driver-utils": "^0.
|
|
39
|
-
"@fluidframework/odsp-doclib-utils": "^0.
|
|
40
|
-
"@fluidframework/odsp-driver": "^0.
|
|
41
|
-
"@fluidframework/odsp-driver-definitions": "^0.
|
|
42
|
-
"@fluidframework/odsp-urlresolver": "^0.
|
|
43
|
-
"@fluidframework/protocol-definitions": "^0.
|
|
44
|
-
"@fluidframework/routerlicious-driver": "^0.
|
|
45
|
-
"@fluidframework/routerlicious-urlresolver": "^0.
|
|
46
|
-
"@fluidframework/runtime-definitions": "^0.
|
|
47
|
-
"@fluidframework/telemetry-utils": "^0.
|
|
48
|
-
"@fluidframework/test-client-utils": "^0.
|
|
49
|
-
"@fluidframework/tool-utils": "^0.
|
|
35
|
+
"@fluidframework/container-runtime": "^0.59.1000",
|
|
36
|
+
"@fluidframework/datastore": "^0.59.1000",
|
|
37
|
+
"@fluidframework/driver-definitions": "^0.46.1000",
|
|
38
|
+
"@fluidframework/driver-utils": "^0.59.1000",
|
|
39
|
+
"@fluidframework/odsp-doclib-utils": "^0.59.1000",
|
|
40
|
+
"@fluidframework/odsp-driver": "^0.59.1000",
|
|
41
|
+
"@fluidframework/odsp-driver-definitions": "^0.59.1000",
|
|
42
|
+
"@fluidframework/odsp-urlresolver": "^0.59.1000",
|
|
43
|
+
"@fluidframework/protocol-definitions": "^0.1028.1000",
|
|
44
|
+
"@fluidframework/routerlicious-driver": "^0.59.1000",
|
|
45
|
+
"@fluidframework/routerlicious-urlresolver": "^0.59.1000",
|
|
46
|
+
"@fluidframework/runtime-definitions": "^0.59.1000",
|
|
47
|
+
"@fluidframework/telemetry-utils": "^0.59.1000",
|
|
48
|
+
"@fluidframework/test-client-utils": "^0.59.1000",
|
|
49
|
+
"@fluidframework/tool-utils": "^0.59.1000"
|
|
50
50
|
},
|
|
51
51
|
"devDependencies": {
|
|
52
52
|
"@fluidframework/build-common": "^0.23.0",
|
|
53
|
-
"@fluidframework/eslint-config-fluid": "^0.
|
|
53
|
+
"@fluidframework/eslint-config-fluid": "^0.28.1000",
|
|
54
54
|
"@rushstack/eslint-config": "^2.5.1",
|
|
55
55
|
"@types/node": "^14.18.0",
|
|
56
56
|
"@typescript-eslint/eslint-plugin": "~5.9.0",
|
package/src/fluidFetchInit.ts
CHANGED
|
@@ -84,7 +84,11 @@ async function initializeODSPCore(
|
|
|
84
84
|
opsBatchSize: 20000,
|
|
85
85
|
concurrentOpsBatches: 4,
|
|
86
86
|
});
|
|
87
|
-
return odspDocumentServiceFactory.createDocumentService(
|
|
87
|
+
return odspDocumentServiceFactory.createDocumentService(
|
|
88
|
+
odspResolvedUrl,
|
|
89
|
+
undefined, // logger
|
|
90
|
+
false, // clientIsSummrizer
|
|
91
|
+
);
|
|
88
92
|
}
|
|
89
93
|
|
|
90
94
|
async function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IFluidResolvedUrl) {
|
|
@@ -115,7 +119,11 @@ async function initializeR11s(server: string, pathname: string, r11sResolvedUrl:
|
|
|
115
119
|
console.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);
|
|
116
120
|
const tokenProvider = new r11s.DefaultTokenProvider(paramJWT);
|
|
117
121
|
const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
|
|
118
|
-
return r11sDocumentServiceFactory.createDocumentService(
|
|
122
|
+
return r11sDocumentServiceFactory.createDocumentService(
|
|
123
|
+
r11sResolvedUrl,
|
|
124
|
+
undefined, // logger
|
|
125
|
+
false, // clientIsSummarizer
|
|
126
|
+
);
|
|
119
127
|
}
|
|
120
128
|
|
|
121
129
|
async function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string) {
|
|
@@ -128,7 +136,11 @@ async function initializeAzure(resolvedUrl: IFluidResolvedUrl, tenantId: string)
|
|
|
128
136
|
const user = generateTestUser();
|
|
129
137
|
const tokenProvider = new InsecureTokenProvider(paramAzureKey, user);
|
|
130
138
|
const r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);
|
|
131
|
-
return r11sDocumentServiceFactory.createDocumentService(
|
|
139
|
+
return r11sDocumentServiceFactory.createDocumentService(
|
|
140
|
+
resolvedUrl,
|
|
141
|
+
undefined, // logger
|
|
142
|
+
false, // clientIsSummarizer
|
|
143
|
+
);
|
|
132
144
|
}
|
|
133
145
|
|
|
134
146
|
async function resolveUrl(url: string): Promise<IResolvedUrl | undefined> {
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
|
|
6
6
|
import fs from "fs";
|
|
7
7
|
import util from "util";
|
|
8
|
-
import {
|
|
8
|
+
import { bufferToString, stringToBuffer, TelemetryNullLogger } from "@fluidframework/common-utils";
|
|
9
9
|
import {
|
|
10
10
|
IDocumentService,
|
|
11
11
|
IDocumentStorageService,
|
|
@@ -114,11 +114,9 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
114
114
|
storage: IDocumentStorageService,
|
|
115
115
|
tree: ISnapshotTree,
|
|
116
116
|
prefix: string = "/",
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
const commit = !perCommitBlobIdMap;
|
|
121
|
-
if (commit && dumpSnapshotTrees) {
|
|
117
|
+
parentBlobIdMap?: Map<string, number>): Promise<IFetchedData[]> {
|
|
118
|
+
const isTopLevel = !parentBlobIdMap;
|
|
119
|
+
if (isTopLevel && dumpSnapshotTrees) {
|
|
122
120
|
console.log(tree);
|
|
123
121
|
}
|
|
124
122
|
|
|
@@ -128,42 +126,16 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
128
126
|
}
|
|
129
127
|
|
|
130
128
|
// Create the tree info before fetching blobs (which will modify it)
|
|
131
|
-
let
|
|
132
|
-
if (
|
|
133
|
-
|
|
129
|
+
let topLevelBlob: IFetchedTree | undefined;
|
|
130
|
+
if (isTopLevel) {
|
|
131
|
+
topLevelBlob = createTreeBlob(tree, prefix, false);
|
|
134
132
|
}
|
|
135
133
|
|
|
136
|
-
const blobIdMap =
|
|
134
|
+
const blobIdMap = parentBlobIdMap ?? new Map<string, number>();
|
|
137
135
|
let result: IFetchedData[] = fetchBlobs(prefix, tree, storage, blobIdMap);
|
|
138
136
|
|
|
139
|
-
for (const dataStore of Object.keys(tree.commits)) {
|
|
140
|
-
const dataStoreVersions = await storage.getVersions(tree.commits[dataStore], 1);
|
|
141
|
-
if (dataStoreVersions.length !== 1) {
|
|
142
|
-
console.error(`ERROR: Unable to get versions for ${dataStore}`);
|
|
143
|
-
continue;
|
|
144
|
-
}
|
|
145
|
-
const dataStoreSnapShotTree = await reportErrors(
|
|
146
|
-
`getSnapshotTree ${dataStoreVersions[0].id}`,
|
|
147
|
-
storage.getSnapshotTree(dataStoreVersions[0]));
|
|
148
|
-
if (dataStoreSnapShotTree === null) {
|
|
149
|
-
// eslint-disable-next-line max-len
|
|
150
|
-
console.error(`No data store tree for data store = ${dataStore}, path = ${prefix}, version = ${dataStoreVersions[0].id}`);
|
|
151
|
-
continue;
|
|
152
|
-
}
|
|
153
|
-
assert(dataStoreSnapShotTree.id === undefined || dataStoreSnapShotTree.id === tree.commits[dataStore],
|
|
154
|
-
0x1bf /* `Unexpected id for tree: ${dataStoreSnapShotTree.id}` */);
|
|
155
|
-
assert(tree.commits[dataStore] === dataStoreVersions[0].id,
|
|
156
|
-
0x1c0 /* "Mismatch between commit id and fetched tree id" */);
|
|
157
|
-
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(
|
|
158
|
-
storage,
|
|
159
|
-
dataStoreSnapShotTree,
|
|
160
|
-
`${prefix}[${dataStore}]/`);
|
|
161
|
-
result = result.concat(dataStoreBlobs);
|
|
162
|
-
}
|
|
163
|
-
|
|
164
137
|
for (const subtreeId of Object.keys(tree.trees)) {
|
|
165
138
|
const subtree = tree.trees[subtreeId];
|
|
166
|
-
assert(Object.keys(subtree.commits).length === 0, 0x1c1 /* "Unexpected subtree properties" */);
|
|
167
139
|
const dataStoreBlobs = await fetchBlobsFromSnapshotTree(
|
|
168
140
|
storage,
|
|
169
141
|
subtree,
|
|
@@ -171,8 +143,8 @@ async function fetchBlobsFromSnapshotTree(
|
|
|
171
143
|
result = result.concat(dataStoreBlobs);
|
|
172
144
|
}
|
|
173
145
|
|
|
174
|
-
if (
|
|
175
|
-
result.push(
|
|
146
|
+
if (topLevelBlob) {
|
|
147
|
+
result.push(topLevelBlob);
|
|
176
148
|
result.push(createTreeBlob(tree, prefix, true));
|
|
177
149
|
}
|
|
178
150
|
return result;
|