@fluid-tools/fetch-tool 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +6 -8
- package/CHANGELOG.md +117 -0
- package/README.md +38 -7
- package/bin/fluid-fetch +0 -0
- package/dist/fluidAnalyzeMessages.d.ts.map +1 -1
- package/dist/fluidAnalyzeMessages.js +106 -116
- package/dist/fluidAnalyzeMessages.js.map +1 -1
- package/dist/fluidFetch.js +5 -3
- package/dist/fluidFetch.js.map +1 -1
- package/dist/fluidFetchArgs.d.ts +0 -3
- package/dist/fluidFetchArgs.d.ts.map +1 -1
- package/dist/fluidFetchArgs.js +10 -14
- package/dist/fluidFetchArgs.js.map +1 -1
- package/dist/fluidFetchInit.d.ts +0 -1
- package/dist/fluidFetchInit.d.ts.map +1 -1
- package/dist/fluidFetchInit.js +41 -34
- package/dist/fluidFetchInit.js.map +1 -1
- package/dist/fluidFetchMessages.d.ts.map +1 -1
- package/dist/fluidFetchMessages.js +168 -200
- package/dist/fluidFetchMessages.js.map +1 -1
- package/dist/fluidFetchSharePoint.d.ts +0 -1
- package/dist/fluidFetchSharePoint.d.ts.map +1 -1
- package/dist/fluidFetchSharePoint.js +20 -6
- package/dist/fluidFetchSharePoint.js.map +1 -1
- package/dist/fluidFetchSnapshot.d.ts.map +1 -1
- package/dist/fluidFetchSnapshot.js +18 -20
- package/dist/fluidFetchSnapshot.js.map +1 -1
- package/package.json +47 -42
- package/prettier.config.cjs +8 -0
- package/src/fluidAnalyzeMessages.ts +701 -630
- package/src/fluidFetch.ts +93 -88
- package/src/fluidFetchArgs.ts +167 -168
- package/src/fluidFetchInit.ts +133 -104
- package/src/fluidFetchMessages.ts +253 -232
- package/src/fluidFetchSharePoint.ts +130 -112
- package/src/fluidFetchSnapshot.ts +313 -295
- package/tsconfig.json +8 -15
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchInit.js","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":";AAAA;;;GAGG
|
|
1
|
+
{"version":3,"file":"fluidFetchInit.js","sourceRoot":"","sources":["../src/fluidFetchInit.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,6BAA0B;AAG1B,sFAAiF;AAEjF,kEAAoD;AAKpD,uEAAmE;AACnE,2EAA6D;AAC7D,yFAAqF;AACrF,2DAAuE;AACvE,qDAA2D;AAC3D,iEAAwD;AAE7C,QAAA,gBAAgB,GAAW,EAAE,CAAC;AAGzC,KAAK,UAAU,kBAAkB,CAChC,eAAiC,EACjC,MAAc,EACd,YAA2B;IAE3B,MAAM,EAAE,OAAO,EAAE,MAAM,EAAE,GAAG,eAAe,CAAC;IAE5C,sBAAc,GAAG;QAChB,MAAM;QACN,KAAK,EAAE,OAAO;QACd,IAAI,EAAE,MAAM;KACZ,CAAC;IAEF,IAAI,8BAAa,EAAE;QAClB,OAAO;KACP;IAED,MAAM,KAAK,GAAG,MAAM,IAAI,CAAC,mBAAmB,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;IAE9D,OAAO,CAAC,GAAG,CAAC;YACD,MAAM;YACN,OAAO;YACP,MAAM;YACN,KAAK,EAAE,CAAC,CAAC;IAEpB,MAAM,mBAAmB,GAAG,KAAK,EAAE,OAAsC,EAAE,EAAE;QAC5E,OAAO,IAAA,qCAAc,EACpB,KAAK,EAAE,eAAqC,EAAE,EAAE;YAC/C,IACC,CAAC,OAAO,CAAC,OAAO,IAAI,CAAC,eAAe,CAAC,WAAW,CAAC;gBACjD,eAAe,CAAC,cAAc,EAC7B;gBACD,OAAO,eAAe,CAAC,cAAc,EAAE,CAAC;aACxC;YACD,OAAO,eAAe,CAAC,WAAW,CAAC;QACpC,CAAC,EACD,MAAM,EACN,YAAY,EACZ,SAAS,EACT,IAAI,CACJ,CAAC;IACH,CAAC,CAAC;IACF,qEAAqE;IACrE,MAAM,qBAAqB,GAAG,CAAC,QAAuC,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;IAC/F,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,0BAA0B,CACrE,mBAAmB,EACnB,qBAAqB,EACrB,SAAS,EACT;QACC,YAAY,EAAE,KAAK;QACnB,oBAAoB,EAAE,CAAC;KACvB,CACD,CAAC;IACF,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC1E,CAAC;AAED,KAAK,UAAU,cAAc,CAAC,MAAc,EAAE,QAAgB,EAAE,eAA6B;IAC5F,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;IACjC,IAAI,QAAgB,CAAC;IACrB,IAAI,UAAkB,CAAC;IACvB,IAAI,MAAM,KAAK,WAAW,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE;QAC9C,QAAQ,GAAG,OAAO,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACrB;SAAM;QACN,QAAQ,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;QACnB,UAAU,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC;KACrB;IAED,+CAA+C;IAC/C,wBAAgB,GAAG,UAAU,CAAC;IAE9B,sBAAc,GAAG;QAChB,MAAM;QACN,QAAQ;QACR,EAAE,EAAE,UAAU;KACd,CAAC;IAEF,IAAI,8BAAa,EAAE;QAClB,OAAO;KACP;IAED,OAAO,CAAC,GAAG,CAAC,gCAAgC,QAAQ,OAAO,UAAU,EAAE,CAAC,CAAC;IACzE,MAAM,aAAa,GAAG,IAAI,IAAI,CAAC,oBAAoB,CAAC,yBAAQ,CAAC,CAAC;IAC9D,MAAM,0BAA0B,GAAG,IAAI,IAAI,CAAC,mCAAmC,CAAC,aAAa,CAAC,CAAC;IAC/F,OAAO,0BAA0B,CAAC,qBAAqB,CAAC,eAAe,CAAC,CAAC;AAC1E,CAAC;AAMD,KAAK,UAAU,UAAU,CAAC,GAAW;IACpC,MAAM,OAAO,GAAa,EAAE,GAAG,EAAE,CAAC;IAClC,IAAI,gBAA0C,CAAC;IAE/C,wFAAwF;IACxF,gBAAgB,GAAG,MAAM,IAAI,kCAAe,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IAChE,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,gBAAgB,GAAG,MAAM,IAAI,mDAAuB,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACxE,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,gBAAgB,GAAG,MAAM,IAAI,oDAAwB,CACpD,SAAS,EACT,KAAK,IAAI,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,yBAAQ,CAAC,EACrC,EAAE,CACF,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,IAAI,gBAAgB,KAAK,SAAS,EAAE;QACnC,OAAO;YACN,WAAW,EAAE,gBAAgB;YAC7B,WAAW,EAAE,MAAM;SACnB,CAAC;KACF;IAED,OAAO,SAAS,CAAC;AAClB,CAAC;AAEM,KAAK,UAAU,cAAc,CAAC,MAAc;IAClD,MAAM,YAAY,GAAG,MAAM,UAAU,CAAC,MAAM,CAAC,CAAC;IAC9C,IAAI,YAAY,KAAK,SAAS,EAAE;QAC/B,MAAM,IAAI,KAAK,CAAC,eAAe,MAAM,EAAE,CAAC,CAAC;KACzC;IACD,MAAM,gBAAgB,GAAG,YAAY,CAAC,WAAW,CAAC;IAClD,IAAI,YAAY,CAAC,WAAW,KAAK,MAAM,EAAE;QACxC,MAAM,eAAe,GAAG,gBAAoC,CAAC;QAC7D,OAAO,kBAAkB,CACxB,eAAe,EACf,IAAI,SAAG,CAAC,eAAe,CAAC,OAAO,CAAC,CAAC,IAAI,EACrC,IAAA,sCAAyB,GAAE,CAC3B,CAAC;KACF;SAAM,IAAI,YAAY,CAAC,WAAW,KAAK,MAAM,EAAE;QAC/C,MAAM,GAAG,GAAG,IAAI,SAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,MAAM,MAAM,GAAG,GAAG,CAAC,QAAQ,CAAC,WAAW,EAAE,CAAC;QAC1C,OAAO,cAAc,CAAC,MAAM,EAAE,GAAG,CAAC,QAAQ,EAAE,gBAAgB,CAAC,CAAC;KAC9D;AACF,CAAC;AAlBD,wCAkBC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { URL } from \"url\";\nimport { IRequest } from \"@fluidframework/core-interfaces\";\nimport { IResolvedUrl } from \"@fluidframework/driver-definitions\";\nimport { FluidAppOdspUrlResolver } from \"@fluid-tools/fluidapp-odsp-urlresolver\";\nimport { IClientConfig, IOdspAuthRequestInfo } from \"@fluidframework/odsp-doclib-utils\";\nimport * as odsp from \"@fluidframework/odsp-driver\";\nimport {\n\tIOdspResolvedUrl,\n\tOdspResourceTokenFetchOptions,\n} from \"@fluidframework/odsp-driver-definitions\";\nimport { OdspUrlResolver } from \"@fluidframework/odsp-urlresolver\";\nimport * as r11s from \"@fluidframework/routerlicious-driver\";\nimport { RouterliciousUrlResolver } from \"@fluidframework/routerlicious-urlresolver\";\nimport { getMicrosoftConfiguration } from \"@fluidframework/tool-utils\";\nimport { localDataOnly, paramJWT } from \"./fluidFetchArgs\";\nimport { resolveWrapper } from \"./fluidFetchSharePoint\";\n\nexport let latestVersionsId: string = \"\";\nexport let connectionInfo: any;\n\nasync function initializeODSPCore(\n\todspResolvedUrl: IOdspResolvedUrl,\n\tserver: string,\n\tclientConfig: IClientConfig,\n) {\n\tconst { driveId, itemId } = odspResolvedUrl;\n\n\tconnectionInfo = {\n\t\tserver,\n\t\tdrive: driveId,\n\t\titem: itemId,\n\t};\n\n\tif (localDataOnly) {\n\t\treturn;\n\t}\n\n\tconst docId = await odsp.getHashedDocumentId(driveId, itemId);\n\n\tconsole.log(`Connecting to ODSP:\n server: ${server}\n drive: ${driveId}\n item: ${itemId}\n docId: ${docId}`);\n\n\tconst getStorageTokenStub = async (options: OdspResourceTokenFetchOptions) => {\n\t\treturn resolveWrapper(\n\t\t\tasync (authRequestInfo: IOdspAuthRequestInfo) => {\n\t\t\t\tif (\n\t\t\t\t\t(options.refresh || !authRequestInfo.accessToken) &&\n\t\t\t\t\tauthRequestInfo.refreshTokenFn\n\t\t\t\t) {\n\t\t\t\t\treturn authRequestInfo.refreshTokenFn();\n\t\t\t\t}\n\t\t\t\treturn authRequestInfo.accessToken;\n\t\t\t},\n\t\t\tserver,\n\t\t\tclientConfig,\n\t\t\tundefined,\n\t\t\ttrue,\n\t\t);\n\t};\n\t// eslint-disable-next-line @typescript-eslint/promise-function-async\n\tconst getWebsocketTokenStub = (_options: OdspResourceTokenFetchOptions) => Promise.resolve(\"\");\n\tconst odspDocumentServiceFactory = new odsp.OdspDocumentServiceFactory(\n\t\tgetStorageTokenStub,\n\t\tgetWebsocketTokenStub,\n\t\tundefined,\n\t\t{\n\t\t\topsBatchSize: 20000,\n\t\t\tconcurrentOpsBatches: 4,\n\t\t},\n\t);\n\treturn odspDocumentServiceFactory.createDocumentService(odspResolvedUrl);\n}\n\nasync function initializeR11s(server: string, pathname: string, r11sResolvedUrl: IResolvedUrl) {\n\tconst path = pathname.split(\"/\");\n\tlet tenantId: string;\n\tlet documentId: string;\n\tif (server === \"localhost\" && path.length < 4) {\n\t\ttenantId = \"fluid\";\n\t\tdocumentId = path[2];\n\t} else {\n\t\ttenantId = path[2];\n\t\tdocumentId = path[3];\n\t}\n\n\t// Latest version id is the documentId for r11s\n\tlatestVersionsId = documentId;\n\n\tconnectionInfo = {\n\t\tserver,\n\t\ttenantId,\n\t\tid: documentId,\n\t};\n\n\tif (localDataOnly) {\n\t\treturn;\n\t}\n\n\tconsole.log(`Connecting to r11s: tenantId=${tenantId} id:${documentId}`);\n\tconst tokenProvider = new r11s.DefaultTokenProvider(paramJWT);\n\tconst r11sDocumentServiceFactory = new r11s.RouterliciousDocumentServiceFactory(tokenProvider);\n\treturn r11sDocumentServiceFactory.createDocumentService(r11sResolvedUrl);\n}\n\ninterface IResolvedInfo {\n\tresolvedUrl: IResolvedUrl;\n\tserviceType: \"odsp\" | \"r11s\";\n}\nasync function resolveUrl(url: string): Promise<IResolvedInfo | undefined> {\n\tconst request: IRequest = { url };\n\tlet maybeResolvedUrl: IResolvedUrl | undefined;\n\n\t// Try each url resolver in turn to figure out which one the request is compatible with.\n\tmaybeResolvedUrl = await new OdspUrlResolver().resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"odsp\",\n\t\t};\n\t}\n\n\tmaybeResolvedUrl = await new FluidAppOdspUrlResolver().resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"odsp\",\n\t\t};\n\t}\n\n\tmaybeResolvedUrl = await new RouterliciousUrlResolver(\n\t\tundefined,\n\t\tasync () => Promise.resolve(paramJWT),\n\t\t\"\",\n\t).resolve(request);\n\tif (maybeResolvedUrl !== undefined) {\n\t\treturn {\n\t\t\tresolvedUrl: maybeResolvedUrl,\n\t\t\tserviceType: \"r11s\",\n\t\t};\n\t}\n\n\treturn undefined;\n}\n\nexport async function fluidFetchInit(urlStr: string) {\n\tconst resolvedInfo = await resolveUrl(urlStr);\n\tif (resolvedInfo === undefined) {\n\t\tthrow new Error(`Unknown URL ${urlStr}`);\n\t}\n\tconst fluidResolvedUrl = resolvedInfo.resolvedUrl;\n\tif (resolvedInfo.serviceType === \"odsp\") {\n\t\tconst odspResolvedUrl = fluidResolvedUrl as IOdspResolvedUrl;\n\t\treturn initializeODSPCore(\n\t\t\todspResolvedUrl,\n\t\t\tnew URL(odspResolvedUrl.siteUrl).host,\n\t\t\tgetMicrosoftConfiguration(),\n\t\t);\n\t} else if (resolvedInfo.serviceType === \"r11s\") {\n\t\tconst url = new URL(urlStr);\n\t\tconst server = url.hostname.toLowerCase();\n\t\treturn initializeR11s(server, url.pathname, fluidResolvedUrl);\n\t}\n}\n"]}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchMessages.d.ts","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,
|
|
1
|
+
{"version":3,"file":"fluidFetchMessages.d.ts","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAqQtE,wBAAsB,kBAAkB,CAAC,eAAe,CAAC,EAAE,gBAAgB,EAAE,OAAO,CAAC,EAAE,MAAM,iBAgC5F"}
|
|
@@ -3,32 +3,13 @@
|
|
|
3
3
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
4
4
|
* Licensed under the MIT License.
|
|
5
5
|
*/
|
|
6
|
-
var __await = (this && this.__await) || function (v) { return this instanceof __await ? (this.v = v, this) : new __await(v); }
|
|
7
|
-
var __asyncGenerator = (this && this.__asyncGenerator) || function (thisArg, _arguments, generator) {
|
|
8
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
9
|
-
var g = generator.apply(thisArg, _arguments || []), i, q = [];
|
|
10
|
-
return i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i;
|
|
11
|
-
function verb(n) { if (g[n]) i[n] = function (v) { return new Promise(function (a, b) { q.push([n, v, a, b]) > 1 || resume(n, v); }); }; }
|
|
12
|
-
function resume(n, v) { try { step(g[n](v)); } catch (e) { settle(q[0][3], e); } }
|
|
13
|
-
function step(r) { r.value instanceof __await ? Promise.resolve(r.value.v).then(fulfill, reject) : settle(q[0][2], r); }
|
|
14
|
-
function fulfill(value) { resume("next", value); }
|
|
15
|
-
function reject(value) { resume("throw", value); }
|
|
16
|
-
function settle(f, v) { if (f(v), q.shift(), q.length) resume(q[0][0], q[0][1]); }
|
|
17
|
-
};
|
|
18
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
19
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
20
|
-
var m = o[Symbol.asyncIterator], i;
|
|
21
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
22
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
23
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
24
|
-
};
|
|
25
6
|
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
7
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
8
|
};
|
|
28
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
10
|
exports.fluidFetchMessages = void 0;
|
|
30
11
|
const fs_1 = __importDefault(require("fs"));
|
|
31
|
-
const
|
|
12
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
32
13
|
const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
33
14
|
const fluidAnalyzeMessages_1 = require("./fluidAnalyzeMessages");
|
|
34
15
|
const fluidFetchArgs_1 = require("./fluidFetchArgs");
|
|
@@ -36,200 +17,197 @@ function filenameFromIndex(index) {
|
|
|
36
17
|
return index === 0 ? "" : index.toString(); // support old tools...
|
|
37
18
|
}
|
|
38
19
|
let firstAvailableDelta = 1;
|
|
39
|
-
function loadAllSequencedMessages(documentService, dir, files) {
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
}
|
|
66
|
-
break;
|
|
20
|
+
async function* loadAllSequencedMessages(documentService, dir, files) {
|
|
21
|
+
let lastSeq = 0;
|
|
22
|
+
// flag for mismatch between last sequence number read and new one to be read
|
|
23
|
+
let seqNumMismatch = false;
|
|
24
|
+
// If we have local save, read ops from there first
|
|
25
|
+
if (files !== undefined) {
|
|
26
|
+
for (let i = 0; i < files.length; i++) {
|
|
27
|
+
const file = filenameFromIndex(i);
|
|
28
|
+
try {
|
|
29
|
+
console.log(`reading messages${file}.json`);
|
|
30
|
+
const fileContent = fs_1.default.readFileSync(`${dir}/messages${file}.json`, {
|
|
31
|
+
encoding: "utf-8",
|
|
32
|
+
});
|
|
33
|
+
const messages = JSON.parse(fileContent);
|
|
34
|
+
// check if there is mismatch
|
|
35
|
+
seqNumMismatch = messages[0].sequenceNumber !== lastSeq + 1;
|
|
36
|
+
(0, core_utils_1.assert)(!seqNumMismatch, 0x1b9 /* "Unexpected value for sequence number of first message in file" */);
|
|
37
|
+
lastSeq = messages[messages.length - 1].sequenceNumber;
|
|
38
|
+
yield messages;
|
|
39
|
+
}
|
|
40
|
+
catch (e) {
|
|
41
|
+
if (seqNumMismatch) {
|
|
42
|
+
if (fluidFetchArgs_1.overWrite) {
|
|
43
|
+
// with overWrite option on, we will delete all exisintg message.json files
|
|
44
|
+
for (let index = 0; index < files.length; index++) {
|
|
45
|
+
const name = filenameFromIndex(index);
|
|
46
|
+
fs_1.default.unlinkSync(`${dir}/messages${name}.json`);
|
|
67
47
|
}
|
|
68
|
-
|
|
69
|
-
console.error("There are deleted ops in the document being requested," +
|
|
70
|
-
" please back up the existing messages.json file and delete it from its directory." +
|
|
71
|
-
" Then try fetch tool again.");
|
|
72
|
-
console.error(e);
|
|
73
|
-
return yield __await(void 0);
|
|
74
|
-
}
|
|
75
|
-
else {
|
|
76
|
-
console.error(`Error reading / parsing messages from ${files}`);
|
|
77
|
-
console.error(e);
|
|
78
|
-
return yield __await(void 0);
|
|
48
|
+
break;
|
|
79
49
|
}
|
|
50
|
+
// prompt user to back up and delete existing files
|
|
51
|
+
console.error("There are deleted ops in the document being requested," +
|
|
52
|
+
" please back up the existing messages.json file and delete it from its directory." +
|
|
53
|
+
" Then try fetch tool again.");
|
|
54
|
+
console.error(e);
|
|
55
|
+
return;
|
|
56
|
+
}
|
|
57
|
+
else {
|
|
58
|
+
console.error(`Error reading / parsing messages from ${files}`);
|
|
59
|
+
console.error(e);
|
|
60
|
+
return;
|
|
80
61
|
}
|
|
81
|
-
}
|
|
82
|
-
if (lastSeq !== 0) {
|
|
83
|
-
console.log(`Read ${lastSeq} ops from local cache`);
|
|
84
62
|
}
|
|
85
63
|
}
|
|
86
|
-
if (
|
|
87
|
-
|
|
64
|
+
if (lastSeq !== 0) {
|
|
65
|
+
console.log(`Read ${lastSeq} ops from local cache`);
|
|
88
66
|
}
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
67
|
+
}
|
|
68
|
+
if (!documentService) {
|
|
69
|
+
return;
|
|
70
|
+
}
|
|
71
|
+
const deltaStorage = await documentService.connectToDeltaStorage();
|
|
72
|
+
let timeStart = Date.now();
|
|
73
|
+
let requests = 0;
|
|
74
|
+
let opsStorage = 0;
|
|
75
|
+
// reading only 1 op to test if there is mismatch
|
|
76
|
+
const teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);
|
|
77
|
+
let statusCode;
|
|
78
|
+
let innerMostErrorCode;
|
|
79
|
+
let response;
|
|
80
|
+
try {
|
|
81
|
+
await teststream.read();
|
|
82
|
+
}
|
|
83
|
+
catch (error) {
|
|
84
|
+
statusCode = error.getTelemetryProperties().statusCode;
|
|
85
|
+
innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;
|
|
86
|
+
// if there is gap between ops, catch the error and check it is the error we need
|
|
87
|
+
if (statusCode !== 410 || innerMostErrorCode !== "fluidDeltaDataNotAvailable") {
|
|
88
|
+
throw error;
|
|
100
89
|
}
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
90
|
+
// get firstAvailableDelta from the error response, and set current sequence number to that
|
|
91
|
+
response = JSON.parse(error.getTelemetryProperties().response);
|
|
92
|
+
firstAvailableDelta = response.error.firstAvailableDelta;
|
|
93
|
+
lastSeq = firstAvailableDelta - 1;
|
|
94
|
+
}
|
|
95
|
+
// continue reading rest of the ops
|
|
96
|
+
const stream = deltaStorage.fetchMessages(lastSeq + 1, // inclusive left
|
|
97
|
+
undefined);
|
|
98
|
+
while (true) {
|
|
99
|
+
const result = await stream.read();
|
|
100
|
+
if (result.done) {
|
|
101
|
+
break;
|
|
112
102
|
}
|
|
113
|
-
|
|
114
|
-
const
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
// console.log(`Loaded ops at ${messages[0].sequenceNumber}`);
|
|
126
|
-
// This parsing of message contents happens in delta manager. But when we analyze messages
|
|
127
|
-
// for message stats, we skip that path. So parsing of json contents needs to happen here.
|
|
128
|
-
for (const message of messages) {
|
|
129
|
-
if (typeof message.contents === "string"
|
|
130
|
-
&& message.contents !== ""
|
|
131
|
-
&& message.type !== protocol_definitions_1.MessageType.ClientLeave) {
|
|
132
|
-
message.contents = JSON.parse(message.contents);
|
|
133
|
-
}
|
|
103
|
+
requests++;
|
|
104
|
+
const messages = result.value;
|
|
105
|
+
// Empty buckets should never be returned
|
|
106
|
+
(0, core_utils_1.assert)(messages.length !== 0, 0x1ba /* "should not return empty buckets" */);
|
|
107
|
+
// console.log(`Loaded ops at ${messages[0].sequenceNumber}`);
|
|
108
|
+
// This parsing of message contents happens in delta manager. But when we analyze messages
|
|
109
|
+
// for message stats, we skip that path. So parsing of json contents needs to happen here.
|
|
110
|
+
for (const message of messages) {
|
|
111
|
+
if (typeof message.contents === "string" &&
|
|
112
|
+
message.contents !== "" &&
|
|
113
|
+
message.type !== protocol_definitions_1.MessageType.ClientLeave) {
|
|
114
|
+
message.contents = JSON.parse(message.contents);
|
|
134
115
|
}
|
|
135
|
-
opsStorage += messages.length;
|
|
136
|
-
lastSeq = messages[messages.length - 1].sequenceNumber;
|
|
137
|
-
yield yield __await(messages);
|
|
138
116
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
}
|
|
167
|
-
|
|
117
|
+
opsStorage += messages.length;
|
|
118
|
+
lastSeq = messages[messages.length - 1].sequenceNumber;
|
|
119
|
+
yield messages;
|
|
120
|
+
}
|
|
121
|
+
console.log(`\n${Math.floor((Date.now() - timeStart) / 1000)} seconds to retrieve ${opsStorage} ops in ${requests} requests`);
|
|
122
|
+
if (fluidFetchArgs_1.connectToWebSocket) {
|
|
123
|
+
let logMsg = "";
|
|
124
|
+
const client = {
|
|
125
|
+
mode: "write",
|
|
126
|
+
permission: [],
|
|
127
|
+
scopes: [protocol_definitions_1.ScopeType.DocRead, protocol_definitions_1.ScopeType.DocWrite, protocol_definitions_1.ScopeType.SummaryWrite],
|
|
128
|
+
details: {
|
|
129
|
+
capabilities: { interactive: true },
|
|
130
|
+
},
|
|
131
|
+
user: { id: "blah" },
|
|
132
|
+
};
|
|
133
|
+
console.log("Retrieving messages from web socket");
|
|
134
|
+
timeStart = Date.now();
|
|
135
|
+
const deltaStream = await documentService.connectToDeltaStream(client);
|
|
136
|
+
const initialMessages = deltaStream.initialMessages;
|
|
137
|
+
deltaStream.dispose();
|
|
138
|
+
console.log(`${Math.floor((Date.now() - timeStart) / 1000)} seconds to connect to web socket`);
|
|
139
|
+
if (initialMessages !== undefined) {
|
|
140
|
+
const lastSequenceNumber = lastSeq;
|
|
141
|
+
const filtered = initialMessages.filter((a) => a.sequenceNumber > lastSequenceNumber);
|
|
142
|
+
const sorted = filtered.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
143
|
+
lastSeq = sorted[sorted.length - 1].sequenceNumber;
|
|
144
|
+
logMsg = ` (${opsStorage} delta storage, ${initialMessages.length} initial ws messages, ${initialMessages.length - sorted.length} dup)`;
|
|
145
|
+
yield sorted;
|
|
168
146
|
}
|
|
169
|
-
|
|
147
|
+
console.log(`${lastSeq} total messages${logMsg}`);
|
|
148
|
+
}
|
|
170
149
|
}
|
|
171
|
-
function saveOps(gen, // AsyncGenerator<ISequencedDocumentMessage[]>,
|
|
150
|
+
async function* saveOps(gen, // AsyncGenerator<ISequencedDocumentMessage[]>,
|
|
172
151
|
dir, files) {
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
152
|
+
// Split into 100K ops
|
|
153
|
+
const chunk = 100 * 1000;
|
|
154
|
+
let sequencedMessages = [];
|
|
155
|
+
// Figure out first file we want to write to
|
|
156
|
+
let index = 0;
|
|
157
|
+
let curr = 1;
|
|
158
|
+
if (files.length !== 0) {
|
|
159
|
+
index = files.length - 1;
|
|
160
|
+
const name = filenameFromIndex(index);
|
|
161
|
+
const fileContent = fs_1.default.readFileSync(`${dir}/messages${name}.json`, { encoding: "utf-8" });
|
|
162
|
+
const messages = JSON.parse(fileContent);
|
|
163
|
+
curr = messages[0].sequenceNumber;
|
|
164
|
+
}
|
|
165
|
+
while (true) {
|
|
166
|
+
const result = await gen.next();
|
|
167
|
+
if (files.length === 0) {
|
|
168
|
+
curr = firstAvailableDelta;
|
|
186
169
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
let messages = result.value;
|
|
194
|
-
yield yield __await(messages);
|
|
195
|
-
if (messages[messages.length - 1].sequenceNumber < curr) {
|
|
196
|
-
// Nothing interesting.
|
|
197
|
-
continue;
|
|
198
|
-
}
|
|
199
|
-
if (messages[0].sequenceNumber < curr) {
|
|
200
|
-
messages = messages.filter((msg) => msg.sequenceNumber >= curr);
|
|
201
|
-
}
|
|
202
|
-
sequencedMessages = sequencedMessages.concat(messages);
|
|
203
|
-
(0, common_utils_1.assert)(sequencedMessages[0].sequenceNumber === curr, 0x1bb /* "Unexpected sequence number on first of messages to save" */);
|
|
204
|
-
(0, common_utils_1.assert)(sequencedMessages[sequencedMessages.length - 1].sequenceNumber
|
|
205
|
-
=== curr + sequencedMessages.length - 1, 0x1bc /* "Unexpected sequence number on last of messages to save" */);
|
|
206
|
-
}
|
|
207
|
-
// Time to write it out?
|
|
208
|
-
while (sequencedMessages.length >= chunk || (result.done === true && sequencedMessages.length !== 0)) {
|
|
209
|
-
const name = filenameFromIndex(index);
|
|
210
|
-
const write = sequencedMessages.splice(0, chunk);
|
|
211
|
-
console.log(`writing messages${name}.json`);
|
|
212
|
-
fs_1.default.writeFileSync(`${dir}/messages${name}.json`, JSON.stringify(write, undefined, fluidFetchArgs_1.paramActualFormatting ? 0 : 2));
|
|
213
|
-
// increment curr by chunk
|
|
214
|
-
curr += chunk;
|
|
215
|
-
(0, common_utils_1.assert)(sequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr, 0x1bd /* "Stopped writing at unexpected sequence number" */);
|
|
216
|
-
index++;
|
|
170
|
+
if (result.done !== true) {
|
|
171
|
+
let messages = result.value;
|
|
172
|
+
yield messages;
|
|
173
|
+
if (messages[messages.length - 1].sequenceNumber < curr) {
|
|
174
|
+
// Nothing interesting.
|
|
175
|
+
continue;
|
|
217
176
|
}
|
|
218
|
-
if (
|
|
219
|
-
|
|
177
|
+
if (messages[0].sequenceNumber < curr) {
|
|
178
|
+
messages = messages.filter((msg) => msg.sequenceNumber >= curr);
|
|
220
179
|
}
|
|
180
|
+
sequencedMessages = sequencedMessages.concat(messages);
|
|
181
|
+
(0, core_utils_1.assert)(sequencedMessages[0].sequenceNumber === curr, 0x1bb /* "Unexpected sequence number on first of messages to save" */);
|
|
182
|
+
(0, core_utils_1.assert)(sequencedMessages[sequencedMessages.length - 1].sequenceNumber ===
|
|
183
|
+
curr + sequencedMessages.length - 1, 0x1bc /* "Unexpected sequence number on last of messages to save" */);
|
|
221
184
|
}
|
|
222
|
-
|
|
185
|
+
// Time to write it out?
|
|
186
|
+
while (sequencedMessages.length >= chunk ||
|
|
187
|
+
(result.done === true && sequencedMessages.length !== 0)) {
|
|
188
|
+
const name = filenameFromIndex(index);
|
|
189
|
+
const write = sequencedMessages.splice(0, chunk);
|
|
190
|
+
console.log(`writing messages${name}.json`);
|
|
191
|
+
fs_1.default.writeFileSync(`${dir}/messages${name}.json`, JSON.stringify(write, undefined, fluidFetchArgs_1.paramActualFormatting ? 0 : 2));
|
|
192
|
+
// increment curr by chunk
|
|
193
|
+
curr += chunk;
|
|
194
|
+
(0, core_utils_1.assert)(sequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr, 0x1bd /* "Stopped writing at unexpected sequence number" */);
|
|
195
|
+
index++;
|
|
196
|
+
}
|
|
197
|
+
if (result.done === true) {
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
}
|
|
223
201
|
}
|
|
224
202
|
async function fluidFetchMessages(documentService, saveDir) {
|
|
225
|
-
var e_1, _a;
|
|
226
203
|
const messageStats = fluidFetchArgs_1.dumpMessageStats || fluidFetchArgs_1.dumpMessages;
|
|
227
204
|
if (!messageStats && (saveDir === undefined || documentService === undefined)) {
|
|
228
205
|
return;
|
|
229
206
|
}
|
|
230
207
|
const files = saveDir === undefined
|
|
231
208
|
? undefined
|
|
232
|
-
: fs_1.default
|
|
209
|
+
: fs_1.default
|
|
210
|
+
.readdirSync(saveDir)
|
|
233
211
|
.filter((file) => {
|
|
234
212
|
if (!file.startsWith("messages")) {
|
|
235
213
|
return false;
|
|
@@ -246,17 +224,7 @@ async function fluidFetchMessages(documentService, saveDir) {
|
|
|
246
224
|
}
|
|
247
225
|
else {
|
|
248
226
|
let item;
|
|
249
|
-
|
|
250
|
-
for (var generator_1 = __asyncValues(generator), generator_1_1; generator_1_1 = await generator_1.next(), !generator_1_1.done;) {
|
|
251
|
-
item = generator_1_1.value;
|
|
252
|
-
}
|
|
253
|
-
}
|
|
254
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
255
|
-
finally {
|
|
256
|
-
try {
|
|
257
|
-
if (generator_1_1 && !generator_1_1.done && (_a = generator_1.return)) await _a.call(generator_1);
|
|
258
|
-
}
|
|
259
|
-
finally { if (e_1) throw e_1.error; }
|
|
227
|
+
for await (item of generator) {
|
|
260
228
|
}
|
|
261
229
|
}
|
|
262
230
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchMessages.js","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;;;;;;;;;;;;;;;;;;;;AAEH,4CAAoB;AACpB,+DAAsD;AAItD,+EAK8C;AAC9C,iEAA2D;AAC3D,qDAO0B;AAE1B,SAAS,iBAAiB,CAAC,KAAa;IACpC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,uBAAuB;AACvE,CAAC;AAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAC5B,SAAgB,wBAAwB,CACpC,eAAkC,EAClC,GAAY,EACZ,KAAgB;;QAChB,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,6EAA6E;QAC7E,IAAI,cAAc,GAAG,KAAK,CAAC;QAE3B,mDAAmD;QACnD,IAAI,KAAK,KAAK,SAAS,EAAE;YACrB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;gBACnC,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;gBAClC,IAAI;oBACA,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;oBAC5C,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;oBAC1F,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;oBACtE,6BAA6B;oBAC7B,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,OAAO,GAAG,CAAC,CAAC;oBAC5D,IAAA,qBAAM,EAAC,CAAC,cAAc,EAAE,KAAK,CAAC,qEAAqE,CAAC,CAAC;oBACrG,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;oBACvD,oBAAM,QAAQ,CAAA,CAAC;iBAClB;gBAAC,OAAO,CAAC,EAAE;oBACR,IAAI,cAAc,EAAE;wBAChB,IAAI,0BAAS,EAAE;4BACX,2EAA2E;4BAC3E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;gCAC/C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;gCACtC,YAAE,CAAC,UAAU,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,CAAC;6BAChD;4BACD,MAAM;yBACT;wBACD,mDAAmD;wBACnD,OAAO,CAAC,KAAK,CAAC,wDAAwD;4BAClE,mFAAmF;4BACnF,6BAA6B,CAAC,CAAC;wBACnC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBACjB,6BAAO;qBACV;yBAAM;wBACH,OAAO,CAAC,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;wBAChE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;wBACjB,6BAAO;qBACV;iBACJ;aACJ;YACD,IAAI,OAAO,KAAK,CAAC,EAAE;gBACf,OAAO,CAAC,GAAG,CAAC,QAAQ,OAAO,uBAAuB,CAAC,CAAC;aACvD;SACJ;QAED,IAAI,CAAC,eAAe,EAAE;YAClB,6BAAO;SACV;QAED,MAAM,YAAY,GAAG,cAAM,eAAe,CAAC,qBAAqB,EAAE,CAAA,CAAC;QAEnE,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;QACjB,IAAI,UAAU,GAAG,CAAC,CAAC;QAEnB,iDAAiD;QACjD,MAAM,UAAU,GAAG,YAAY,CAAC,aAAa,CACzC,OAAO,GAAG,CAAC,EACX,OAAO,GAAG,CAAC,CAAC,CAAC;QAEjB,IAAI,UAAU,CAAC;QACf,IAAI,kBAAkB,CAAC;QACvB,IAAI,QAAQ,CAAC;QAEb,IAAI;YACA,cAAM,UAAU,CAAC,IAAI,EAAE,CAAA,CAAC;SAC3B;QAAC,OAAO,KAAU,EAAE;YACjB,UAAU,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,UAAU,CAAC;YACvD,kBAAkB,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,kBAAkB,CAAC;YACvE,iFAAiF;YACjF,IAAI,UAAU,KAAK,GAAG,IAAI,kBAAkB,KAAK,4BAA4B,EAAE;gBAC3E,MAAM,KAAK,CAAC;aACf;YACD,2FAA2F;YAC3F,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC,QAAQ,CAAC,CAAC;YAC/D,mBAAmB,GAAG,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC;YACzD,OAAO,GAAG,mBAAmB,GAAG,CAAC,CAAC;SACrC;QAED,mCAAmC;QACnC,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,CACrC,OAAO,GAAG,CAAC,EAAE,iBAAiB;QAC9B,SAAS,CACZ,CAAC;QAEF,OAAO,IAAI,EAAE;YACT,MAAM,MAAM,GAAG,cAAM,MAAM,CAAC,IAAI,EAAE,CAAA,CAAC;YACnC,IAAI,MAAM,CAAC,IAAI,EAAE;gBACb,MAAM;aACT;YACD,QAAQ,EAAE,CAAC;YACX,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAE9B,yCAAyC;YACzC,IAAA,qBAAM,EAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;YAC7E,8DAA8D;YAE9D,0FAA0F;YAC1F,0FAA0F;YAC1F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;gBAC5B,IAAI,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;uBACjC,OAAO,CAAC,QAAQ,KAAK,EAAE;uBACvB,OAAO,CAAC,IAAI,KAAK,kCAAW,CAAC,WAAW,EAC7C;oBACE,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;iBACnD;aACJ;YAED,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;YAC9B,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;YACvD,oBAAM,QAAQ,CAAA,CAAC;SAClB;QAED,mCAAmC;QACnC,OAAO,CAAC,GAAG,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,wBAAwB,UAAU,WAAW,QAAQ,WAAW,CAAC,CAAC;QAE9H,IAAI,mCAAkB,EAAE;YACpB,IAAI,MAAM,GAAG,EAAE,CAAC;YAChB,MAAM,MAAM,GAAY;gBACpB,IAAI,EAAE,OAAO;gBACb,UAAU,EAAE,EAAE;gBACd,MAAM,EAAE,CAAC,gCAAS,CAAC,OAAO,EAAE,gCAAS,CAAC,QAAQ,EAAE,gCAAS,CAAC,YAAY,CAAC;gBACvE,OAAO,EAAE;oBACL,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;iBACtC;gBACD,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE;aACvB,CAAC;YACF,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;YACnD,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACvB,MAAM,WAAW,GAAG,cAAM,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAA,CAAC;YACvE,MAAM,eAAe,GAAG,WAAW,CAAC,eAAe,CAAC;YACpD,WAAW,CAAC,OAAO,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAE/F,IAAI,eAAe,KAAK,SAAS,EAAE;gBAC/B,MAAM,kBAAkB,GAAG,OAAO,CAAC;gBACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,kBAAkB,CAAC,CAAC;gBACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;gBAC5E,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;gBACnD,mCAAmC;gBACnC,MAAM,GAAG,KAAK,UAAU,mBAAmB,eAAe,CAAC,MAAM,yBAAyB,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC;gBACxI,oBAAM,MAAM,CAAA,CAAC;aAChB;YACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,kBAAkB,MAAM,EAAE,CAAC,CAAC;SACrD;IACL,CAAC;CAAA;AAED,SAAgB,OAAO,CACnB,GAAG,EAAE,+CAA+C;AACpD,GAAW,EACX,KAAe;;QACf,sBAAsB;QACtB,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC;QAEzB,IAAI,iBAAiB,GAAgC,EAAE,CAAC;QAExD,4CAA4C;QAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;QACd,IAAI,IAAI,GAAW,CAAC,CAAC;QACrB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACpB,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;YACzB,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;YAC1F,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;YACtE,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;SACrC;QAED,OAAO,IAAI,EAAE;YACT,MAAM,MAAM,GAAgD,cAAM,GAAG,CAAC,IAAI,EAAE,CAAA,CAAC;YAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;gBACpB,IAAI,GAAG,mBAAmB,CAAC;aAC9B;YACD,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;gBACtB,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;gBAC5B,oBAAM,QAAQ,CAAA,CAAC;gBACf,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;oBACrD,uBAAuB;oBACvB,SAAS;iBACZ;gBACD,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;oBACnC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;iBACnE;gBACD,iBAAiB,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;gBACvD,IAAA,qBAAM,EAAC,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAC/C,KAAK,CAAC,+DAA+D,CAAC,CAAC;gBAC3E,IAAA,qBAAM,EAAC,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc;wBAC7D,IAAI,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,EACvC,KAAK,CAAC,8DAA8D,CAAC,CAAC;aAC7E;YAED,wBAAwB;YACxB,OAAO,iBAAiB,CAAC,MAAM,IAAI,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,EAAE;gBAClG,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;gBACtC,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;gBACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;gBAC5C,YAAE,CAAC,aAAa,CACZ,GAAG,GAAG,YAAY,IAAI,OAAO,EAC7B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,sCAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;gBACrE,0BAA0B;gBAC1B,IAAI,IAAI,KAAK,CAAC;gBACd,IAAA,qBAAM,EAAC,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EACjF,KAAK,CAAC,qDAAqD,CAAC,CAAC;gBACjE,KAAK,EAAE,CAAC;aACX;YAED,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;gBACtB,MAAM;aACT;SACJ;IACL,CAAC;CAAA;AAEM,KAAK,UAAU,kBAAkB,CAAC,eAAkC,EAAE,OAAgB;;IACzF,MAAM,YAAY,GAAG,iCAAgB,IAAI,6BAAY,CAAC;IACtD,IAAI,CAAC,YAAY,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS,CAAC,EAAE;QAC3E,OAAO;KACV;IAED,MAAM,KAAK,GAAG,OAAO,KAAK,SAAS;QAC/B,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,YAAE,CAAC,WAAW,CAAC,OAAO,CAAC;aACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YACb,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBAC9B,OAAO,KAAK,CAAC;aAChB;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAE5C,IAAI,SAAS,GAAG,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAE1E,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE;QACjE,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;KAClD;IAED,IAAI,YAAY,EAAE;QACd,OAAO,IAAA,wCAAiB,EACpB,SAAS,EACT,iCAAgB,EAChB,6BAAY,EACZ,kCAAiB,CAAC,CAAC;KAC1B;SAAM;QACH,IAAI,IAAI,CAAC;;YACT,KAAmB,IAAA,cAAA,cAAA,SAAS,CAAA,eAAA;gBAAjB,IAAI,sBAAA,CAAA;aAAkB;;;;;;;;;KACpC;AACL,CAAC;AAjCD,gDAiCC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport { assert } from \"@fluidframework/common-utils\";\nimport {\n IDocumentService,\n} from \"@fluidframework/driver-definitions\";\nimport {\n IClient,\n ISequencedDocumentMessage,\n MessageType,\n ScopeType,\n} from \"@fluidframework/protocol-definitions\";\nimport { printMessageStats } from \"./fluidAnalyzeMessages\";\nimport {\n connectToWebSocket,\n dumpMessages,\n dumpMessageStats,\n overWrite,\n paramActualFormatting,\n messageTypeFilter,\n} from \"./fluidFetchArgs\";\n\nfunction filenameFromIndex(index: number): string {\n return index === 0 ? \"\" : index.toString(); // support old tools...\n}\n\nlet firstAvailableDelta = 1;\nasync function* loadAllSequencedMessages(\n documentService?: IDocumentService,\n dir?: string,\n files?: string[]) {\n let lastSeq = 0;\n // flag for mismatch between last sequence number read and new one to be read\n let seqNumMismatch = false;\n\n // If we have local save, read ops from there first\n if (files !== undefined) {\n for (let i = 0; i < files.length; i++) {\n const file = filenameFromIndex(i);\n try {\n console.log(`reading messages${file}.json`);\n const fileContent = fs.readFileSync(`${dir}/messages${file}.json`, { encoding: \"utf-8\" });\n const messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n // check if there is mismatch\n seqNumMismatch = messages[0].sequenceNumber !== lastSeq + 1;\n assert(!seqNumMismatch, 0x1b9 /* \"Unexpected value for sequence number of first message in file\" */);\n lastSeq = messages[messages.length - 1].sequenceNumber;\n yield messages;\n } catch (e) {\n if (seqNumMismatch) {\n if (overWrite) {\n // with overWrite option on, we will delete all exisintg message.json files\n for (let index = 0; index < files.length; index++) {\n const name = filenameFromIndex(index);\n fs.unlinkSync(`${dir}/messages${name}.json`);\n }\n break;\n }\n // prompt user to back up and delete existing files\n console.error(\"There are deleted ops in the document being requested,\" +\n \" please back up the existing messages.json file and delete it from its directory.\" +\n \" Then try fetch tool again.\");\n console.error(e);\n return;\n } else {\n console.error(`Error reading / parsing messages from ${files}`);\n console.error(e);\n return;\n }\n }\n }\n if (lastSeq !== 0) {\n console.log(`Read ${lastSeq} ops from local cache`);\n }\n }\n\n if (!documentService) {\n return;\n }\n\n const deltaStorage = await documentService.connectToDeltaStorage();\n\n let timeStart = Date.now();\n let requests = 0;\n let opsStorage = 0;\n\n // reading only 1 op to test if there is mismatch\n const teststream = deltaStorage.fetchMessages(\n lastSeq + 1,\n lastSeq + 2);\n\n let statusCode;\n let innerMostErrorCode;\n let response;\n\n try {\n await teststream.read();\n } catch (error: any) {\n statusCode = error.getTelemetryProperties().statusCode;\n innerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;\n // if there is gap between ops, catch the error and check it is the error we need\n if (statusCode !== 410 || innerMostErrorCode !== \"fluidDeltaDataNotAvailable\") {\n throw error;\n }\n // get firstAvailableDelta from the error response, and set current sequence number to that\n response = JSON.parse(error.getTelemetryProperties().response);\n firstAvailableDelta = response.error.firstAvailableDelta;\n lastSeq = firstAvailableDelta - 1;\n }\n\n // continue reading rest of the ops\n const stream = deltaStorage.fetchMessages(\n lastSeq + 1, // inclusive left\n undefined, // to\n );\n\n while (true) {\n const result = await stream.read();\n if (result.done) {\n break;\n }\n requests++;\n const messages = result.value;\n\n // Empty buckets should never be returned\n assert(messages.length !== 0, 0x1ba /* \"should not return empty buckets\" */);\n // console.log(`Loaded ops at ${messages[0].sequenceNumber}`);\n\n // This parsing of message contents happens in delta manager. But when we analyze messages\n // for message stats, we skip that path. So parsing of json contents needs to happen here.\n for (const message of messages) {\n if (typeof message.contents === \"string\"\n && message.contents !== \"\"\n && message.type !== MessageType.ClientLeave\n ) {\n message.contents = JSON.parse(message.contents);\n }\n }\n\n opsStorage += messages.length;\n lastSeq = messages[messages.length - 1].sequenceNumber;\n yield messages;\n }\n\n // eslint-disable-next-line max-len\n console.log(`\\n${Math.floor((Date.now() - timeStart) / 1000)} seconds to retrieve ${opsStorage} ops in ${requests} requests`);\n\n if (connectToWebSocket) {\n let logMsg = \"\";\n const client: IClient = {\n mode: \"write\",\n permission: [],\n scopes: [ScopeType.DocRead, ScopeType.DocWrite, ScopeType.SummaryWrite],\n details: {\n capabilities: { interactive: true },\n },\n user: { id: \"blah\" },\n };\n console.log(\"Retrieving messages from web socket\");\n timeStart = Date.now();\n const deltaStream = await documentService.connectToDeltaStream(client);\n const initialMessages = deltaStream.initialMessages;\n deltaStream.dispose();\n console.log(`${Math.floor((Date.now() - timeStart) / 1000)} seconds to connect to web socket`);\n\n if (initialMessages !== undefined) {\n const lastSequenceNumber = lastSeq;\n const filtered = initialMessages.filter((a) => a.sequenceNumber > lastSequenceNumber);\n const sorted = filtered.sort((a, b) => a.sequenceNumber - b.sequenceNumber);\n lastSeq = sorted[sorted.length - 1].sequenceNumber;\n // eslint-disable-next-line max-len\n logMsg = ` (${opsStorage} delta storage, ${initialMessages.length} initial ws messages, ${initialMessages.length - sorted.length} dup)`;\n yield sorted;\n }\n console.log(`${lastSeq} total messages${logMsg}`);\n }\n}\n\nasync function* saveOps(\n gen, // AsyncGenerator<ISequencedDocumentMessage[]>,\n dir: string,\n files: string[]) {\n // Split into 100K ops\n const chunk = 100 * 1000;\n\n let sequencedMessages: ISequencedDocumentMessage[] = [];\n\n // Figure out first file we want to write to\n let index = 0;\n let curr: number = 1;\n if (files.length !== 0) {\n index = files.length - 1;\n const name = filenameFromIndex(index);\n const fileContent = fs.readFileSync(`${dir}/messages${name}.json`, { encoding: \"utf-8\" });\n const messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n curr = messages[0].sequenceNumber;\n }\n\n while (true) {\n const result: IteratorResult<ISequencedDocumentMessage[]> = await gen.next();\n if (files.length === 0) {\n curr = firstAvailableDelta;\n }\n if (result.done !== true) {\n let messages = result.value;\n yield messages;\n if (messages[messages.length - 1].sequenceNumber < curr) {\n // Nothing interesting.\n continue;\n }\n if (messages[0].sequenceNumber < curr) {\n messages = messages.filter((msg) => msg.sequenceNumber >= curr);\n }\n sequencedMessages = sequencedMessages.concat(messages);\n assert(sequencedMessages[0].sequenceNumber === curr,\n 0x1bb /* \"Unexpected sequence number on first of messages to save\" */);\n assert(sequencedMessages[sequencedMessages.length - 1].sequenceNumber\n === curr + sequencedMessages.length - 1,\n 0x1bc /* \"Unexpected sequence number on last of messages to save\" */);\n }\n\n // Time to write it out?\n while (sequencedMessages.length >= chunk || (result.done === true && sequencedMessages.length !== 0)) {\n const name = filenameFromIndex(index);\n const write = sequencedMessages.splice(0, chunk);\n console.log(`writing messages${name}.json`);\n fs.writeFileSync(\n `${dir}/messages${name}.json`,\n JSON.stringify(write, undefined, paramActualFormatting ? 0 : 2));\n // increment curr by chunk\n curr += chunk;\n assert(sequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr,\n 0x1bd /* \"Stopped writing at unexpected sequence number\" */);\n index++;\n }\n\n if (result.done === true) {\n break;\n }\n }\n}\n\nexport async function fluidFetchMessages(documentService?: IDocumentService, saveDir?: string) {\n const messageStats = dumpMessageStats || dumpMessages;\n if (!messageStats && (saveDir === undefined || documentService === undefined)) {\n return;\n }\n\n const files = saveDir === undefined\n ? undefined\n : fs.readdirSync(saveDir)\n .filter((file) => {\n if (!file.startsWith(\"messages\")) {\n return false;\n }\n return true;\n })\n .sort((a, b) => a.localeCompare(b));\n\n let generator = loadAllSequencedMessages(documentService, saveDir, files);\n\n if (saveDir !== undefined && files !== undefined && documentService) {\n generator = saveOps(generator, saveDir, files);\n }\n\n if (messageStats) {\n return printMessageStats(\n generator,\n dumpMessageStats,\n dumpMessages,\n messageTypeFilter);\n } else {\n let item;\n for await (item of generator) { }\n }\n}\n"]}
|
|
1
|
+
{"version":3,"file":"fluidFetchMessages.js","sourceRoot":"","sources":["../src/fluidFetchMessages.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;;;;AAEH,4CAAoB;AACpB,2DAAoD;AAEpD,+EAK8C;AAC9C,iEAA2D;AAC3D,qDAO0B;AAE1B,SAAS,iBAAiB,CAAC,KAAa;IACvC,OAAO,KAAK,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,uBAAuB;AACpE,CAAC;AAED,IAAI,mBAAmB,GAAG,CAAC,CAAC;AAC5B,KAAK,SAAS,CAAC,CAAC,wBAAwB,CACvC,eAAkC,EAClC,GAAY,EACZ,KAAgB;IAEhB,IAAI,OAAO,GAAG,CAAC,CAAC;IAChB,6EAA6E;IAC7E,IAAI,cAAc,GAAG,KAAK,CAAC;IAE3B,mDAAmD;IACnD,IAAI,KAAK,KAAK,SAAS,EAAE;QACxB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YACtC,MAAM,IAAI,GAAG,iBAAiB,CAAC,CAAC,CAAC,CAAC;YAClC,IAAI;gBACH,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;gBAC5C,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE;oBAClE,QAAQ,EAAE,OAAO;iBACjB,CAAC,CAAC;gBACH,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;gBACtE,6BAA6B;gBAC7B,cAAc,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,OAAO,GAAG,CAAC,CAAC;gBAC5D,IAAA,mBAAM,EACL,CAAC,cAAc,EACf,KAAK,CAAC,qEAAqE,CAC3E,CAAC;gBACF,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;gBACvD,MAAM,QAAQ,CAAC;aACf;YAAC,OAAO,CAAC,EAAE;gBACX,IAAI,cAAc,EAAE;oBACnB,IAAI,0BAAS,EAAE;wBACd,2EAA2E;wBAC3E,KAAK,IAAI,KAAK,GAAG,CAAC,EAAE,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,KAAK,EAAE,EAAE;4BAClD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;4BACtC,YAAE,CAAC,UAAU,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,CAAC,CAAC;yBAC7C;wBACD,MAAM;qBACN;oBACD,mDAAmD;oBACnD,OAAO,CAAC,KAAK,CACZ,wDAAwD;wBACvD,mFAAmF;wBACnF,6BAA6B,CAC9B,CAAC;oBACF,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACjB,OAAO;iBACP;qBAAM;oBACN,OAAO,CAAC,KAAK,CAAC,yCAAyC,KAAK,EAAE,CAAC,CAAC;oBAChE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;oBACjB,OAAO;iBACP;aACD;SACD;QACD,IAAI,OAAO,KAAK,CAAC,EAAE;YAClB,OAAO,CAAC,GAAG,CAAC,QAAQ,OAAO,uBAAuB,CAAC,CAAC;SACpD;KACD;IAED,IAAI,CAAC,eAAe,EAAE;QACrB,OAAO;KACP;IAED,MAAM,YAAY,GAAG,MAAM,eAAe,CAAC,qBAAqB,EAAE,CAAC;IAEnE,IAAI,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC3B,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,UAAU,GAAG,CAAC,CAAC;IAEnB,iDAAiD;IACjD,MAAM,UAAU,GAAG,YAAY,CAAC,aAAa,CAAC,OAAO,GAAG,CAAC,EAAE,OAAO,GAAG,CAAC,CAAC,CAAC;IAExE,IAAI,UAAU,CAAC;IACf,IAAI,kBAAkB,CAAC;IACvB,IAAI,QAAQ,CAAC;IAEb,IAAI;QACH,MAAM,UAAU,CAAC,IAAI,EAAE,CAAC;KACxB;IAAC,OAAO,KAAU,EAAE;QACpB,UAAU,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,UAAU,CAAC;QACvD,kBAAkB,GAAG,KAAK,CAAC,sBAAsB,EAAE,CAAC,kBAAkB,CAAC;QACvE,iFAAiF;QACjF,IAAI,UAAU,KAAK,GAAG,IAAI,kBAAkB,KAAK,4BAA4B,EAAE;YAC9E,MAAM,KAAK,CAAC;SACZ;QACD,2FAA2F;QAC3F,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC,QAAQ,CAAC,CAAC;QAC/D,mBAAmB,GAAG,QAAQ,CAAC,KAAK,CAAC,mBAAmB,CAAC;QACzD,OAAO,GAAG,mBAAmB,GAAG,CAAC,CAAC;KAClC;IAED,mCAAmC;IACnC,MAAM,MAAM,GAAG,YAAY,CAAC,aAAa,CACxC,OAAO,GAAG,CAAC,EAAE,iBAAiB;IAC9B,SAAS,CACT,CAAC;IAEF,OAAO,IAAI,EAAE;QACZ,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,IAAI,EAAE,CAAC;QACnC,IAAI,MAAM,CAAC,IAAI,EAAE;YAChB,MAAM;SACN;QACD,QAAQ,EAAE,CAAC;QACX,MAAM,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;QAE9B,yCAAyC;QACzC,IAAA,mBAAM,EAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC7E,8DAA8D;QAE9D,0FAA0F;QAC1F,0FAA0F;QAC1F,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE;YAC/B,IACC,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,OAAO,CAAC,QAAQ,KAAK,EAAE;gBACvB,OAAO,CAAC,IAAI,KAAK,kCAAW,CAAC,WAAW,EACvC;gBACD,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;aAChD;SACD;QAED,UAAU,IAAI,QAAQ,CAAC,MAAM,CAAC;QAC9B,OAAO,GAAG,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;QACvD,MAAM,QAAQ,CAAC;KACf;IAED,OAAO,CAAC,GAAG,CACV,KAAK,IAAI,CAAC,KAAK,CACd,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAC/B,wBAAwB,UAAU,WAAW,QAAQ,WAAW,CACjE,CAAC;IAEF,IAAI,mCAAkB,EAAE;QACvB,IAAI,MAAM,GAAG,EAAE,CAAC;QAChB,MAAM,MAAM,GAAY;YACvB,IAAI,EAAE,OAAO;YACb,UAAU,EAAE,EAAE;YACd,MAAM,EAAE,CAAC,gCAAS,CAAC,OAAO,EAAE,gCAAS,CAAC,QAAQ,EAAE,gCAAS,CAAC,YAAY,CAAC;YACvE,OAAO,EAAE;gBACR,YAAY,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE;aACnC;YACD,IAAI,EAAE,EAAE,EAAE,EAAE,MAAM,EAAE;SACpB,CAAC;QACF,OAAO,CAAC,GAAG,CAAC,qCAAqC,CAAC,CAAC;QACnD,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACvB,MAAM,WAAW,GAAG,MAAM,eAAe,CAAC,oBAAoB,CAAC,MAAM,CAAC,CAAC;QACvE,MAAM,eAAe,GAAG,WAAW,CAAC,eAAe,CAAC;QACpD,WAAW,CAAC,OAAO,EAAE,CAAC;QACtB,OAAO,CAAC,GAAG,CACV,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,mCAAmC,CACjF,CAAC;QAEF,IAAI,eAAe,KAAK,SAAS,EAAE;YAClC,MAAM,kBAAkB,GAAG,OAAO,CAAC;YACnC,MAAM,QAAQ,GAAG,eAAe,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,kBAAkB,CAAC,CAAC;YACtF,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,cAAc,CAAC,CAAC;YAC5E,OAAO,GAAG,MAAM,CAAC,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,CAAC;YACnD,MAAM,GAAG,KAAK,UAAU,mBACvB,eAAe,CAAC,MACjB,yBAAyB,eAAe,CAAC,MAAM,GAAG,MAAM,CAAC,MAAM,OAAO,CAAC;YACvE,MAAM,MAAM,CAAC;SACb;QACD,OAAO,CAAC,GAAG,CAAC,GAAG,OAAO,kBAAkB,MAAM,EAAE,CAAC,CAAC;KAClD;AACF,CAAC;AAED,KAAK,SAAS,CAAC,CAAC,OAAO,CACtB,GAAG,EAAE,+CAA+C;AACpD,GAAW,EACX,KAAe;IAEf,sBAAsB;IACtB,MAAM,KAAK,GAAG,GAAG,GAAG,IAAI,CAAC;IAEzB,IAAI,iBAAiB,GAAgC,EAAE,CAAC;IAExD,4CAA4C;IAC5C,IAAI,KAAK,GAAG,CAAC,CAAC;IACd,IAAI,IAAI,GAAW,CAAC,CAAC;IACrB,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;QACvB,KAAK,GAAG,KAAK,CAAC,MAAM,GAAG,CAAC,CAAC;QACzB,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,WAAW,GAAG,YAAE,CAAC,YAAY,CAAC,GAAG,GAAG,YAAY,IAAI,OAAO,EAAE,EAAE,QAAQ,EAAE,OAAO,EAAE,CAAC,CAAC;QAC1F,MAAM,QAAQ,GAAgC,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QACtE,IAAI,GAAG,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,CAAC;KAClC;IAED,OAAO,IAAI,EAAE;QACZ,MAAM,MAAM,GAAgD,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC;QAC7E,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE;YACvB,IAAI,GAAG,mBAAmB,CAAC;SAC3B;QACD,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;YACzB,IAAI,QAAQ,GAAG,MAAM,CAAC,KAAK,CAAC;YAC5B,MAAM,QAAQ,CAAC;YACf,IAAI,QAAQ,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;gBACxD,uBAAuB;gBACvB,SAAS;aACT;YACD,IAAI,QAAQ,CAAC,CAAC,CAAC,CAAC,cAAc,GAAG,IAAI,EAAE;gBACtC,QAAQ,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,cAAc,IAAI,IAAI,CAAC,CAAC;aAChE;YACD,iBAAiB,GAAG,iBAAiB,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;YACvD,IAAA,mBAAM,EACL,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAC5C,KAAK,CAAC,+DAA+D,CACrE,CAAC;YACF,IAAA,mBAAM,EACL,iBAAiB,CAAC,iBAAiB,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,cAAc;gBAC7D,IAAI,GAAG,iBAAiB,CAAC,MAAM,GAAG,CAAC,EACpC,KAAK,CAAC,8DAA8D,CACpE,CAAC;SACF;QAED,wBAAwB;QACxB,OACC,iBAAiB,CAAC,MAAM,IAAI,KAAK;YACjC,CAAC,MAAM,CAAC,IAAI,KAAK,IAAI,IAAI,iBAAiB,CAAC,MAAM,KAAK,CAAC,CAAC,EACvD;YACD,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,CAAC;YACtC,MAAM,KAAK,GAAG,iBAAiB,CAAC,MAAM,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC;YACjD,OAAO,CAAC,GAAG,CAAC,mBAAmB,IAAI,OAAO,CAAC,CAAC;YAC5C,YAAE,CAAC,aAAa,CACf,GAAG,GAAG,YAAY,IAAI,OAAO,EAC7B,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,SAAS,EAAE,sCAAqB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAC/D,CAAC;YACF,0BAA0B;YAC1B,IAAI,IAAI,KAAK,CAAC;YACd,IAAA,mBAAM,EACL,iBAAiB,CAAC,MAAM,KAAK,CAAC,IAAI,iBAAiB,CAAC,CAAC,CAAC,CAAC,cAAc,KAAK,IAAI,EAC9E,KAAK,CAAC,qDAAqD,CAC3D,CAAC;YACF,KAAK,EAAE,CAAC;SACR;QAED,IAAI,MAAM,CAAC,IAAI,KAAK,IAAI,EAAE;YACzB,MAAM;SACN;KACD;AACF,CAAC;AAEM,KAAK,UAAU,kBAAkB,CAAC,eAAkC,EAAE,OAAgB;IAC5F,MAAM,YAAY,GAAG,iCAAgB,IAAI,6BAAY,CAAC;IACtD,IAAI,CAAC,YAAY,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,eAAe,KAAK,SAAS,CAAC,EAAE;QAC9E,OAAO;KACP;IAED,MAAM,KAAK,GACV,OAAO,KAAK,SAAS;QACpB,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,YAAE;aACD,WAAW,CAAC,OAAO,CAAC;aACpB,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE;YAChB,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE;gBACjC,OAAO,KAAK,CAAC;aACb;YACD,OAAO,IAAI,CAAC;QACb,CAAC,CAAC;aACD,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC;IAExC,IAAI,SAAS,GAAG,wBAAwB,CAAC,eAAe,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;IAE1E,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,eAAe,EAAE;QACpE,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,OAAO,EAAE,KAAK,CAAC,CAAC;KAC/C;IAED,IAAI,YAAY,EAAE;QACjB,OAAO,IAAA,wCAAiB,EAAC,SAAS,EAAE,iCAAgB,EAAE,6BAAY,EAAE,kCAAiB,CAAC,CAAC;KACvF;SAAM;QACN,IAAI,IAAI,CAAC;QACT,IAAI,KAAK,EAAE,IAAI,IAAI,SAAS,EAAE;SAC7B;KACD;AACF,CAAC;AAhCD,gDAgCC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport fs from \"fs\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IDocumentService } from \"@fluidframework/driver-definitions\";\nimport {\n\tIClient,\n\tISequencedDocumentMessage,\n\tMessageType,\n\tScopeType,\n} from \"@fluidframework/protocol-definitions\";\nimport { printMessageStats } from \"./fluidAnalyzeMessages\";\nimport {\n\tconnectToWebSocket,\n\tdumpMessages,\n\tdumpMessageStats,\n\toverWrite,\n\tparamActualFormatting,\n\tmessageTypeFilter,\n} from \"./fluidFetchArgs\";\n\nfunction filenameFromIndex(index: number): string {\n\treturn index === 0 ? \"\" : index.toString(); // support old tools...\n}\n\nlet firstAvailableDelta = 1;\nasync function* loadAllSequencedMessages(\n\tdocumentService?: IDocumentService,\n\tdir?: string,\n\tfiles?: string[],\n) {\n\tlet lastSeq = 0;\n\t// flag for mismatch between last sequence number read and new one to be read\n\tlet seqNumMismatch = false;\n\n\t// If we have local save, read ops from there first\n\tif (files !== undefined) {\n\t\tfor (let i = 0; i < files.length; i++) {\n\t\t\tconst file = filenameFromIndex(i);\n\t\t\ttry {\n\t\t\t\tconsole.log(`reading messages${file}.json`);\n\t\t\t\tconst fileContent = fs.readFileSync(`${dir}/messages${file}.json`, {\n\t\t\t\t\tencoding: \"utf-8\",\n\t\t\t\t});\n\t\t\t\tconst messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n\t\t\t\t// check if there is mismatch\n\t\t\t\tseqNumMismatch = messages[0].sequenceNumber !== lastSeq + 1;\n\t\t\t\tassert(\n\t\t\t\t\t!seqNumMismatch,\n\t\t\t\t\t0x1b9 /* \"Unexpected value for sequence number of first message in file\" */,\n\t\t\t\t);\n\t\t\t\tlastSeq = messages[messages.length - 1].sequenceNumber;\n\t\t\t\tyield messages;\n\t\t\t} catch (e) {\n\t\t\t\tif (seqNumMismatch) {\n\t\t\t\t\tif (overWrite) {\n\t\t\t\t\t\t// with overWrite option on, we will delete all exisintg message.json files\n\t\t\t\t\t\tfor (let index = 0; index < files.length; index++) {\n\t\t\t\t\t\t\tconst name = filenameFromIndex(index);\n\t\t\t\t\t\t\tfs.unlinkSync(`${dir}/messages${name}.json`);\n\t\t\t\t\t\t}\n\t\t\t\t\t\tbreak;\n\t\t\t\t\t}\n\t\t\t\t\t// prompt user to back up and delete existing files\n\t\t\t\t\tconsole.error(\n\t\t\t\t\t\t\"There are deleted ops in the document being requested,\" +\n\t\t\t\t\t\t\t\" please back up the existing messages.json file and delete it from its directory.\" +\n\t\t\t\t\t\t\t\" Then try fetch tool again.\",\n\t\t\t\t\t);\n\t\t\t\t\tconsole.error(e);\n\t\t\t\t\treturn;\n\t\t\t\t} else {\n\t\t\t\t\tconsole.error(`Error reading / parsing messages from ${files}`);\n\t\t\t\t\tconsole.error(e);\n\t\t\t\t\treturn;\n\t\t\t\t}\n\t\t\t}\n\t\t}\n\t\tif (lastSeq !== 0) {\n\t\t\tconsole.log(`Read ${lastSeq} ops from local cache`);\n\t\t}\n\t}\n\n\tif (!documentService) {\n\t\treturn;\n\t}\n\n\tconst deltaStorage = await documentService.connectToDeltaStorage();\n\n\tlet timeStart = Date.now();\n\tlet requests = 0;\n\tlet opsStorage = 0;\n\n\t// reading only 1 op to test if there is mismatch\n\tconst teststream = deltaStorage.fetchMessages(lastSeq + 1, lastSeq + 2);\n\n\tlet statusCode;\n\tlet innerMostErrorCode;\n\tlet response;\n\n\ttry {\n\t\tawait teststream.read();\n\t} catch (error: any) {\n\t\tstatusCode = error.getTelemetryProperties().statusCode;\n\t\tinnerMostErrorCode = error.getTelemetryProperties().innerMostErrorCode;\n\t\t// if there is gap between ops, catch the error and check it is the error we need\n\t\tif (statusCode !== 410 || innerMostErrorCode !== \"fluidDeltaDataNotAvailable\") {\n\t\t\tthrow error;\n\t\t}\n\t\t// get firstAvailableDelta from the error response, and set current sequence number to that\n\t\tresponse = JSON.parse(error.getTelemetryProperties().response);\n\t\tfirstAvailableDelta = response.error.firstAvailableDelta;\n\t\tlastSeq = firstAvailableDelta - 1;\n\t}\n\n\t// continue reading rest of the ops\n\tconst stream = deltaStorage.fetchMessages(\n\t\tlastSeq + 1, // inclusive left\n\t\tundefined, // to\n\t);\n\n\twhile (true) {\n\t\tconst result = await stream.read();\n\t\tif (result.done) {\n\t\t\tbreak;\n\t\t}\n\t\trequests++;\n\t\tconst messages = result.value;\n\n\t\t// Empty buckets should never be returned\n\t\tassert(messages.length !== 0, 0x1ba /* \"should not return empty buckets\" */);\n\t\t// console.log(`Loaded ops at ${messages[0].sequenceNumber}`);\n\n\t\t// This parsing of message contents happens in delta manager. But when we analyze messages\n\t\t// for message stats, we skip that path. So parsing of json contents needs to happen here.\n\t\tfor (const message of messages) {\n\t\t\tif (\n\t\t\t\ttypeof message.contents === \"string\" &&\n\t\t\t\tmessage.contents !== \"\" &&\n\t\t\t\tmessage.type !== MessageType.ClientLeave\n\t\t\t) {\n\t\t\t\tmessage.contents = JSON.parse(message.contents);\n\t\t\t}\n\t\t}\n\n\t\topsStorage += messages.length;\n\t\tlastSeq = messages[messages.length - 1].sequenceNumber;\n\t\tyield messages;\n\t}\n\n\tconsole.log(\n\t\t`\\n${Math.floor(\n\t\t\t(Date.now() - timeStart) / 1000,\n\t\t)} seconds to retrieve ${opsStorage} ops in ${requests} requests`,\n\t);\n\n\tif (connectToWebSocket) {\n\t\tlet logMsg = \"\";\n\t\tconst client: IClient = {\n\t\t\tmode: \"write\",\n\t\t\tpermission: [],\n\t\t\tscopes: [ScopeType.DocRead, ScopeType.DocWrite, ScopeType.SummaryWrite],\n\t\t\tdetails: {\n\t\t\t\tcapabilities: { interactive: true },\n\t\t\t},\n\t\t\tuser: { id: \"blah\" },\n\t\t};\n\t\tconsole.log(\"Retrieving messages from web socket\");\n\t\ttimeStart = Date.now();\n\t\tconst deltaStream = await documentService.connectToDeltaStream(client);\n\t\tconst initialMessages = deltaStream.initialMessages;\n\t\tdeltaStream.dispose();\n\t\tconsole.log(\n\t\t\t`${Math.floor((Date.now() - timeStart) / 1000)} seconds to connect to web socket`,\n\t\t);\n\n\t\tif (initialMessages !== undefined) {\n\t\t\tconst lastSequenceNumber = lastSeq;\n\t\t\tconst filtered = initialMessages.filter((a) => a.sequenceNumber > lastSequenceNumber);\n\t\t\tconst sorted = filtered.sort((a, b) => a.sequenceNumber - b.sequenceNumber);\n\t\t\tlastSeq = sorted[sorted.length - 1].sequenceNumber;\n\t\t\tlogMsg = ` (${opsStorage} delta storage, ${\n\t\t\t\tinitialMessages.length\n\t\t\t} initial ws messages, ${initialMessages.length - sorted.length} dup)`;\n\t\t\tyield sorted;\n\t\t}\n\t\tconsole.log(`${lastSeq} total messages${logMsg}`);\n\t}\n}\n\nasync function* saveOps(\n\tgen, // AsyncGenerator<ISequencedDocumentMessage[]>,\n\tdir: string,\n\tfiles: string[],\n) {\n\t// Split into 100K ops\n\tconst chunk = 100 * 1000;\n\n\tlet sequencedMessages: ISequencedDocumentMessage[] = [];\n\n\t// Figure out first file we want to write to\n\tlet index = 0;\n\tlet curr: number = 1;\n\tif (files.length !== 0) {\n\t\tindex = files.length - 1;\n\t\tconst name = filenameFromIndex(index);\n\t\tconst fileContent = fs.readFileSync(`${dir}/messages${name}.json`, { encoding: \"utf-8\" });\n\t\tconst messages: ISequencedDocumentMessage[] = JSON.parse(fileContent);\n\t\tcurr = messages[0].sequenceNumber;\n\t}\n\n\twhile (true) {\n\t\tconst result: IteratorResult<ISequencedDocumentMessage[]> = await gen.next();\n\t\tif (files.length === 0) {\n\t\t\tcurr = firstAvailableDelta;\n\t\t}\n\t\tif (result.done !== true) {\n\t\t\tlet messages = result.value;\n\t\t\tyield messages;\n\t\t\tif (messages[messages.length - 1].sequenceNumber < curr) {\n\t\t\t\t// Nothing interesting.\n\t\t\t\tcontinue;\n\t\t\t}\n\t\t\tif (messages[0].sequenceNumber < curr) {\n\t\t\t\tmessages = messages.filter((msg) => msg.sequenceNumber >= curr);\n\t\t\t}\n\t\t\tsequencedMessages = sequencedMessages.concat(messages);\n\t\t\tassert(\n\t\t\t\tsequencedMessages[0].sequenceNumber === curr,\n\t\t\t\t0x1bb /* \"Unexpected sequence number on first of messages to save\" */,\n\t\t\t);\n\t\t\tassert(\n\t\t\t\tsequencedMessages[sequencedMessages.length - 1].sequenceNumber ===\n\t\t\t\t\tcurr + sequencedMessages.length - 1,\n\t\t\t\t0x1bc /* \"Unexpected sequence number on last of messages to save\" */,\n\t\t\t);\n\t\t}\n\n\t\t// Time to write it out?\n\t\twhile (\n\t\t\tsequencedMessages.length >= chunk ||\n\t\t\t(result.done === true && sequencedMessages.length !== 0)\n\t\t) {\n\t\t\tconst name = filenameFromIndex(index);\n\t\t\tconst write = sequencedMessages.splice(0, chunk);\n\t\t\tconsole.log(`writing messages${name}.json`);\n\t\t\tfs.writeFileSync(\n\t\t\t\t`${dir}/messages${name}.json`,\n\t\t\t\tJSON.stringify(write, undefined, paramActualFormatting ? 0 : 2),\n\t\t\t);\n\t\t\t// increment curr by chunk\n\t\t\tcurr += chunk;\n\t\t\tassert(\n\t\t\t\tsequencedMessages.length === 0 || sequencedMessages[0].sequenceNumber === curr,\n\t\t\t\t0x1bd /* \"Stopped writing at unexpected sequence number\" */,\n\t\t\t);\n\t\t\tindex++;\n\t\t}\n\n\t\tif (result.done === true) {\n\t\t\tbreak;\n\t\t}\n\t}\n}\n\nexport async function fluidFetchMessages(documentService?: IDocumentService, saveDir?: string) {\n\tconst messageStats = dumpMessageStats || dumpMessages;\n\tif (!messageStats && (saveDir === undefined || documentService === undefined)) {\n\t\treturn;\n\t}\n\n\tconst files =\n\t\tsaveDir === undefined\n\t\t\t? undefined\n\t\t\t: fs\n\t\t\t\t\t.readdirSync(saveDir)\n\t\t\t\t\t.filter((file) => {\n\t\t\t\t\t\tif (!file.startsWith(\"messages\")) {\n\t\t\t\t\t\t\treturn false;\n\t\t\t\t\t\t}\n\t\t\t\t\t\treturn true;\n\t\t\t\t\t})\n\t\t\t\t\t.sort((a, b) => a.localeCompare(b));\n\n\tlet generator = loadAllSequencedMessages(documentService, saveDir, files);\n\n\tif (saveDir !== undefined && files !== undefined && documentService) {\n\t\tgenerator = saveOps(generator, saveDir, files);\n\t}\n\n\tif (messageStats) {\n\t\treturn printMessageStats(generator, dumpMessageStats, dumpMessages, messageTypeFilter);\n\t} else {\n\t\tlet item;\n\t\tfor await (item of generator) {\n\t\t}\n\t}\n}\n"]}
|
|
@@ -4,7 +4,6 @@
|
|
|
4
4
|
*/
|
|
5
5
|
import { IClientConfig, IOdspDriveItem, IOdspAuthRequestInfo } from "@fluidframework/odsp-doclib-utils";
|
|
6
6
|
export declare function resolveWrapper<T>(callback: (authRequestInfo: IOdspAuthRequestInfo) => Promise<T>, server: string, clientConfig: IClientConfig, forceTokenReauth?: boolean, forToken?: boolean): Promise<T>;
|
|
7
|
-
export declare function resolveDriveItemByServerRelativePath(server: string, serverRelativePath: string, clientConfig: IClientConfig): Promise<IOdspDriveItem>;
|
|
8
7
|
export declare function getSharepointFiles(server: string, serverRelativePath: string, recurse: boolean): Promise<IOdspDriveItem[]>;
|
|
9
8
|
export declare function getSingleSharePointFile(server: string, drive: string, item: string): Promise<IOdspDriveItem>;
|
|
10
9
|
//# sourceMappingURL=fluidFetchSharePoint.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidFetchSharePoint.d.ts","sourceRoot":"","sources":["../src/fluidFetchSharePoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"fluidFetchSharePoint.d.ts","sourceRoot":"","sources":["../src/fluidFetchSharePoint.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAIN,aAAa,EACb,cAAc,EAEd,oBAAoB,EACpB,MAAM,mCAAmC,CAAC;AAU3C,wBAAsB,cAAc,CAAC,CAAC,EACrC,QAAQ,EAAE,CAAC,eAAe,EAAE,oBAAoB,KAAK,OAAO,CAAC,CAAC,CAAC,EAC/D,MAAM,EAAE,MAAM,EACd,YAAY,EAAE,aAAa,EAC3B,gBAAgB,UAAQ,EACxB,QAAQ,UAAQ,GACd,OAAO,CAAC,CAAC,CAAC,CAoCZ;AA6BD,wBAAsB,kBAAkB,CACvC,MAAM,EAAE,MAAM,EACd,kBAAkB,EAAE,MAAM,EAC1B,OAAO,EAAE,OAAO,6BAsChB;AAED,wBAAsB,uBAAuB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,2BASxF"}
|