@fluid-tools/fetch-tool 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +6 -8
- package/CHANGELOG.md +117 -0
- package/README.md +38 -7
- package/bin/fluid-fetch +0 -0
- package/dist/fluidAnalyzeMessages.d.ts.map +1 -1
- package/dist/fluidAnalyzeMessages.js +106 -116
- package/dist/fluidAnalyzeMessages.js.map +1 -1
- package/dist/fluidFetch.js +5 -3
- package/dist/fluidFetch.js.map +1 -1
- package/dist/fluidFetchArgs.d.ts +0 -3
- package/dist/fluidFetchArgs.d.ts.map +1 -1
- package/dist/fluidFetchArgs.js +10 -14
- package/dist/fluidFetchArgs.js.map +1 -1
- package/dist/fluidFetchInit.d.ts +0 -1
- package/dist/fluidFetchInit.d.ts.map +1 -1
- package/dist/fluidFetchInit.js +41 -34
- package/dist/fluidFetchInit.js.map +1 -1
- package/dist/fluidFetchMessages.d.ts.map +1 -1
- package/dist/fluidFetchMessages.js +168 -200
- package/dist/fluidFetchMessages.js.map +1 -1
- package/dist/fluidFetchSharePoint.d.ts +0 -1
- package/dist/fluidFetchSharePoint.d.ts.map +1 -1
- package/dist/fluidFetchSharePoint.js +20 -6
- package/dist/fluidFetchSharePoint.js.map +1 -1
- package/dist/fluidFetchSnapshot.d.ts.map +1 -1
- package/dist/fluidFetchSnapshot.js +18 -20
- package/dist/fluidFetchSnapshot.js.map +1 -1
- package/package.json +47 -42
- package/prettier.config.cjs +8 -0
- package/src/fluidAnalyzeMessages.ts +701 -630
- package/src/fluidFetch.ts +93 -88
- package/src/fluidFetchArgs.ts +167 -168
- package/src/fluidFetchInit.ts +133 -104
- package/src/fluidFetchMessages.ts +253 -232
- package/src/fluidFetchSharePoint.ts +130 -112
- package/src/fluidFetchSnapshot.ts +313 -295
- package/tsconfig.json +8 -15
package/.eslintrc.js
CHANGED
|
@@ -4,11 +4,9 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
module.exports = {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
}
|
|
14
|
-
}
|
|
7
|
+
extends: [require.resolve("@fluidframework/eslint-config-fluid/minimal"), "prettier"],
|
|
8
|
+
rules: {
|
|
9
|
+
// This library is used in the browser, so we don't want dependencies on most node libraries.
|
|
10
|
+
"import/no-nodejs-modules": ["error", { allow: ["child_process", "fs", "url", "util"] }],
|
|
11
|
+
},
|
|
12
|
+
};
|
package/CHANGELOG.md
ADDED
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
# @fluid-tools/fetch-tool
|
|
2
|
+
|
|
3
|
+
## 2.0.0-internal.8.0.0
|
|
4
|
+
|
|
5
|
+
Dependency updates only.
|
|
6
|
+
|
|
7
|
+
## 2.0.0-internal.7.4.0
|
|
8
|
+
|
|
9
|
+
Dependency updates only.
|
|
10
|
+
|
|
11
|
+
## 2.0.0-internal.7.3.0
|
|
12
|
+
|
|
13
|
+
Dependency updates only.
|
|
14
|
+
|
|
15
|
+
## 2.0.0-internal.7.2.0
|
|
16
|
+
|
|
17
|
+
Dependency updates only.
|
|
18
|
+
|
|
19
|
+
## 2.0.0-internal.7.1.0
|
|
20
|
+
|
|
21
|
+
Dependency updates only.
|
|
22
|
+
|
|
23
|
+
## 2.0.0-internal.7.0.0
|
|
24
|
+
|
|
25
|
+
### Major Changes
|
|
26
|
+
|
|
27
|
+
- Dependencies on @fluidframework/protocol-definitions package updated to 3.0.0 [871b3493dd](https://github.com/microsoft/FluidFramework/commits/871b3493dd0d7ea3a89be64998ceb6cb9021a04e)
|
|
28
|
+
|
|
29
|
+
This included the following changes from the protocol-definitions release:
|
|
30
|
+
|
|
31
|
+
- Updating signal interfaces for some planned improvements. The intention is split the interface between signals
|
|
32
|
+
submitted by clients to the server and the resulting signals sent from the server to clients.
|
|
33
|
+
- A new optional type member is available on the ISignalMessage interface and a new ISentSignalMessage interface has
|
|
34
|
+
been added, which will be the typing for signals sent from the client to the server. Both extend a new
|
|
35
|
+
ISignalMessageBase interface that contains common members.
|
|
36
|
+
- The @fluidframework/common-definitions package dependency has been updated to version 1.0.0.
|
|
37
|
+
|
|
38
|
+
- Server upgrade: dependencies on Fluid server packages updated to 2.0.1 [871b3493dd](https://github.com/microsoft/FluidFramework/commits/871b3493dd0d7ea3a89be64998ceb6cb9021a04e)
|
|
39
|
+
|
|
40
|
+
Dependencies on the following Fluid server package have been updated to version 2.0.1:
|
|
41
|
+
|
|
42
|
+
- @fluidframework/gitresources: 2.0.1
|
|
43
|
+
- @fluidframework/server-kafka-orderer: 2.0.1
|
|
44
|
+
- @fluidframework/server-lambdas: 2.0.1
|
|
45
|
+
- @fluidframework/server-lambdas-driver: 2.0.1
|
|
46
|
+
- @fluidframework/server-local-server: 2.0.1
|
|
47
|
+
- @fluidframework/server-memory-orderer: 2.0.1
|
|
48
|
+
- @fluidframework/protocol-base: 2.0.1
|
|
49
|
+
- @fluidframework/server-routerlicious: 2.0.1
|
|
50
|
+
- @fluidframework/server-routerlicious-base: 2.0.1
|
|
51
|
+
- @fluidframework/server-services: 2.0.1
|
|
52
|
+
- @fluidframework/server-services-client: 2.0.1
|
|
53
|
+
- @fluidframework/server-services-core: 2.0.1
|
|
54
|
+
- @fluidframework/server-services-ordering-kafkanode: 2.0.1
|
|
55
|
+
- @fluidframework/server-services-ordering-rdkafka: 2.0.1
|
|
56
|
+
- @fluidframework/server-services-ordering-zookeeper: 2.0.1
|
|
57
|
+
- @fluidframework/server-services-shared: 2.0.1
|
|
58
|
+
- @fluidframework/server-services-telemetry: 2.0.1
|
|
59
|
+
- @fluidframework/server-services-utils: 2.0.1
|
|
60
|
+
- @fluidframework/server-test-utils: 2.0.1
|
|
61
|
+
- tinylicious: 2.0.1
|
|
62
|
+
|
|
63
|
+
- Minimum TypeScript version now 5.1.6 [871b3493dd](https://github.com/microsoft/FluidFramework/commits/871b3493dd0d7ea3a89be64998ceb6cb9021a04e)
|
|
64
|
+
|
|
65
|
+
The minimum supported TypeScript version for Fluid 2.0 clients is now 5.1.6.
|
|
66
|
+
|
|
67
|
+
## 2.0.0-internal.6.4.0
|
|
68
|
+
|
|
69
|
+
Dependency updates only.
|
|
70
|
+
|
|
71
|
+
## 2.0.0-internal.6.3.0
|
|
72
|
+
|
|
73
|
+
Dependency updates only.
|
|
74
|
+
|
|
75
|
+
## 2.0.0-internal.6.2.0
|
|
76
|
+
|
|
77
|
+
Dependency updates only.
|
|
78
|
+
|
|
79
|
+
## 2.0.0-internal.6.1.0
|
|
80
|
+
|
|
81
|
+
Dependency updates only.
|
|
82
|
+
|
|
83
|
+
## 2.0.0-internal.6.0.0
|
|
84
|
+
|
|
85
|
+
### Major Changes
|
|
86
|
+
|
|
87
|
+
- Upgraded typescript transpilation target to ES2020 [8abce8cdb4](https://github.com/microsoft/FluidFramework/commits/8abce8cdb4e2832fb6405fb44e393bef03d5648a)
|
|
88
|
+
|
|
89
|
+
Upgraded typescript transpilation target to ES2020. This is done in order to decrease the bundle sizes of Fluid Framework packages. This has provided size improvements across the board for ex. Loader, Driver, Runtime etc. Reduced bundle sizes helps to load lesser code in apps and hence also helps to improve the perf.If any app wants to target any older versions of browsers with which this target version is not compatible, then they can use packages like babel to transpile to a older target.
|
|
90
|
+
|
|
91
|
+
## 2.0.0-internal.5.4.0
|
|
92
|
+
|
|
93
|
+
Dependency updates only.
|
|
94
|
+
|
|
95
|
+
## 2.0.0-internal.5.3.0
|
|
96
|
+
|
|
97
|
+
Dependency updates only.
|
|
98
|
+
|
|
99
|
+
## 2.0.0-internal.5.2.0
|
|
100
|
+
|
|
101
|
+
Dependency updates only.
|
|
102
|
+
|
|
103
|
+
## 2.0.0-internal.5.1.0
|
|
104
|
+
|
|
105
|
+
Dependency updates only.
|
|
106
|
+
|
|
107
|
+
## 2.0.0-internal.5.0.0
|
|
108
|
+
|
|
109
|
+
Dependency updates only.
|
|
110
|
+
|
|
111
|
+
## 2.0.0-internal.4.4.0
|
|
112
|
+
|
|
113
|
+
Dependency updates only.
|
|
114
|
+
|
|
115
|
+
## 2.0.0-internal.4.1.0
|
|
116
|
+
|
|
117
|
+
Dependency updates only.
|
package/README.md
CHANGED
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
# @fluid-tools/fetch-tool
|
|
2
2
|
|
|
3
3
|
Connection using ODSP or routerlicious driver to dump the messages or snapshot information on the server.
|
|
4
|
-
In order to connect to ODSP, the clientID and clientSecret must be set as environment variables login__microsoft__clientId and login__microsoft__secret
|
|
4
|
+
In order to connect to ODSP, the clientID and clientSecret must be set as environment variables `login__microsoft__clientId` and `login__microsoft__secret`, respectively. If you have access to the keyvault this can be done by running [this tool](../../../tools/getkeys).
|
|
5
|
+
Beware that to use fetch-tool on documents in the Microsoft tenant, you will need to follow the fetch tool usage instructions on the "Debugging Tools" page of the internal Fluid wiki.
|
|
5
6
|
|
|
6
7
|
## Usage
|
|
7
8
|
|
|
@@ -24,6 +25,20 @@ In order to connect to ODSP, the clientID and clientSecret must be set as enviro
|
|
|
24
25
|
--snapshotVersionIndex <number> : Index of the version to dump
|
|
25
26
|
--saveDir <outdir> : Save data of the snapshots and messages
|
|
26
27
|
|
|
28
|
+
### Tips
|
|
29
|
+
|
|
30
|
+
- If not done already run `npm run build`
|
|
31
|
+
- Example command - run `node bin/fluid-fetch --saveDir example 'URL'` in the `fetch-tool` directory
|
|
32
|
+
- An example URL is something from office.com that looks like `https://www.office.com/launch/fluid/...`
|
|
33
|
+
- This command creates an `example` directory (if it doesn't exist) in the `fetch-tool` folder.
|
|
34
|
+
- If run multiple times without clearing the `example` directory, the snapshot will overwrite any old folders or files.
|
|
35
|
+
- Looking at the `example` directory:
|
|
36
|
+
- Go to `1-XYZ/decoded/tree.json` to see the snapshot tree.
|
|
37
|
+
- Each `'#-XYZ'` string in the `tree.json` correlates to a file in the decoded folder. These files are essentially blobs.
|
|
38
|
+
- `0-XYZ/decoded/tree.json` is an older snapshot tree.
|
|
39
|
+
- The `messages.json` is a list of ops/messages that are stored.
|
|
40
|
+
- For 401 authentication errors, as stated above, check that [getkeys](../../../tools/getkeys) has been run.
|
|
41
|
+
|
|
27
42
|
## Example Output
|
|
28
43
|
|
|
29
44
|
### Messages Stats
|
|
@@ -42,7 +57,6 @@ In order to connect to ODSP, the clientID and clientSecret must be set as enviro
|
|
|
42
57
|
----------------------------------------------------------------------------------------------------
|
|
43
58
|
Total | 105 38605
|
|
44
59
|
|
|
45
|
-
|
|
46
60
|
**--stat:dataType**
|
|
47
61
|
|
|
48
62
|
107 total messages (105 delta storage, 2 initial ws messages, 0 dup)
|
|
@@ -102,13 +116,30 @@ If you would like to debug fetch-tool, you can create a unit test. Remember to a
|
|
|
102
116
|
In the unit test, you can use `setArguments()` from fluidFetchArgs to pass in arguments you want to test. Then call the methods you want to run and you will be able to set breakpoints in vscode.
|
|
103
117
|
|
|
104
118
|
**Example**
|
|
119
|
+
|
|
105
120
|
```js
|
|
106
121
|
describe("fetch tool", () => {
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
122
|
+
it("can fetch messages", async () => {
|
|
123
|
+
fluidFetchArgs.setArguments(your_args);
|
|
124
|
+
const documentService = await fluidFetchInit(your_paramURL);
|
|
125
|
+
await fluidFetchMessages(documentService, your_saveDir);
|
|
126
|
+
});
|
|
112
127
|
});
|
|
113
128
|
```
|
|
114
129
|
|
|
130
|
+
<!-- AUTO-GENERATED-CONTENT:START (README_TRADEMARK_SECTION:includeHeading=TRUE) -->
|
|
131
|
+
|
|
132
|
+
<!-- prettier-ignore-start -->
|
|
133
|
+
<!-- NOTE: This section is automatically generated using @fluid-tools/markdown-magic. Do not update these generated contents directly. -->
|
|
134
|
+
|
|
135
|
+
## Trademark
|
|
136
|
+
|
|
137
|
+
This project may contain Microsoft trademarks or logos for Microsoft projects, products, or services.
|
|
138
|
+
|
|
139
|
+
Use of these trademarks or logos must follow Microsoft's [Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general).
|
|
140
|
+
|
|
141
|
+
Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship.
|
|
142
|
+
|
|
143
|
+
<!-- prettier-ignore-end -->
|
|
144
|
+
|
|
145
|
+
<!-- AUTO-GENERATED-CONTENT:END -->
|
package/bin/fluid-fetch
CHANGED
|
File without changes
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"fluidAnalyzeMessages.d.ts","sourceRoot":"","sources":["../src/fluidAnalyzeMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"fluidAnalyzeMessages.d.ts","sourceRoot":"","sources":["../src/fluidAnalyzeMessages.ts"],"names":[],"mappings":"AAAA;;;GAGG;AA2EH,eAAO,MAAM,YAAY,QAAS,MAAM,KAAG,MAEU,CAAC;AAgZtD,wBAAsB,iBAAiB,CACtC,SAAS,KAAA,EAAE,+CAA+C;AAC1D,gBAAgB,EAAE,OAAO,EACzB,YAAY,EAAE,OAAO,EACrB,iBAAiB,GAAE,GAAG,CAAC,MAAM,CAAqB,iBA0ClD"}
|
|
@@ -3,16 +3,9 @@
|
|
|
3
3
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
4
4
|
* Licensed under the MIT License.
|
|
5
5
|
*/
|
|
6
|
-
var __asyncValues = (this && this.__asyncValues) || function (o) {
|
|
7
|
-
if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
|
|
8
|
-
var m = o[Symbol.asyncIterator], i;
|
|
9
|
-
return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
|
|
10
|
-
function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
|
|
11
|
-
function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
|
|
12
|
-
};
|
|
13
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
14
7
|
exports.printMessageStats = exports.formatNumber = void 0;
|
|
15
|
-
const
|
|
8
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
16
9
|
const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
17
10
|
const container_runtime_1 = require("@fluidframework/container-runtime");
|
|
18
11
|
const datastore_1 = require("@fluidframework/datastore");
|
|
@@ -33,14 +26,14 @@ function incr(map, key, size, count = 1) {
|
|
|
33
26
|
* Helper class to track session statistics
|
|
34
27
|
*/
|
|
35
28
|
class ActiveSession {
|
|
29
|
+
static create(email, message) {
|
|
30
|
+
return new ActiveSession(email, message);
|
|
31
|
+
}
|
|
36
32
|
constructor(email, startMessage) {
|
|
37
33
|
this.email = email;
|
|
38
34
|
this.startMessage = startMessage;
|
|
39
35
|
this.opCount = 0;
|
|
40
36
|
}
|
|
41
|
-
static create(email, message) {
|
|
42
|
-
return new ActiveSession(email, message);
|
|
43
|
-
}
|
|
44
37
|
reportOp(timestamp) {
|
|
45
38
|
this.opCount++;
|
|
46
39
|
}
|
|
@@ -54,16 +47,16 @@ class ActiveSession {
|
|
|
54
47
|
}
|
|
55
48
|
}
|
|
56
49
|
// Format a number separating 3 digits by comma
|
|
50
|
+
const formatNumber = (num) =>
|
|
57
51
|
// eslint-disable-next-line unicorn/no-unsafe-regex
|
|
58
|
-
|
|
52
|
+
num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
59
53
|
exports.formatNumber = formatNumber;
|
|
60
54
|
function dumpStats(map, props) {
|
|
61
|
-
var _a;
|
|
62
55
|
const fieldSizes = [10, 14];
|
|
63
56
|
const nameLength = 72;
|
|
64
57
|
const fieldsLength = fieldSizes[0] + fieldSizes[1] + 1;
|
|
65
58
|
let headers = props.headers;
|
|
66
|
-
let recordsToShow =
|
|
59
|
+
let recordsToShow = props.lines ?? 10;
|
|
67
60
|
if (map.size !== recordsToShow && props.removeTotals === undefined && recordsToShow > 1) {
|
|
68
61
|
recordsToShow--;
|
|
69
62
|
}
|
|
@@ -116,11 +109,9 @@ function dumpStats(map, props) {
|
|
|
116
109
|
}
|
|
117
110
|
if (props.removeTotals === undefined) {
|
|
118
111
|
if (allOtherCount || allOtherSize) {
|
|
119
|
-
// eslint-disable-next-line max-len
|
|
120
112
|
console.log(`${`All Others (${sorted.length - recordsToShow})`.padEnd(nameLength)} │ ${(0, exports.formatNumber)(allOtherCount).padStart(fieldSizes[0])} ${(0, exports.formatNumber)(allOtherSize).padStart(fieldSizes[1])}`);
|
|
121
113
|
}
|
|
122
114
|
console.log(`${"─".repeat(nameLength + 1)}┼${"─".repeat(fieldsLength + 1)}`);
|
|
123
|
-
// eslint-disable-next-line max-len
|
|
124
115
|
console.log(`${"Total".padEnd(nameLength)} │ ${(0, exports.formatNumber)(totalCount).padStart(fieldSizes[0])} ${(0, exports.formatNumber)(sizeTotal).padStart(fieldSizes[1])}`);
|
|
125
116
|
}
|
|
126
117
|
}
|
|
@@ -181,6 +172,7 @@ class DataStructureAnalyzer {
|
|
|
181
172
|
this.dataType = new Map();
|
|
182
173
|
this.dataTypeStats = new Map();
|
|
183
174
|
this.objectStats = new Map();
|
|
175
|
+
// eslint-disable-next-line @typescript-eslint/member-delimiter-style
|
|
184
176
|
this.chunkMap = new Map();
|
|
185
177
|
}
|
|
186
178
|
processOp(message, msgSize, skipMessage) {
|
|
@@ -231,7 +223,6 @@ class FilteredMessageAnalyzer {
|
|
|
231
223
|
}
|
|
232
224
|
reportAnalyzes(lastOp) {
|
|
233
225
|
if (this.filtered) {
|
|
234
|
-
// eslint-disable-next-line max-len
|
|
235
226
|
console.log(`\nData is filtered according to --filter:messageType argument(s):\nOp size: ${this.sizeFiltered} / ${this.sizeTotal}\nOp count ${this.opsFiltered} / ${this.opsTotal}`);
|
|
236
227
|
}
|
|
237
228
|
if (this.opsTotal === 0) {
|
|
@@ -256,7 +247,6 @@ class MessageDensityAnalyzer {
|
|
|
256
247
|
if (message.sequenceNumber !== 1) {
|
|
257
248
|
const timeDiff = durationFromTime(message.timestamp - this.timeStart);
|
|
258
249
|
const opsString = `ops = [${this.opLimit - this.opChunk}, ${this.opLimit - 1}]`.padEnd(26);
|
|
259
|
-
// eslint-disable-next-line max-len
|
|
260
250
|
const timeString = `time = [${durationFromTime(this.timeStart - this.doctimerStart)}, ${durationFromTime(message.timestamp - this.doctimerStart)}]`;
|
|
261
251
|
this.ranges.set(`${opsString} ${timeString}`, [timeDiff, this.size]);
|
|
262
252
|
}
|
|
@@ -328,7 +318,8 @@ class SummaryAnalyzer {
|
|
|
328
318
|
this.lastSummaryOp = message.sequenceNumber;
|
|
329
319
|
}
|
|
330
320
|
if (message.type === protocol_definitions_1.MessageType.SummaryAck || message.type === protocol_definitions_1.MessageType.SummaryNack) {
|
|
331
|
-
const contents = message.contents
|
|
321
|
+
const contents = message.contents
|
|
322
|
+
.summaryProposal;
|
|
332
323
|
const distance = message.sequenceNumber - contents.summarySequenceNumber;
|
|
333
324
|
if (distance > this.maxResponse) {
|
|
334
325
|
this.maxResponse = distance;
|
|
@@ -362,12 +353,10 @@ class MessageDumper {
|
|
|
362
353
|
console.log(JSON.stringify(message, undefined, 2));
|
|
363
354
|
}
|
|
364
355
|
}
|
|
365
|
-
reportAnalyzes(lastOp) {
|
|
366
|
-
}
|
|
356
|
+
reportAnalyzes(lastOp) { }
|
|
367
357
|
}
|
|
368
358
|
async function printMessageStats(generator, // AsyncGenerator<ISequencedDocumentMessage[]>,
|
|
369
359
|
dumpMessageStats, dumpMessages, messageTypeFilter = new Set()) {
|
|
370
|
-
var e_1, _a;
|
|
371
360
|
let lastMessage;
|
|
372
361
|
const analyzers = [
|
|
373
362
|
new FilteredMessageAnalyzer(),
|
|
@@ -380,26 +369,16 @@ dumpMessageStats, dumpMessages, messageTypeFilter = new Set()) {
|
|
|
380
369
|
if (dumpMessages) {
|
|
381
370
|
analyzers.push(new MessageDumper());
|
|
382
371
|
}
|
|
383
|
-
|
|
384
|
-
for (
|
|
385
|
-
const
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
for (const analyzer of analyzers) {
|
|
391
|
-
analyzer.processOp(message, msgSize, skipMessage);
|
|
392
|
-
}
|
|
372
|
+
for await (const messages of generator) {
|
|
373
|
+
for (const message of messages) {
|
|
374
|
+
const msgSize = JSON.stringify(message).length;
|
|
375
|
+
lastMessage = message;
|
|
376
|
+
const skipMessage = messageTypeFilter.size !== 0 && !messageTypeFilter.has(message.type);
|
|
377
|
+
for (const analyzer of analyzers) {
|
|
378
|
+
analyzer.processOp(message, msgSize, skipMessage);
|
|
393
379
|
}
|
|
394
380
|
}
|
|
395
381
|
}
|
|
396
|
-
catch (e_1_1) { e_1 = { error: e_1_1 }; }
|
|
397
|
-
finally {
|
|
398
|
-
try {
|
|
399
|
-
if (generator_1_1 && !generator_1_1.done && (_a = generator_1.return)) await _a.call(generator_1);
|
|
400
|
-
}
|
|
401
|
-
finally { if (e_1) throw e_1.error; }
|
|
402
|
-
}
|
|
403
382
|
if (lastMessage !== undefined) {
|
|
404
383
|
if (dumpMessageStats) {
|
|
405
384
|
for (const analyzer of analyzers) {
|
|
@@ -414,34 +393,41 @@ dumpMessageStats, dumpMessages, messageTypeFilter = new Set()) {
|
|
|
414
393
|
console.log("");
|
|
415
394
|
}
|
|
416
395
|
exports.printMessageStats = printMessageStats;
|
|
417
|
-
function processOp(
|
|
418
|
-
let type =
|
|
396
|
+
function processOp(runtimeMessage, dataType, objectStats, msgSize, dataTypeStats, messageTypeStats, chunkMap) {
|
|
397
|
+
let type = runtimeMessage.type;
|
|
419
398
|
let recorded = false;
|
|
420
399
|
let totalMsgSize = msgSize;
|
|
421
400
|
let opCount = 1;
|
|
422
|
-
if ((0, container_runtime_1.
|
|
423
|
-
let runtimeMessage = (0, container_runtime_1.unpackRuntimeMessage)(message);
|
|
401
|
+
if ((0, container_runtime_1.unpackRuntimeMessage)(runtimeMessage)) {
|
|
424
402
|
const messageType = runtimeMessage.type;
|
|
425
403
|
switch (messageType) {
|
|
426
|
-
case container_runtime_1.
|
|
404
|
+
case container_runtime_1.ContainerMessageType.Attach: {
|
|
427
405
|
const attachMessage = runtimeMessage.contents;
|
|
428
406
|
processDataStoreAttachOp(attachMessage, dataType);
|
|
429
407
|
break;
|
|
430
408
|
}
|
|
431
409
|
// skip for now because these ops do not have contents
|
|
432
|
-
case container_runtime_1.
|
|
410
|
+
case container_runtime_1.ContainerMessageType.BlobAttach: {
|
|
411
|
+
break;
|
|
412
|
+
}
|
|
413
|
+
// The default method to count stats should be used for GC messages.
|
|
414
|
+
case container_runtime_1.ContainerMessageType.GC: {
|
|
433
415
|
break;
|
|
434
416
|
}
|
|
435
|
-
case container_runtime_1.
|
|
417
|
+
case container_runtime_1.ContainerMessageType.ChunkedOp: {
|
|
436
418
|
const chunk = runtimeMessage.contents;
|
|
419
|
+
// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
|
|
420
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
437
421
|
if (!chunkMap.has(runtimeMessage.clientId)) {
|
|
422
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
438
423
|
chunkMap.set(runtimeMessage.clientId, {
|
|
439
424
|
chunks: new Array(chunk.totalChunks),
|
|
440
425
|
totalSize: 0,
|
|
441
426
|
});
|
|
442
427
|
}
|
|
428
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
443
429
|
const value = chunkMap.get(runtimeMessage.clientId);
|
|
444
|
-
(0,
|
|
430
|
+
(0, core_utils_1.assert)(value !== undefined, 0x2b8 /* "Chunk should be set in map" */);
|
|
445
431
|
const chunks = value.chunks;
|
|
446
432
|
const chunkIndex = chunk.chunkId - 1;
|
|
447
433
|
if (chunks[chunkIndex] !== undefined) {
|
|
@@ -451,78 +437,77 @@ function processOp(message, dataType, objectStats, msgSize, dataTypeStats, messa
|
|
|
451
437
|
value.totalSize += msgSize;
|
|
452
438
|
if (chunk.chunkId === chunk.totalChunks) {
|
|
453
439
|
opCount = chunk.totalChunks; // 1 op for each chunk.
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
440
|
+
const patchedMessage = Object.create(runtimeMessage);
|
|
441
|
+
patchedMessage.contents = chunks.join("");
|
|
442
|
+
patchedMessage.type = chunk.originalType;
|
|
457
443
|
type = chunk.originalType;
|
|
458
444
|
totalMsgSize = value.totalSize;
|
|
459
|
-
chunkMap.delete(
|
|
445
|
+
chunkMap.delete(patchedMessage.clientId);
|
|
460
446
|
}
|
|
461
447
|
else {
|
|
462
448
|
return;
|
|
463
449
|
}
|
|
464
450
|
// eslint-disable-next-line no-fallthrough
|
|
465
451
|
}
|
|
466
|
-
case container_runtime_1.
|
|
467
|
-
case container_runtime_1.
|
|
468
|
-
case container_runtime_1.
|
|
469
|
-
case container_runtime_1.
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
452
|
+
case container_runtime_1.ContainerMessageType.IdAllocation:
|
|
453
|
+
case container_runtime_1.ContainerMessageType.FluidDataStoreOp:
|
|
454
|
+
case container_runtime_1.ContainerMessageType.Alias:
|
|
455
|
+
case container_runtime_1.ContainerMessageType.Rejoin: {
|
|
456
|
+
let envelope = runtimeMessage.contents;
|
|
457
|
+
// TODO: Legacy?
|
|
458
|
+
if (envelope !== undefined && typeof envelope === "string") {
|
|
459
|
+
envelope = JSON.parse(envelope);
|
|
460
|
+
}
|
|
461
|
+
const innerContent = envelope.contents;
|
|
462
|
+
const address = envelope.address;
|
|
463
|
+
type = `${type}/${innerContent.type}`;
|
|
464
|
+
switch (innerContent.type) {
|
|
465
|
+
case datastore_1.DataStoreMessageType.Attach: {
|
|
466
|
+
const attachMessage = innerContent.content;
|
|
467
|
+
let objectType = attachMessage.type;
|
|
468
|
+
if (objectType.startsWith(objectTypePrefix)) {
|
|
469
|
+
objectType = objectType.substring(objectTypePrefix.length);
|
|
470
|
+
}
|
|
471
|
+
dataType.set(getObjectId(address, attachMessage.id), objectType);
|
|
472
|
+
break;
|
|
475
473
|
}
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
break;
|
|
474
|
+
case datastore_1.DataStoreMessageType.ChannelOp:
|
|
475
|
+
default: {
|
|
476
|
+
const innerEnvelope = innerContent.content;
|
|
477
|
+
const innerContent2 = innerEnvelope.contents;
|
|
478
|
+
const objectId = getObjectId(address, innerEnvelope.address);
|
|
479
|
+
incr(objectStats, objectId, totalMsgSize, opCount);
|
|
480
|
+
let objectType = dataType.get(objectId);
|
|
481
|
+
if (objectType === undefined) {
|
|
482
|
+
// Somehow we do not have data...
|
|
483
|
+
dataType.set(objectId, objectId);
|
|
484
|
+
objectType = objectId;
|
|
488
485
|
}
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
recorded = true;
|
|
503
|
-
let subType = innerContent2.type;
|
|
504
|
-
if (innerContent2.type === "set" &&
|
|
505
|
-
typeof innerContent2.value === "object" &&
|
|
506
|
-
innerContent2.value !== null) {
|
|
507
|
-
type = `${type}/${subType}`;
|
|
508
|
-
subType = innerContent2.value.type;
|
|
509
|
-
}
|
|
510
|
-
else if (objectType === "mergeTree" && subType !== undefined) {
|
|
511
|
-
const types = ["insert", "remove", "annotate", "group"];
|
|
512
|
-
if (types[subType] !== undefined) {
|
|
513
|
-
subType = types[subType];
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
if (subType !== undefined) {
|
|
517
|
-
type = `${type}/${subType}`;
|
|
486
|
+
incr(dataTypeStats, objectType, totalMsgSize, opCount);
|
|
487
|
+
recorded = true;
|
|
488
|
+
let subType = innerContent2.type;
|
|
489
|
+
if (innerContent2.type === "set" &&
|
|
490
|
+
typeof innerContent2.value === "object" &&
|
|
491
|
+
innerContent2.value !== null) {
|
|
492
|
+
type = `${type}/${subType}`;
|
|
493
|
+
subType = innerContent2.value.type;
|
|
494
|
+
}
|
|
495
|
+
else if (objectType === "mergeTree" && subType !== undefined) {
|
|
496
|
+
const types = ["insert", "remove", "annotate", "group"];
|
|
497
|
+
if (types[subType] !== undefined) {
|
|
498
|
+
subType = types[subType];
|
|
518
499
|
}
|
|
519
|
-
type = `${type} (${objectType})`;
|
|
520
500
|
}
|
|
501
|
+
if (subType !== undefined) {
|
|
502
|
+
type = `${type}/${subType}`;
|
|
503
|
+
}
|
|
504
|
+
type = `${type} (${objectType})`;
|
|
521
505
|
}
|
|
522
|
-
break;
|
|
523
506
|
}
|
|
507
|
+
break;
|
|
508
|
+
}
|
|
524
509
|
default:
|
|
525
|
-
(0,
|
|
510
|
+
(0, core_utils_1.unreachableCase)(messageType, "Message type not recognized!");
|
|
526
511
|
}
|
|
527
512
|
}
|
|
528
513
|
incr(messageTypeStats, type, totalMsgSize, opCount);
|
|
@@ -541,13 +526,7 @@ function processDataStoreAttachOp(attachMessage, dataType) {
|
|
|
541
526
|
// dataType.set(getObjectId(attachMessage.id), attachMessage.type);
|
|
542
527
|
// That's data store, and it brings a bunch of data structures.
|
|
543
528
|
// Let's try to crack it.
|
|
544
|
-
|
|
545
|
-
if (typeof attachMessage === "string") {
|
|
546
|
-
parsedAttachMessage = JSON.parse(attachMessage);
|
|
547
|
-
}
|
|
548
|
-
else {
|
|
549
|
-
parsedAttachMessage = attachMessage;
|
|
550
|
-
}
|
|
529
|
+
const parsedAttachMessage = typeof attachMessage === "string" ? JSON.parse(attachMessage) : attachMessage;
|
|
551
530
|
for (const entry of parsedAttachMessage.snapshot.entries) {
|
|
552
531
|
if (entry.type === protocol_definitions_1.TreeEntry.Tree) {
|
|
553
532
|
for (const entry2 of entry.value.entries) {
|
|
@@ -569,12 +548,18 @@ function reportOpenSessions(lastOpTimestamp, sessionsInProgress, sessions, users
|
|
|
569
548
|
const sessionInfo = ses.leave(lastOpTimestamp);
|
|
570
549
|
if (clientId !== noClientName) {
|
|
571
550
|
const sessionName = `${clientId} (${sessionInfo.email})`;
|
|
572
|
-
const sessionPayload = [
|
|
551
|
+
const sessionPayload = [
|
|
552
|
+
durationFromTime(sessionInfo.duration),
|
|
553
|
+
sessionInfo.opCount,
|
|
554
|
+
];
|
|
573
555
|
sessions.set(sessionName, sessionPayload);
|
|
574
556
|
activeSessions.set(sessionName, sessionPayload);
|
|
575
557
|
}
|
|
576
558
|
else {
|
|
577
|
-
sessions.set(`Full file lifespan (noClient messages)`, [
|
|
559
|
+
sessions.set(`Full file lifespan (noClient messages)`, [
|
|
560
|
+
durationFromTime(sessionInfo.duration),
|
|
561
|
+
sessionInfo.opCount,
|
|
562
|
+
]);
|
|
578
563
|
}
|
|
579
564
|
incr(users, sessionInfo.email, sessionInfo.opCount);
|
|
580
565
|
}
|
|
@@ -616,23 +601,28 @@ function processQuorumMessages(message, skipMessage, sessionsInProgress, session
|
|
|
616
601
|
const clientId = JSON.parse(dataString);
|
|
617
602
|
session = sessionsInProgress.get(clientId);
|
|
618
603
|
sessionsInProgress.delete(clientId);
|
|
619
|
-
(0,
|
|
604
|
+
(0, core_utils_1.assert)(!!session, 0x1b7 /* "Bad session state for processing quorum messages" */);
|
|
620
605
|
if (session !== undefined) {
|
|
621
606
|
if (!skipMessage) {
|
|
622
607
|
session.reportOp(message.timestamp);
|
|
623
608
|
}
|
|
624
609
|
const sessionInfo = session.leave(message.timestamp);
|
|
625
|
-
sessions.set(`${clientId} (${sessionInfo.email})`, [
|
|
610
|
+
sessions.set(`${clientId} (${sessionInfo.email})`, [
|
|
611
|
+
durationFromTime(sessionInfo.duration),
|
|
612
|
+
sessionInfo.opCount,
|
|
613
|
+
]);
|
|
626
614
|
incr(users, sessionInfo.email, sessionInfo.opCount);
|
|
627
615
|
session = undefined; // Do not record it second time
|
|
628
616
|
}
|
|
629
617
|
}
|
|
630
618
|
else {
|
|
631
619
|
// message.clientId can be null
|
|
620
|
+
// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
|
|
621
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
632
622
|
session = sessionsInProgress.get(message.clientId);
|
|
633
623
|
if (session === undefined) {
|
|
634
624
|
session = sessionsInProgress.get(noClientName);
|
|
635
|
-
(0,
|
|
625
|
+
(0, core_utils_1.assert)(!!session, 0x1b8 /* "Bad session state for processing quorum messages" */);
|
|
636
626
|
}
|
|
637
627
|
}
|
|
638
628
|
return session;
|