@fluid-tools/fetch-tool 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +6 -8
- package/CHANGELOG.md +117 -0
- package/README.md +38 -7
- package/bin/fluid-fetch +0 -0
- package/dist/fluidAnalyzeMessages.d.ts.map +1 -1
- package/dist/fluidAnalyzeMessages.js +106 -116
- package/dist/fluidAnalyzeMessages.js.map +1 -1
- package/dist/fluidFetch.js +5 -3
- package/dist/fluidFetch.js.map +1 -1
- package/dist/fluidFetchArgs.d.ts +0 -3
- package/dist/fluidFetchArgs.d.ts.map +1 -1
- package/dist/fluidFetchArgs.js +10 -14
- package/dist/fluidFetchArgs.js.map +1 -1
- package/dist/fluidFetchInit.d.ts +0 -1
- package/dist/fluidFetchInit.d.ts.map +1 -1
- package/dist/fluidFetchInit.js +41 -34
- package/dist/fluidFetchInit.js.map +1 -1
- package/dist/fluidFetchMessages.d.ts.map +1 -1
- package/dist/fluidFetchMessages.js +168 -200
- package/dist/fluidFetchMessages.js.map +1 -1
- package/dist/fluidFetchSharePoint.d.ts +0 -1
- package/dist/fluidFetchSharePoint.d.ts.map +1 -1
- package/dist/fluidFetchSharePoint.js +20 -6
- package/dist/fluidFetchSharePoint.js.map +1 -1
- package/dist/fluidFetchSnapshot.d.ts.map +1 -1
- package/dist/fluidFetchSnapshot.js +18 -20
- package/dist/fluidFetchSnapshot.js.map +1 -1
- package/package.json +47 -42
- package/prettier.config.cjs +8 -0
- package/src/fluidAnalyzeMessages.ts +701 -630
- package/src/fluidFetch.ts +93 -88
- package/src/fluidFetchArgs.ts +167 -168
- package/src/fluidFetchInit.ts +133 -104
- package/src/fluidFetchMessages.ts +253 -232
- package/src/fluidFetchSharePoint.ts +130 -112
- package/src/fluidFetchSnapshot.ts +313 -295
- package/tsconfig.json +8 -15
|
@@ -3,19 +3,20 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { assert, unreachableCase } from "@fluidframework/
|
|
6
|
+
import { assert, unreachableCase } from "@fluidframework/core-utils";
|
|
7
7
|
import {
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
8
|
+
ISequencedDocumentMessage,
|
|
9
|
+
ISummaryAck,
|
|
10
|
+
ISummaryNack,
|
|
11
|
+
ISummaryProposal,
|
|
12
|
+
MessageType,
|
|
13
|
+
TreeEntry,
|
|
12
14
|
} from "@fluidframework/protocol-definitions";
|
|
13
15
|
import { IAttachMessage, IEnvelope } from "@fluidframework/runtime-definitions";
|
|
14
16
|
import {
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
unpackRuntimeMessage,
|
|
17
|
+
IChunkedOp,
|
|
18
|
+
ContainerMessageType,
|
|
19
|
+
unpackRuntimeMessage,
|
|
19
20
|
} from "@fluidframework/container-runtime";
|
|
20
21
|
import { DataStoreMessageType } from "@fluidframework/datastore";
|
|
21
22
|
|
|
@@ -23,139 +24,155 @@ const noClientName = "No Client";
|
|
|
23
24
|
const objectTypePrefix = "https://graph.microsoft.com/types/";
|
|
24
25
|
|
|
25
26
|
function incr(map: Map<string, [number, number]>, key: string, size: number, count = 1) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
27
|
+
const value = map.get(key);
|
|
28
|
+
if (value === undefined) {
|
|
29
|
+
map.set(key, [count, size]);
|
|
30
|
+
} else {
|
|
31
|
+
value[0] += count;
|
|
32
|
+
value[1] += size;
|
|
33
|
+
map.set(key, value);
|
|
34
|
+
}
|
|
34
35
|
}
|
|
35
36
|
|
|
36
37
|
interface ISessionInfo {
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
38
|
+
startSeq: number;
|
|
39
|
+
opCount: number;
|
|
40
|
+
email: string;
|
|
41
|
+
duration: number;
|
|
41
42
|
}
|
|
42
43
|
|
|
43
44
|
interface IMessageAnalyzer {
|
|
44
|
-
|
|
45
|
-
|
|
45
|
+
processOp(op: ISequencedDocumentMessage, msgSize: number, filteredOutOp: boolean): void;
|
|
46
|
+
reportAnalyzes(lastOp: ISequencedDocumentMessage): void;
|
|
46
47
|
}
|
|
47
48
|
|
|
48
49
|
/**
|
|
49
50
|
* Helper class to track session statistics
|
|
50
51
|
*/
|
|
51
52
|
class ActiveSession {
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
53
|
+
public static create(email: string, message: ISequencedDocumentMessage) {
|
|
54
|
+
return new ActiveSession(email, message);
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
private opCount = 0;
|
|
58
|
+
|
|
59
|
+
constructor(
|
|
60
|
+
private readonly email: string,
|
|
61
|
+
private readonly startMessage: ISequencedDocumentMessage,
|
|
62
|
+
) {}
|
|
63
|
+
|
|
64
|
+
public reportOp(timestamp: number) {
|
|
65
|
+
this.opCount++;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
public leave(timestamp: number): ISessionInfo {
|
|
69
|
+
return {
|
|
70
|
+
opCount: this.opCount,
|
|
71
|
+
email: this.email,
|
|
72
|
+
startSeq: this.startMessage.sequenceNumber,
|
|
73
|
+
duration: timestamp - this.startMessage.timestamp,
|
|
74
|
+
};
|
|
75
|
+
}
|
|
73
76
|
}
|
|
74
77
|
|
|
75
78
|
// Format a number separating 3 digits by comma
|
|
76
|
-
|
|
77
|
-
|
|
79
|
+
export const formatNumber = (num: number): string =>
|
|
80
|
+
// eslint-disable-next-line unicorn/no-unsafe-regex
|
|
81
|
+
num.toString().replace(/\B(?=(\d{3})+(?!\d))/g, ",");
|
|
78
82
|
|
|
79
83
|
function dumpStats(
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
84
|
+
map: Map<string, [number, number]>,
|
|
85
|
+
props: {
|
|
86
|
+
title: string;
|
|
87
|
+
headers: [string, string];
|
|
88
|
+
lines?: number;
|
|
89
|
+
orderByFirstColumn?: boolean;
|
|
90
|
+
reverseColumnsInUI?: boolean;
|
|
91
|
+
removeTotals?: boolean;
|
|
92
|
+
reverseSort?: boolean;
|
|
93
|
+
},
|
|
94
|
+
) {
|
|
95
|
+
const fieldSizes = [10, 14];
|
|
96
|
+
const nameLength = 72;
|
|
97
|
+
const fieldsLength = fieldSizes[0] + fieldSizes[1] + 1;
|
|
98
|
+
let headers = props.headers;
|
|
99
|
+
|
|
100
|
+
let recordsToShow = props.lines ?? 10;
|
|
101
|
+
if (map.size !== recordsToShow && props.removeTotals === undefined && recordsToShow > 1) {
|
|
102
|
+
recordsToShow--;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
let sorted: [string, [number, number]][];
|
|
106
|
+
const sortIndex = props.orderByFirstColumn === true ? 0 : 1;
|
|
107
|
+
let add: string;
|
|
108
|
+
if (props.reverseSort !== undefined) {
|
|
109
|
+
sorted = [...map.entries()].sort((a, b) => a[1][sortIndex] - b[1][sortIndex]);
|
|
110
|
+
add = "↑";
|
|
111
|
+
} else {
|
|
112
|
+
sorted = [...map.entries()].sort((a, b) => b[1][sortIndex] - a[1][sortIndex]);
|
|
113
|
+
add = "↓";
|
|
114
|
+
}
|
|
115
|
+
headers[sortIndex] = `${headers[sortIndex]} ${add}`;
|
|
116
|
+
|
|
117
|
+
if (props.reverseColumnsInUI !== undefined) {
|
|
118
|
+
headers = [headers[1], headers[0]];
|
|
119
|
+
const sorted2: [string, [number, number]][] = [];
|
|
120
|
+
for (const [name, [count, size]] of sorted) {
|
|
121
|
+
sorted2.push([name, [size, count]]);
|
|
122
|
+
}
|
|
123
|
+
sorted = sorted2;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
let totalCount = 0;
|
|
127
|
+
let sizeTotal = 0;
|
|
128
|
+
|
|
129
|
+
props.title = `${props.title} (${sorted.length})`;
|
|
130
|
+
const header0 = headers[0].padStart(fieldSizes[0]);
|
|
131
|
+
let overflow = header0.length - fieldSizes[0];
|
|
132
|
+
console.log(
|
|
133
|
+
`\n\n${props.title.padEnd(nameLength)} │ ${header0} ${headers[1].padStart(
|
|
134
|
+
fieldSizes[1] - overflow,
|
|
135
|
+
)}`,
|
|
136
|
+
);
|
|
137
|
+
|
|
138
|
+
console.log(`${"─".repeat(nameLength + 1)}┼${"─".repeat(fieldsLength + 1)}`);
|
|
139
|
+
let index = 0;
|
|
140
|
+
let allOtherCount = 0;
|
|
141
|
+
let allOtherSize = 0;
|
|
142
|
+
for (const [name, [count, size]] of sorted) {
|
|
143
|
+
index++;
|
|
144
|
+
totalCount += count;
|
|
145
|
+
sizeTotal += size;
|
|
146
|
+
if (index <= recordsToShow) {
|
|
147
|
+
const item = name.padEnd(nameLength);
|
|
148
|
+
overflow = item.length - nameLength;
|
|
149
|
+
const col1 = formatNumber(count).padStart(fieldSizes[0] - overflow);
|
|
150
|
+
overflow += col1.length - fieldSizes[0];
|
|
151
|
+
const col2 = formatNumber(size).padStart(fieldSizes[1] - overflow);
|
|
152
|
+
console.log(`${item} │ ${col1} ${col2}`);
|
|
153
|
+
} else {
|
|
154
|
+
allOtherCount += count;
|
|
155
|
+
allOtherSize += size;
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
if (props.removeTotals === undefined) {
|
|
160
|
+
if (allOtherCount || allOtherSize) {
|
|
161
|
+
console.log(
|
|
162
|
+
`${`All Others (${sorted.length - recordsToShow})`.padEnd(
|
|
163
|
+
nameLength,
|
|
164
|
+
)} │ ${formatNumber(allOtherCount).padStart(fieldSizes[0])} ${formatNumber(
|
|
165
|
+
allOtherSize,
|
|
166
|
+
).padStart(fieldSizes[1])}`,
|
|
167
|
+
);
|
|
168
|
+
}
|
|
169
|
+
console.log(`${"─".repeat(nameLength + 1)}┼${"─".repeat(fieldsLength + 1)}`);
|
|
170
|
+
console.log(
|
|
171
|
+
`${"Total".padEnd(nameLength)} │ ${formatNumber(totalCount).padStart(
|
|
172
|
+
fieldSizes[0],
|
|
173
|
+
)} ${formatNumber(sizeTotal).padStart(fieldSizes[1])}`,
|
|
174
|
+
);
|
|
175
|
+
}
|
|
159
176
|
}
|
|
160
177
|
|
|
161
178
|
const getObjectId = (dataStoreId: string, id: string) => `[${dataStoreId}]/${id}`;
|
|
@@ -164,569 +181,623 @@ const getObjectId = (dataStoreId: string, id: string) => `[${dataStoreId}]/${id}
|
|
|
164
181
|
* Analyzer for sessions
|
|
165
182
|
*/
|
|
166
183
|
class SessionAnalyzer implements IMessageAnalyzer {
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
184
|
+
private readonly sessionsInProgress = new Map<string, ActiveSession>();
|
|
185
|
+
private readonly sessions = new Map<string, [number, number]>();
|
|
186
|
+
private readonly users = new Map<string, [number, number]>();
|
|
187
|
+
|
|
188
|
+
private first = true;
|
|
189
|
+
|
|
190
|
+
public processOp(
|
|
191
|
+
message: ISequencedDocumentMessage,
|
|
192
|
+
msgSize: number,
|
|
193
|
+
skipMessage: boolean,
|
|
194
|
+
): void {
|
|
195
|
+
if (this.first) {
|
|
196
|
+
this.first = false;
|
|
197
|
+
// Start of the road.
|
|
198
|
+
const noNameSession = ActiveSession.create(noClientName, message);
|
|
199
|
+
this.sessionsInProgress.set(noClientName, noNameSession);
|
|
200
|
+
}
|
|
201
|
+
const session = processQuorumMessages(
|
|
202
|
+
message,
|
|
203
|
+
skipMessage,
|
|
204
|
+
this.sessionsInProgress,
|
|
205
|
+
this.sessions,
|
|
206
|
+
this.users,
|
|
207
|
+
);
|
|
208
|
+
if (!skipMessage && session) {
|
|
209
|
+
session.reportOp(message.timestamp);
|
|
210
|
+
}
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
214
|
+
// Close any open sessions
|
|
215
|
+
reportOpenSessions(lastOp.timestamp, this.sessionsInProgress, this.sessions, this.users);
|
|
216
|
+
dumpStats(this.users, {
|
|
217
|
+
title: "Users",
|
|
218
|
+
headers: ["Sessions", "Op count"],
|
|
219
|
+
reverseColumnsInUI: true,
|
|
220
|
+
lines: 6,
|
|
221
|
+
});
|
|
222
|
+
dumpStats(this.sessions, {
|
|
223
|
+
title: "Sessions",
|
|
224
|
+
headers: ["Duration(s)", "Op count"],
|
|
225
|
+
reverseColumnsInUI: true,
|
|
226
|
+
lines: 6,
|
|
227
|
+
});
|
|
228
|
+
dumpStats(this.sessions, {
|
|
229
|
+
title: "Sessions",
|
|
230
|
+
headers: ["Duration(s)", "Op count"],
|
|
231
|
+
orderByFirstColumn: true,
|
|
232
|
+
reverseColumnsInUI: true,
|
|
233
|
+
removeTotals: true,
|
|
234
|
+
lines: 5,
|
|
235
|
+
});
|
|
236
|
+
}
|
|
219
237
|
}
|
|
220
238
|
|
|
221
239
|
/**
|
|
222
240
|
* Analyzer for data structures
|
|
223
241
|
*/
|
|
224
242
|
class DataStructureAnalyzer implements IMessageAnalyzer {
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
243
|
+
private readonly messageTypeStats = new Map<string, [number, number]>();
|
|
244
|
+
private readonly dataType = new Map<string, string>();
|
|
245
|
+
private readonly dataTypeStats = new Map<string, [number, number]>();
|
|
246
|
+
private readonly objectStats = new Map<string, [number, number]>();
|
|
247
|
+
// eslint-disable-next-line @typescript-eslint/member-delimiter-style
|
|
248
|
+
private readonly chunkMap = new Map<string, { chunks: string[]; totalSize: number }>();
|
|
249
|
+
|
|
250
|
+
public processOp(
|
|
251
|
+
message: ISequencedDocumentMessage,
|
|
252
|
+
msgSize: number,
|
|
253
|
+
skipMessage: boolean,
|
|
254
|
+
): void {
|
|
255
|
+
if (!skipMessage) {
|
|
256
|
+
processOp(
|
|
257
|
+
message,
|
|
258
|
+
this.dataType,
|
|
259
|
+
this.objectStats,
|
|
260
|
+
msgSize,
|
|
261
|
+
this.dataTypeStats,
|
|
262
|
+
this.messageTypeStats,
|
|
263
|
+
this.chunkMap,
|
|
264
|
+
);
|
|
265
|
+
}
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
269
|
+
dumpStats(this.messageTypeStats, {
|
|
270
|
+
title: "Message Type",
|
|
271
|
+
headers: ["Op count", "Bytes"],
|
|
272
|
+
lines: 20,
|
|
273
|
+
});
|
|
274
|
+
dumpStats(calcChannelStats(this.dataType, this.objectStats), {
|
|
275
|
+
title: "Channel name",
|
|
276
|
+
headers: ["Op count", "Bytes"],
|
|
277
|
+
lines: 7,
|
|
278
|
+
});
|
|
279
|
+
/*
|
|
256
280
|
dumpStats(this.dataTypeStats, {
|
|
257
281
|
title: "Channel type",
|
|
258
282
|
headers: ["Op count", "Bytes"],
|
|
259
283
|
});
|
|
260
284
|
*/
|
|
261
|
-
|
|
285
|
+
}
|
|
262
286
|
}
|
|
263
287
|
|
|
264
288
|
/**
|
|
265
289
|
* Helper class to report if we filtered out any messages.
|
|
266
290
|
*/
|
|
267
291
|
class FilteredMessageAnalyzer implements IMessageAnalyzer {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
292
|
+
private sizeTotal = 0;
|
|
293
|
+
private opsTotal = 0;
|
|
294
|
+
private sizeFiltered = 0;
|
|
295
|
+
private opsFiltered = 0;
|
|
296
|
+
private filtered = false;
|
|
297
|
+
|
|
298
|
+
public processOp(
|
|
299
|
+
message: ISequencedDocumentMessage,
|
|
300
|
+
msgSize: number,
|
|
301
|
+
skipMessage: boolean,
|
|
302
|
+
): void {
|
|
303
|
+
this.sizeTotal += msgSize;
|
|
304
|
+
this.opsTotal++;
|
|
305
|
+
if (!skipMessage) {
|
|
306
|
+
this.sizeFiltered += msgSize;
|
|
307
|
+
this.opsFiltered++;
|
|
308
|
+
} else {
|
|
309
|
+
this.filtered = true;
|
|
310
|
+
}
|
|
311
|
+
}
|
|
312
|
+
|
|
313
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
314
|
+
if (this.filtered) {
|
|
315
|
+
console.log(
|
|
316
|
+
`\nData is filtered according to --filter:messageType argument(s):\nOp size: ${this.sizeFiltered} / ${this.sizeTotal}\nOp count ${this.opsFiltered} / ${this.opsTotal}`,
|
|
317
|
+
);
|
|
318
|
+
}
|
|
319
|
+
if (this.opsTotal === 0) {
|
|
320
|
+
console.error("No ops were found");
|
|
321
|
+
}
|
|
322
|
+
}
|
|
294
323
|
}
|
|
295
324
|
|
|
296
325
|
/**
|
|
297
326
|
* Helper class to find places where we generated too many ops
|
|
298
327
|
*/
|
|
299
328
|
class MessageDensityAnalyzer implements IMessageAnalyzer {
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
320
|
-
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
|
|
334
|
-
|
|
335
|
-
|
|
336
|
-
|
|
337
|
-
|
|
338
|
-
|
|
329
|
+
private readonly opChunk = 1000;
|
|
330
|
+
private opLimit = 1;
|
|
331
|
+
private size = 0;
|
|
332
|
+
private timeStart = 0;
|
|
333
|
+
private doctimerStart = 0;
|
|
334
|
+
private readonly ranges = new Map<string, [number, number]>();
|
|
335
|
+
|
|
336
|
+
public processOp(
|
|
337
|
+
message: ISequencedDocumentMessage,
|
|
338
|
+
msgSize: number,
|
|
339
|
+
skipMessage: boolean,
|
|
340
|
+
): void {
|
|
341
|
+
if (message.sequenceNumber >= this.opLimit) {
|
|
342
|
+
if (message.sequenceNumber !== 1) {
|
|
343
|
+
const timeDiff = durationFromTime(message.timestamp - this.timeStart);
|
|
344
|
+
const opsString = `ops = [${this.opLimit - this.opChunk}, ${
|
|
345
|
+
this.opLimit - 1
|
|
346
|
+
}]`.padEnd(26);
|
|
347
|
+
const timeString = `time = [${durationFromTime(
|
|
348
|
+
this.timeStart - this.doctimerStart,
|
|
349
|
+
)}, ${durationFromTime(message.timestamp - this.doctimerStart)}]`;
|
|
350
|
+
this.ranges.set(`${opsString} ${timeString}`, [timeDiff, this.size]);
|
|
351
|
+
} else {
|
|
352
|
+
this.doctimerStart = message.timestamp;
|
|
353
|
+
}
|
|
354
|
+
this.opLimit += this.opChunk;
|
|
355
|
+
this.size = 0;
|
|
356
|
+
this.timeStart = message.timestamp;
|
|
357
|
+
}
|
|
358
|
+
if (!skipMessage) {
|
|
359
|
+
this.size += msgSize;
|
|
360
|
+
}
|
|
361
|
+
}
|
|
362
|
+
|
|
363
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
364
|
+
dumpStats(this.ranges, {
|
|
365
|
+
title: "Fastest 1000 op ranges",
|
|
366
|
+
headers: ["Duration(s)", "Bytes"],
|
|
367
|
+
orderByFirstColumn: true,
|
|
368
|
+
reverseSort: true,
|
|
369
|
+
removeTotals: true,
|
|
370
|
+
lines: 3,
|
|
371
|
+
});
|
|
372
|
+
}
|
|
339
373
|
}
|
|
340
374
|
|
|
341
375
|
/**
|
|
342
376
|
* Helper class to analyze collab window size
|
|
343
377
|
*/
|
|
344
378
|
class CollabWindowSizeAnalyzer implements IMessageAnalyzer {
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
|
|
350
|
-
|
|
351
|
-
|
|
352
|
-
|
|
353
|
-
|
|
354
|
-
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
379
|
+
private maxCollabWindow = 0;
|
|
380
|
+
private opSeq = 0;
|
|
381
|
+
|
|
382
|
+
public processOp(
|
|
383
|
+
message: ISequencedDocumentMessage,
|
|
384
|
+
msgSize: number,
|
|
385
|
+
skipMessage: boolean,
|
|
386
|
+
): void {
|
|
387
|
+
const value = message.sequenceNumber - message.minimumSequenceNumber;
|
|
388
|
+
if (value > this.maxCollabWindow) {
|
|
389
|
+
this.maxCollabWindow = value;
|
|
390
|
+
this.opSeq = message.sequenceNumber;
|
|
391
|
+
}
|
|
392
|
+
}
|
|
393
|
+
|
|
394
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
395
|
+
console.log(`\nMaximum collab window size: ${this.maxCollabWindow}, seq# ${this.opSeq}`);
|
|
396
|
+
}
|
|
359
397
|
}
|
|
360
398
|
|
|
361
399
|
/**
|
|
362
400
|
* Helper class to analyze frequency of summaries
|
|
363
401
|
*/
|
|
364
402
|
class SummaryAnalyzer implements IMessageAnalyzer {
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
403
|
+
private lastSummaryOp = 0;
|
|
404
|
+
private maxDistance = 0;
|
|
405
|
+
private maxSeq = 0;
|
|
406
|
+
private minDistance = Number.MAX_SAFE_INTEGER;
|
|
407
|
+
private minSeq = 0;
|
|
408
|
+
private maxResponse = 0;
|
|
409
|
+
private maxResponseSeq = 0;
|
|
410
|
+
|
|
411
|
+
public processOp(
|
|
412
|
+
message: ISequencedDocumentMessage,
|
|
413
|
+
msgSize: number,
|
|
414
|
+
skipMessage: boolean,
|
|
415
|
+
): void {
|
|
416
|
+
if (message.type === MessageType.SummaryAck) {
|
|
417
|
+
const distance = message.sequenceNumber - this.lastSummaryOp - 1;
|
|
418
|
+
if (this.maxDistance < distance) {
|
|
419
|
+
this.maxDistance = distance;
|
|
420
|
+
this.maxSeq = message.sequenceNumber;
|
|
421
|
+
}
|
|
422
|
+
if (this.minDistance > distance) {
|
|
423
|
+
this.minDistance = distance;
|
|
424
|
+
this.minSeq = message.sequenceNumber;
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
this.lastSummaryOp = message.sequenceNumber;
|
|
428
|
+
}
|
|
429
|
+
if (message.type === MessageType.SummaryAck || message.type === MessageType.SummaryNack) {
|
|
430
|
+
const contents: ISummaryProposal = (message.contents as ISummaryAck | ISummaryNack)
|
|
431
|
+
.summaryProposal;
|
|
432
|
+
const distance = message.sequenceNumber - contents.summarySequenceNumber;
|
|
433
|
+
if (distance > this.maxResponse) {
|
|
434
|
+
this.maxResponse = distance;
|
|
435
|
+
this.maxResponseSeq = message.sequenceNumber;
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {
|
|
441
|
+
const distance = lastOp.sequenceNumber - this.lastSummaryOp;
|
|
442
|
+
if (this.maxDistance < distance) {
|
|
443
|
+
this.maxDistance = distance;
|
|
444
|
+
this.maxSeq = lastOp.sequenceNumber + 1;
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
console.log("");
|
|
448
|
+
if (this.minDistance === Number.MAX_SAFE_INTEGER) {
|
|
449
|
+
console.log("No summaries found in this document");
|
|
450
|
+
} else {
|
|
451
|
+
console.log(
|
|
452
|
+
`Maximum distance between summaries: ${this.maxDistance}, seq# ${this.maxSeq}`,
|
|
453
|
+
);
|
|
454
|
+
console.log(
|
|
455
|
+
`Maximum server response for summary: ${this.maxResponse}, seq# ${this.maxResponseSeq}`,
|
|
456
|
+
);
|
|
457
|
+
console.log(
|
|
458
|
+
`Minimum distance between summaries: ${this.minDistance}, seq# ${this.minSeq}`,
|
|
459
|
+
);
|
|
460
|
+
}
|
|
461
|
+
}
|
|
413
462
|
}
|
|
414
463
|
|
|
415
464
|
/**
|
|
416
465
|
* Helper class to dump messages to console
|
|
417
466
|
*/
|
|
418
467
|
class MessageDumper implements IMessageAnalyzer {
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
|
|
468
|
+
public processOp(
|
|
469
|
+
message: ISequencedDocumentMessage,
|
|
470
|
+
msgSize: number,
|
|
471
|
+
skipMessage: boolean,
|
|
472
|
+
): void {
|
|
473
|
+
if (!skipMessage) {
|
|
474
|
+
console.log(JSON.stringify(message, undefined, 2));
|
|
475
|
+
}
|
|
476
|
+
}
|
|
477
|
+
|
|
478
|
+
public reportAnalyzes(lastOp: ISequencedDocumentMessage): void {}
|
|
427
479
|
}
|
|
428
480
|
|
|
429
481
|
export async function printMessageStats(
|
|
430
|
-
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
482
|
+
generator, // AsyncGenerator<ISequencedDocumentMessage[]>,
|
|
483
|
+
dumpMessageStats: boolean,
|
|
484
|
+
dumpMessages: boolean,
|
|
485
|
+
messageTypeFilter: Set<string> = new Set<string>(),
|
|
486
|
+
) {
|
|
487
|
+
let lastMessage: ISequencedDocumentMessage | undefined;
|
|
488
|
+
|
|
489
|
+
const analyzers: IMessageAnalyzer[] = [
|
|
490
|
+
new FilteredMessageAnalyzer(), // Should come first
|
|
491
|
+
new SessionAnalyzer(),
|
|
492
|
+
new DataStructureAnalyzer(),
|
|
493
|
+
new MessageDensityAnalyzer(),
|
|
494
|
+
new CollabWindowSizeAnalyzer(),
|
|
495
|
+
new SummaryAnalyzer(),
|
|
496
|
+
];
|
|
497
|
+
|
|
498
|
+
if (dumpMessages) {
|
|
499
|
+
analyzers.push(new MessageDumper());
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
for await (const messages of generator) {
|
|
503
|
+
for (const message of messages as ISequencedDocumentMessage[]) {
|
|
504
|
+
const msgSize = JSON.stringify(message).length;
|
|
505
|
+
lastMessage = message;
|
|
506
|
+
|
|
507
|
+
const skipMessage =
|
|
508
|
+
messageTypeFilter.size !== 0 && !messageTypeFilter.has(message.type);
|
|
509
|
+
|
|
510
|
+
for (const analyzer of analyzers) {
|
|
511
|
+
analyzer.processOp(message, msgSize, skipMessage);
|
|
512
|
+
}
|
|
513
|
+
}
|
|
514
|
+
}
|
|
515
|
+
|
|
516
|
+
if (lastMessage !== undefined) {
|
|
517
|
+
if (dumpMessageStats) {
|
|
518
|
+
for (const analyzer of analyzers) {
|
|
519
|
+
analyzer.reportAnalyzes(lastMessage);
|
|
520
|
+
}
|
|
521
|
+
} else {
|
|
522
|
+
// Warn about filtered messages
|
|
523
|
+
analyzers[0].reportAnalyzes(lastMessage);
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
console.log("");
|
|
473
527
|
}
|
|
474
528
|
|
|
475
529
|
function processOp(
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
|
|
492
|
-
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
499
|
-
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
594
|
-
|
|
595
|
-
|
|
596
|
-
|
|
597
|
-
|
|
598
|
-
|
|
599
|
-
|
|
600
|
-
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
530
|
+
runtimeMessage: ISequencedDocumentMessage,
|
|
531
|
+
dataType: Map<string, string>,
|
|
532
|
+
objectStats: Map<string, [number, number]>,
|
|
533
|
+
msgSize: number,
|
|
534
|
+
dataTypeStats: Map<string, [number, number]>,
|
|
535
|
+
messageTypeStats: Map<string, [number, number]>,
|
|
536
|
+
chunkMap: Map<string, { chunks: string[]; totalSize: number }>,
|
|
537
|
+
) {
|
|
538
|
+
let type = runtimeMessage.type;
|
|
539
|
+
let recorded = false;
|
|
540
|
+
let totalMsgSize = msgSize;
|
|
541
|
+
let opCount = 1;
|
|
542
|
+
if (unpackRuntimeMessage(runtimeMessage)) {
|
|
543
|
+
const messageType = runtimeMessage.type as ContainerMessageType;
|
|
544
|
+
switch (messageType) {
|
|
545
|
+
case ContainerMessageType.Attach: {
|
|
546
|
+
const attachMessage = runtimeMessage.contents as IAttachMessage;
|
|
547
|
+
processDataStoreAttachOp(attachMessage, dataType);
|
|
548
|
+
break;
|
|
549
|
+
}
|
|
550
|
+
// skip for now because these ops do not have contents
|
|
551
|
+
case ContainerMessageType.BlobAttach: {
|
|
552
|
+
break;
|
|
553
|
+
}
|
|
554
|
+
// The default method to count stats should be used for GC messages.
|
|
555
|
+
case ContainerMessageType.GC: {
|
|
556
|
+
break;
|
|
557
|
+
}
|
|
558
|
+
case ContainerMessageType.ChunkedOp: {
|
|
559
|
+
const chunk = runtimeMessage.contents as IChunkedOp;
|
|
560
|
+
// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
|
|
561
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
562
|
+
if (!chunkMap.has(runtimeMessage.clientId as string)) {
|
|
563
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
564
|
+
chunkMap.set(runtimeMessage.clientId as string, {
|
|
565
|
+
chunks: new Array<string>(chunk.totalChunks),
|
|
566
|
+
totalSize: 0,
|
|
567
|
+
});
|
|
568
|
+
}
|
|
569
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
570
|
+
const value = chunkMap.get(runtimeMessage.clientId as string);
|
|
571
|
+
assert(value !== undefined, 0x2b8 /* "Chunk should be set in map" */);
|
|
572
|
+
const chunks = value.chunks;
|
|
573
|
+
const chunkIndex = chunk.chunkId - 1;
|
|
574
|
+
if (chunks[chunkIndex] !== undefined) {
|
|
575
|
+
throw new Error("Chunk already assigned");
|
|
576
|
+
}
|
|
577
|
+
chunks[chunkIndex] = chunk.contents;
|
|
578
|
+
value.totalSize += msgSize;
|
|
579
|
+
if (chunk.chunkId === chunk.totalChunks) {
|
|
580
|
+
opCount = chunk.totalChunks; // 1 op for each chunk.
|
|
581
|
+
const patchedMessage = Object.create(runtimeMessage);
|
|
582
|
+
patchedMessage.contents = chunks.join("");
|
|
583
|
+
patchedMessage.type = chunk.originalType;
|
|
584
|
+
type = chunk.originalType;
|
|
585
|
+
totalMsgSize = value.totalSize;
|
|
586
|
+
chunkMap.delete(patchedMessage.clientId);
|
|
587
|
+
} else {
|
|
588
|
+
return;
|
|
589
|
+
}
|
|
590
|
+
// eslint-disable-next-line no-fallthrough
|
|
591
|
+
}
|
|
592
|
+
case ContainerMessageType.IdAllocation:
|
|
593
|
+
case ContainerMessageType.FluidDataStoreOp:
|
|
594
|
+
case ContainerMessageType.Alias:
|
|
595
|
+
case ContainerMessageType.Rejoin: {
|
|
596
|
+
let envelope = runtimeMessage.contents as IEnvelope;
|
|
597
|
+
// TODO: Legacy?
|
|
598
|
+
if (envelope !== undefined && typeof envelope === "string") {
|
|
599
|
+
envelope = JSON.parse(envelope);
|
|
600
|
+
}
|
|
601
|
+
const innerContent = envelope.contents as {
|
|
602
|
+
content: any;
|
|
603
|
+
type: string;
|
|
604
|
+
};
|
|
605
|
+
const address = envelope.address;
|
|
606
|
+
type = `${type}/${innerContent.type}`;
|
|
607
|
+
switch (innerContent.type) {
|
|
608
|
+
case DataStoreMessageType.Attach: {
|
|
609
|
+
const attachMessage = innerContent.content as IAttachMessage;
|
|
610
|
+
let objectType = attachMessage.type;
|
|
611
|
+
if (objectType.startsWith(objectTypePrefix)) {
|
|
612
|
+
objectType = objectType.substring(objectTypePrefix.length);
|
|
613
|
+
}
|
|
614
|
+
dataType.set(getObjectId(address, attachMessage.id), objectType);
|
|
615
|
+
break;
|
|
616
|
+
}
|
|
617
|
+
case DataStoreMessageType.ChannelOp:
|
|
618
|
+
default: {
|
|
619
|
+
const innerEnvelope = innerContent.content as IEnvelope;
|
|
620
|
+
const innerContent2 = innerEnvelope.contents as {
|
|
621
|
+
type?: string;
|
|
622
|
+
value?: any;
|
|
623
|
+
};
|
|
624
|
+
|
|
625
|
+
const objectId = getObjectId(address, innerEnvelope.address);
|
|
626
|
+
incr(objectStats, objectId, totalMsgSize, opCount);
|
|
627
|
+
let objectType = dataType.get(objectId);
|
|
628
|
+
if (objectType === undefined) {
|
|
629
|
+
// Somehow we do not have data...
|
|
630
|
+
dataType.set(objectId, objectId);
|
|
631
|
+
objectType = objectId;
|
|
632
|
+
}
|
|
633
|
+
incr(dataTypeStats, objectType, totalMsgSize, opCount);
|
|
634
|
+
recorded = true;
|
|
635
|
+
|
|
636
|
+
let subType = innerContent2.type;
|
|
637
|
+
if (
|
|
638
|
+
innerContent2.type === "set" &&
|
|
639
|
+
typeof innerContent2.value === "object" &&
|
|
640
|
+
innerContent2.value !== null
|
|
641
|
+
) {
|
|
642
|
+
type = `${type}/${subType}`;
|
|
643
|
+
subType = innerContent2.value.type;
|
|
644
|
+
} else if (objectType === "mergeTree" && subType !== undefined) {
|
|
645
|
+
const types = ["insert", "remove", "annotate", "group"];
|
|
646
|
+
if (types[subType] !== undefined) {
|
|
647
|
+
subType = types[subType];
|
|
648
|
+
}
|
|
649
|
+
}
|
|
650
|
+
if (subType !== undefined) {
|
|
651
|
+
type = `${type}/${subType}`;
|
|
652
|
+
}
|
|
653
|
+
|
|
654
|
+
type = `${type} (${objectType})`;
|
|
655
|
+
}
|
|
656
|
+
}
|
|
657
|
+
break;
|
|
658
|
+
}
|
|
659
|
+
default:
|
|
660
|
+
unreachableCase(messageType, "Message type not recognized!");
|
|
661
|
+
}
|
|
662
|
+
}
|
|
663
|
+
|
|
664
|
+
incr(messageTypeStats, type, totalMsgSize, opCount);
|
|
665
|
+
if (!recorded) {
|
|
666
|
+
// const objectId = `${type} (system)`;
|
|
667
|
+
const objectId = `(system messages)`;
|
|
668
|
+
const objectType = objectId;
|
|
669
|
+
if (dataType.get(objectId) === undefined) {
|
|
670
|
+
dataType.set(objectId, objectId);
|
|
671
|
+
}
|
|
672
|
+
incr(objectStats, objectId, totalMsgSize, opCount);
|
|
673
|
+
incr(dataTypeStats, objectType, totalMsgSize, opCount);
|
|
674
|
+
}
|
|
613
675
|
}
|
|
614
676
|
|
|
615
677
|
function processDataStoreAttachOp(
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
}
|
|
640
|
-
}
|
|
641
|
-
}
|
|
678
|
+
attachMessage: IAttachMessage | string,
|
|
679
|
+
dataType: Map<string, string>,
|
|
680
|
+
) {
|
|
681
|
+
// dataType.set(getObjectId(attachMessage.id), attachMessage.type);
|
|
682
|
+
|
|
683
|
+
// That's data store, and it brings a bunch of data structures.
|
|
684
|
+
// Let's try to crack it.
|
|
685
|
+
const parsedAttachMessage =
|
|
686
|
+
typeof attachMessage === "string" ? JSON.parse(attachMessage) : attachMessage;
|
|
687
|
+
for (const entry of parsedAttachMessage.snapshot.entries) {
|
|
688
|
+
if (entry.type === TreeEntry.Tree) {
|
|
689
|
+
for (const entry2 of entry.value.entries) {
|
|
690
|
+
if (entry2.path === ".attributes" && entry2.type === TreeEntry.Blob) {
|
|
691
|
+
const attrib = JSON.parse(entry2.value.contents);
|
|
692
|
+
let objectType: string = attrib.type;
|
|
693
|
+
if (objectType.startsWith(objectTypePrefix)) {
|
|
694
|
+
objectType = objectType.substring(objectTypePrefix.length);
|
|
695
|
+
}
|
|
696
|
+
dataType.set(getObjectId(parsedAttachMessage.id, entry.path), objectType);
|
|
697
|
+
}
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
}
|
|
642
701
|
}
|
|
643
702
|
|
|
644
703
|
function reportOpenSessions(
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
668
|
-
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
704
|
+
lastOpTimestamp: number,
|
|
705
|
+
sessionsInProgress: Map<string, ActiveSession>,
|
|
706
|
+
sessions: Map<string, [number, number]>,
|
|
707
|
+
users: Map<string, [number, number]>,
|
|
708
|
+
) {
|
|
709
|
+
const activeSessions = new Map<string, [number, number]>();
|
|
710
|
+
|
|
711
|
+
for (const [clientId, ses] of sessionsInProgress) {
|
|
712
|
+
const sessionInfo = ses.leave(lastOpTimestamp);
|
|
713
|
+
if (clientId !== noClientName) {
|
|
714
|
+
const sessionName = `${clientId} (${sessionInfo.email})`;
|
|
715
|
+
const sessionPayload: [number, number] = [
|
|
716
|
+
durationFromTime(sessionInfo.duration),
|
|
717
|
+
sessionInfo.opCount,
|
|
718
|
+
];
|
|
719
|
+
sessions.set(sessionName, sessionPayload);
|
|
720
|
+
activeSessions.set(sessionName, sessionPayload);
|
|
721
|
+
} else {
|
|
722
|
+
sessions.set(`Full file lifespan (noClient messages)`, [
|
|
723
|
+
durationFromTime(sessionInfo.duration),
|
|
724
|
+
sessionInfo.opCount,
|
|
725
|
+
]);
|
|
726
|
+
}
|
|
727
|
+
incr(users, sessionInfo.email, sessionInfo.opCount);
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
if (activeSessions.size > 0) {
|
|
731
|
+
dumpStats(activeSessions, {
|
|
732
|
+
title: "Active sessions",
|
|
733
|
+
headers: ["Duration", "Op count"],
|
|
734
|
+
lines: 6,
|
|
735
|
+
orderByFirstColumn: true,
|
|
736
|
+
removeTotals: true,
|
|
737
|
+
});
|
|
738
|
+
}
|
|
675
739
|
}
|
|
676
740
|
|
|
677
|
-
function calcChannelStats(
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
741
|
+
function calcChannelStats(
|
|
742
|
+
dataType: Map<string, string>,
|
|
743
|
+
objectStats: Map<string, [number, number]>,
|
|
744
|
+
) {
|
|
745
|
+
const channelStats = new Map<string, [number, number]>();
|
|
746
|
+
for (const [objectId, type] of dataType) {
|
|
747
|
+
let value = objectStats.get(objectId);
|
|
748
|
+
if (value === undefined) {
|
|
749
|
+
value = [0, 0];
|
|
750
|
+
}
|
|
751
|
+
if (type === objectId) {
|
|
752
|
+
channelStats.set(`${objectId}`, value);
|
|
753
|
+
} else {
|
|
754
|
+
channelStats.set(`${objectId} (${type})`, value);
|
|
755
|
+
}
|
|
756
|
+
}
|
|
757
|
+
return channelStats;
|
|
691
758
|
}
|
|
692
759
|
|
|
693
760
|
function processQuorumMessages(
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
761
|
+
message: ISequencedDocumentMessage,
|
|
762
|
+
skipMessage: boolean,
|
|
763
|
+
sessionsInProgress: Map<string, ActiveSession>,
|
|
764
|
+
sessions: Map<string, [number, number]>,
|
|
765
|
+
users: Map<string, [number, number]>,
|
|
766
|
+
) {
|
|
767
|
+
let session: ActiveSession | undefined;
|
|
768
|
+
const dataString = (message as any).data;
|
|
769
|
+
if (message.type === "join") {
|
|
770
|
+
const data = JSON.parse(dataString);
|
|
771
|
+
session = ActiveSession.create(data.detail.user.id, message);
|
|
772
|
+
sessionsInProgress.set(data.clientId, session);
|
|
773
|
+
} else if (message.type === "leave") {
|
|
774
|
+
const clientId = JSON.parse(dataString);
|
|
775
|
+
session = sessionsInProgress.get(clientId);
|
|
776
|
+
sessionsInProgress.delete(clientId);
|
|
777
|
+
assert(!!session, 0x1b7 /* "Bad session state for processing quorum messages" */);
|
|
778
|
+
if (session !== undefined) {
|
|
779
|
+
if (!skipMessage) {
|
|
780
|
+
session.reportOp(message.timestamp);
|
|
781
|
+
}
|
|
782
|
+
const sessionInfo: ISessionInfo = session.leave(message.timestamp);
|
|
783
|
+
sessions.set(`${clientId} (${sessionInfo.email})`, [
|
|
784
|
+
durationFromTime(sessionInfo.duration),
|
|
785
|
+
sessionInfo.opCount,
|
|
786
|
+
]);
|
|
787
|
+
incr(users, sessionInfo.email, sessionInfo.opCount);
|
|
788
|
+
session = undefined; // Do not record it second time
|
|
789
|
+
}
|
|
790
|
+
} else {
|
|
791
|
+
// message.clientId can be null
|
|
792
|
+
// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
|
|
793
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
794
|
+
session = sessionsInProgress.get(message.clientId as string);
|
|
795
|
+
if (session === undefined) {
|
|
796
|
+
session = sessionsInProgress.get(noClientName);
|
|
797
|
+
assert(!!session, 0x1b8 /* "Bad session state for processing quorum messages" */);
|
|
798
|
+
}
|
|
799
|
+
}
|
|
800
|
+
return session;
|
|
730
801
|
}
|
|
731
802
|
|
|
732
803
|
const durationFromTime = (time: number): number => Math.floor(time / 1000);
|