@fluidframework/container-runtime 2.0.0-internal.4.2.1 → 2.0.0-internal.4.3.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +1 -0
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +9 -6
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStores.d.ts +5 -5
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +3 -6
- package/dist/dataStores.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +1 -27
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +30 -227
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +0 -10
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +1 -20
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +91 -0
- package/dist/gc/gcTelemetry.d.ts.map +1 -0
- package/dist/gc/gcTelemetry.js +262 -0
- package/dist/gc/gcTelemetry.js.map +1 -0
- package/dist/gc/index.d.ts +2 -1
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +4 -2
- package/dist/gc/index.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +1 -1
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/outbox.js +1 -1
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +25 -22
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +1 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts.map +1 -1
- package/dist/summary/orderedClientElection.js +14 -17
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/summarizer.d.ts +2 -0
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +9 -4
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerHeuristics.d.ts +8 -9
- package/dist/summary/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summary/summarizerHeuristics.js +15 -16
- package/dist/summary/summarizerHeuristics.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +2 -0
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryGenerator.d.ts.map +1 -1
- package/dist/summary/summaryGenerator.js +4 -3
- package/dist/summary/summaryGenerator.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +1 -0
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +9 -6
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStores.d.ts +5 -5
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +3 -6
- package/lib/dataStores.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +1 -27
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +34 -231
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +0 -10
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +0 -18
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +91 -0
- package/lib/gc/gcTelemetry.d.ts.map +1 -0
- package/lib/gc/gcTelemetry.js +257 -0
- package/lib/gc/gcTelemetry.js.map +1 -0
- package/lib/gc/index.d.ts +2 -1
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +1 -1
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/outbox.js +1 -1
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +25 -22
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +1 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts.map +1 -1
- package/lib/summary/orderedClientElection.js +14 -17
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/summarizer.d.ts +2 -0
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +9 -4
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerHeuristics.d.ts +8 -9
- package/lib/summary/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summary/summarizerHeuristics.js +15 -16
- package/lib/summary/summarizerHeuristics.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +2 -0
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryGenerator.d.ts.map +1 -1
- package/lib/summary/summaryGenerator.js +4 -3
- package/lib/summary/summaryGenerator.js.map +1 -1
- package/package.json +17 -17
- package/src/connectionTelemetry.ts +1 -0
- package/src/containerRuntime.ts +12 -11
- package/src/dataStores.ts +4 -7
- package/src/gc/garbageCollection.ts +53 -315
- package/src/gc/gcHelpers.ts +1 -38
- package/src/gc/gcTelemetry.ts +393 -0
- package/src/gc/index.ts +1 -1
- package/src/opLifecycle/README.md +13 -0
- package/src/opLifecycle/opGroupingManager.ts +1 -1
- package/src/opLifecycle/outbox.ts +2 -2
- package/src/opLifecycle/remoteMessageProcessor.ts +37 -28
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +1 -4
- package/src/summary/orderedClientElection.ts +14 -17
- package/src/summary/summarizer.ts +17 -5
- package/src/summary/summarizerHeuristics.ts +15 -16
- package/src/summary/summarizerTypes.ts +2 -0
- package/src/summary/summaryGenerator.ts +5 -4
|
@@ -0,0 +1,393 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { ITelemetryGenericEvent, ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
|
+
import { IGarbageCollectionData } from "@fluidframework/runtime-definitions";
|
|
8
|
+
import { packagePathToTelemetryProperty } from "@fluidframework/runtime-utils";
|
|
9
|
+
import {
|
|
10
|
+
generateStack,
|
|
11
|
+
MonitoringContext,
|
|
12
|
+
TelemetryDataTag,
|
|
13
|
+
} from "@fluidframework/telemetry-utils";
|
|
14
|
+
import { ICreateContainerMetadata } from "../summary";
|
|
15
|
+
import {
|
|
16
|
+
disableSweepLogKey,
|
|
17
|
+
GCNodeType,
|
|
18
|
+
UnreferencedState,
|
|
19
|
+
IGarbageCollectorConfigs,
|
|
20
|
+
disableTombstoneKey,
|
|
21
|
+
throwOnTombstoneUsageKey,
|
|
22
|
+
throwOnTombstoneLoadKey,
|
|
23
|
+
runSweepKey,
|
|
24
|
+
} from "./gcDefinitions";
|
|
25
|
+
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
26
|
+
|
|
27
|
+
type NodeUsageType = "Changed" | "Loaded" | "Revived";
|
|
28
|
+
|
|
29
|
+
/** Properties that are common to IUnreferencedEventProps and INodeUsageProps */
|
|
30
|
+
interface ICommonProps {
|
|
31
|
+
usageType: NodeUsageType;
|
|
32
|
+
completedGCRuns: number;
|
|
33
|
+
isTombstoned: boolean;
|
|
34
|
+
lastSummaryTime?: number;
|
|
35
|
+
fromId?: string;
|
|
36
|
+
viaHandle?: boolean;
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/** The event that is logged when unreferenced node is used after a certain time. */
|
|
40
|
+
interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
|
|
41
|
+
state: UnreferencedState;
|
|
42
|
+
id: string;
|
|
43
|
+
type: GCNodeType;
|
|
44
|
+
unrefTime: number;
|
|
45
|
+
age: number;
|
|
46
|
+
timeout?: number;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/** Properties passed to nodeUsed function when a node is used. */
|
|
50
|
+
interface INodeUsageProps extends ICommonProps {
|
|
51
|
+
nodeId: string;
|
|
52
|
+
currentReferenceTimestampMs: number | undefined;
|
|
53
|
+
packagePath: readonly string[] | undefined;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Encapsulates the logic that tracks the various telemetry logged by the Garbage Collector. There are 4 types of
|
|
58
|
+
* telemetry logged:
|
|
59
|
+
* 1. inactiveObject telemetry - When an inactive node is used - A node that has been unreferenced for inactiveTimeoutMs.
|
|
60
|
+
* 2. sweepReadyObject telemetry - When a sweep ready node is used - A node that has been unreferenced for sweepTimeoutMs.
|
|
61
|
+
* 3. Tombstone telemetry - When a tombstoned node is used - A node that that has been marked as tombstone.
|
|
62
|
+
* 4. Sweep / deleted telemetry - When a node is detected as sweep ready in the sweep phase.
|
|
63
|
+
* 5. Unknown outbound reference telemetry - When a node is referenced but GC is not explicitly notified of it.
|
|
64
|
+
*/
|
|
65
|
+
export class GCTelemetryTracker {
|
|
66
|
+
// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
|
|
67
|
+
// per event per node.
|
|
68
|
+
private readonly loggedUnreferencedEvents: Set<string> = new Set();
|
|
69
|
+
// Queue for unreferenced events that should be logged the next time GC runs.
|
|
70
|
+
private pendingEventsQueue: IUnreferencedEventProps[] = [];
|
|
71
|
+
|
|
72
|
+
constructor(
|
|
73
|
+
private readonly mc: MonitoringContext,
|
|
74
|
+
private readonly configs: Pick<
|
|
75
|
+
IGarbageCollectorConfigs,
|
|
76
|
+
"inactiveTimeoutMs" | "sweepTimeoutMs"
|
|
77
|
+
>,
|
|
78
|
+
private readonly isSummarizerClient: boolean,
|
|
79
|
+
private readonly gcTombstoneEnforcementAllowed: boolean,
|
|
80
|
+
private readonly createContainerMetadata: ICreateContainerMetadata,
|
|
81
|
+
private readonly getNodeType: (nodeId: string) => GCNodeType,
|
|
82
|
+
private readonly getNodeStateTracker: (
|
|
83
|
+
nodeId: string,
|
|
84
|
+
) => UnreferencedStateTracker | undefined,
|
|
85
|
+
private readonly getNodePackagePath: (
|
|
86
|
+
nodePath: string,
|
|
87
|
+
) => Promise<readonly string[] | undefined>,
|
|
88
|
+
) {}
|
|
89
|
+
|
|
90
|
+
/**
|
|
91
|
+
* Returns whether an event should be logged for a node that isn't active anymore. Some scenarios where we won't log:
|
|
92
|
+
* 1. When a DDS is changed or loaded. The corresponding data store's event will be logged instead.
|
|
93
|
+
* 2. An event is logged only once per container instance per event per node.
|
|
94
|
+
*/
|
|
95
|
+
private shouldLogNonActiveEvent(
|
|
96
|
+
nodeId: string,
|
|
97
|
+
nodeType: GCNodeType,
|
|
98
|
+
usageType: NodeUsageType,
|
|
99
|
+
nodeStateTracker: UnreferencedStateTracker,
|
|
100
|
+
uniqueEventId: string,
|
|
101
|
+
) {
|
|
102
|
+
if (nodeStateTracker.state === UnreferencedState.Active) {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// For sub data store (DDS) nodes, if they are changed or loaded, its data store will also be changed or loaded,
|
|
107
|
+
// so skip logging to make the telemetry less noisy.
|
|
108
|
+
if (nodeType === GCNodeType.SubDataStore && usageType !== "Revived") {
|
|
109
|
+
return false;
|
|
110
|
+
}
|
|
111
|
+
if (nodeType === GCNodeType.Other) {
|
|
112
|
+
return false;
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
116
|
+
return false;
|
|
117
|
+
}
|
|
118
|
+
return true;
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
/**
|
|
122
|
+
* Called when a node is used. If the node is not active, log an event indicating object is used when its not active.
|
|
123
|
+
*/
|
|
124
|
+
public nodeUsed(nodeUsageProps: INodeUsageProps) {
|
|
125
|
+
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
126
|
+
// logging as nothing interesting would have happened worth logging.
|
|
127
|
+
// If the node is not unreferenced, skip logging.
|
|
128
|
+
const nodeStateTracker = this.getNodeStateTracker(nodeUsageProps.nodeId);
|
|
129
|
+
if (!nodeStateTracker || nodeUsageProps.currentReferenceTimestampMs === undefined) {
|
|
130
|
+
return;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
// We log these events once per event per node. A unique id is generated by joining node state (inactive / sweep ready),
|
|
134
|
+
// node's id and usage (loaded / changed / revived).
|
|
135
|
+
const uniqueEventId = `${nodeStateTracker.state}-${nodeUsageProps.nodeId}-${nodeUsageProps.usageType}`;
|
|
136
|
+
const nodeType = this.getNodeType(nodeUsageProps.nodeId);
|
|
137
|
+
if (
|
|
138
|
+
!this.shouldLogNonActiveEvent(
|
|
139
|
+
nodeUsageProps.nodeId,
|
|
140
|
+
nodeType,
|
|
141
|
+
nodeUsageProps.usageType,
|
|
142
|
+
nodeStateTracker,
|
|
143
|
+
uniqueEventId,
|
|
144
|
+
)
|
|
145
|
+
) {
|
|
146
|
+
return;
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
// Add the unique event id so that we don't generate a log for this event again in this session..
|
|
150
|
+
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
151
|
+
|
|
152
|
+
const state = nodeStateTracker.state;
|
|
153
|
+
const { usageType, currentReferenceTimestampMs, packagePath, ...propsToLog } =
|
|
154
|
+
nodeUsageProps;
|
|
155
|
+
const eventProps: Omit<IUnreferencedEventProps, "state" | "usageType"> = {
|
|
156
|
+
id: nodeUsageProps.nodeId,
|
|
157
|
+
type: nodeType,
|
|
158
|
+
unrefTime: nodeStateTracker.unreferencedTimestampMs,
|
|
159
|
+
age:
|
|
160
|
+
nodeUsageProps.currentReferenceTimestampMs -
|
|
161
|
+
nodeStateTracker.unreferencedTimestampMs,
|
|
162
|
+
timeout:
|
|
163
|
+
state === UnreferencedState.Inactive
|
|
164
|
+
? this.configs.inactiveTimeoutMs
|
|
165
|
+
: this.configs.sweepTimeoutMs,
|
|
166
|
+
...propsToLog,
|
|
167
|
+
...this.createContainerMetadata,
|
|
168
|
+
};
|
|
169
|
+
|
|
170
|
+
// This will log the following events:
|
|
171
|
+
// GC_Tombstone_DataStore_Revived, GC_Tombstone_SubDataStore_Revived, GC_Tombstone_Blob_Revived
|
|
172
|
+
if (nodeUsageProps.usageType === "Revived" && nodeUsageProps.isTombstoned) {
|
|
173
|
+
sendGCUnexpectedUsageEvent(
|
|
174
|
+
this.mc,
|
|
175
|
+
{
|
|
176
|
+
eventName: `GC_Tombstone_${nodeType}_Revived`,
|
|
177
|
+
category: "generic",
|
|
178
|
+
url: nodeUsageProps.nodeId,
|
|
179
|
+
gcTombstoneEnforcementAllowed: this.gcTombstoneEnforcementAllowed,
|
|
180
|
+
},
|
|
181
|
+
undefined /* packagePath */,
|
|
182
|
+
);
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
186
|
+
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
187
|
+
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|
|
188
|
+
// Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
|
|
189
|
+
// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
|
|
190
|
+
if (this.isSummarizerClient) {
|
|
191
|
+
this.pendingEventsQueue.push({
|
|
192
|
+
...eventProps,
|
|
193
|
+
usageType: nodeUsageProps.usageType,
|
|
194
|
+
state,
|
|
195
|
+
});
|
|
196
|
+
} else {
|
|
197
|
+
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
198
|
+
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
199
|
+
// Events generated:
|
|
200
|
+
// InactiveObject_Loaded, SweepReadyObject_Loaded
|
|
201
|
+
if (nodeUsageProps.usageType === "Loaded") {
|
|
202
|
+
const event = {
|
|
203
|
+
eventName: `${state}Object_${nodeUsageProps.usageType}`,
|
|
204
|
+
pkg: packagePathToTelemetryProperty(nodeUsageProps.packagePath),
|
|
205
|
+
stack: generateStack(),
|
|
206
|
+
...eventProps,
|
|
207
|
+
};
|
|
208
|
+
|
|
209
|
+
// Do not log the inactive object x events as error events as they are not the best signal for
|
|
210
|
+
// detecting something wrong with GC either from the partner or from the runtime itself.
|
|
211
|
+
if (state === UnreferencedState.Inactive) {
|
|
212
|
+
this.mc.logger.sendTelemetryEvent(event);
|
|
213
|
+
} else {
|
|
214
|
+
this.mc.logger.sendErrorEvent(event);
|
|
215
|
+
}
|
|
216
|
+
}
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
|
|
220
|
+
/**
|
|
221
|
+
* Log all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
|
|
222
|
+
* The principle is that every new reference or outbound route must be notified to GC via the
|
|
223
|
+
* addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
|
|
224
|
+
*
|
|
225
|
+
* In more simple terms:
|
|
226
|
+
* Missing Explicit References = Current References - Previous References - Explicitly Added References;
|
|
227
|
+
*
|
|
228
|
+
* @param currentGCData - The GC data (reference graph) from the current GC run.
|
|
229
|
+
* @param previousGCData - The GC data (reference graph) from the previous GC run.
|
|
230
|
+
* @param explicitReferences - New references added explicity between the previous and the current run.
|
|
231
|
+
*/
|
|
232
|
+
public logIfMissingExplicitReferences(
|
|
233
|
+
currentGCData: IGarbageCollectionData,
|
|
234
|
+
previousGCData: IGarbageCollectionData,
|
|
235
|
+
explicitReferences: Map<string, string[]>,
|
|
236
|
+
logger: ITelemetryLogger,
|
|
237
|
+
) {
|
|
238
|
+
for (const [nodeId, currentOutboundRoutes] of Object.entries(currentGCData.gcNodes)) {
|
|
239
|
+
const previousRoutes = previousGCData.gcNodes[nodeId] ?? [];
|
|
240
|
+
const explicitRoutes = explicitReferences.get(nodeId) ?? [];
|
|
241
|
+
|
|
242
|
+
/**
|
|
243
|
+
* 1. For routes in the current GC data, routes that were not present in previous GC data and did not have
|
|
244
|
+
* explicit references should be added to missing explicit routes list.
|
|
245
|
+
* 2. Only include data store and blob routes since GC only works for these two.
|
|
246
|
+
* Note: Due to a bug with de-duped blobs, only adding data store routes for now.
|
|
247
|
+
* 3. Ignore DDS routes to their parent datastores since those were added implicitly. So, there won't be
|
|
248
|
+
* explicit routes to them.
|
|
249
|
+
*/
|
|
250
|
+
const missingExplicitRoutes: string[] = [];
|
|
251
|
+
for (const route of currentOutboundRoutes) {
|
|
252
|
+
const nodeType = this.getNodeType(route);
|
|
253
|
+
if (
|
|
254
|
+
(nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
|
|
255
|
+
!nodeId.startsWith(route) &&
|
|
256
|
+
!previousRoutes.includes(route) &&
|
|
257
|
+
!explicitRoutes.includes(route)
|
|
258
|
+
) {
|
|
259
|
+
missingExplicitRoutes.push(route);
|
|
260
|
+
}
|
|
261
|
+
}
|
|
262
|
+
|
|
263
|
+
if (missingExplicitRoutes.length > 0) {
|
|
264
|
+
logger.sendErrorEvent({
|
|
265
|
+
eventName: "gcUnknownOutboundReferences",
|
|
266
|
+
gcNodeId: nodeId,
|
|
267
|
+
gcRoutes: JSON.stringify(missingExplicitRoutes),
|
|
268
|
+
});
|
|
269
|
+
}
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
/**
|
|
274
|
+
* Log events that are pending in pendingEventsQueue. This is called after GC runs in the summarizer client
|
|
275
|
+
* so that the state of an unreferenced node is updated.
|
|
276
|
+
*/
|
|
277
|
+
public async logPendingEvents(logger: ITelemetryLogger) {
|
|
278
|
+
// Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
|
|
279
|
+
// summary time they are then logged.
|
|
280
|
+
// Events generated:
|
|
281
|
+
// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
|
|
282
|
+
// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
|
|
283
|
+
for (const eventProps of this.pendingEventsQueue) {
|
|
284
|
+
const { usageType, state, ...propsToLog } = eventProps;
|
|
285
|
+
/**
|
|
286
|
+
* Revived event is logged only if the node is active. If the node is not active, the reference to it was
|
|
287
|
+
* from another unreferenced node and this scenario is not interesting to log.
|
|
288
|
+
* Loaded and Changed events are logged only if the node is not active. If the node is active, it was
|
|
289
|
+
* revived and a Revived event will be logged for it.
|
|
290
|
+
*/
|
|
291
|
+
const nodeStateTracker = this.getNodeStateTracker(eventProps.id);
|
|
292
|
+
const active =
|
|
293
|
+
nodeStateTracker === undefined ||
|
|
294
|
+
nodeStateTracker.state === UnreferencedState.Active;
|
|
295
|
+
if ((usageType === "Revived") === active) {
|
|
296
|
+
const pkg = await this.getNodePackagePath(eventProps.id);
|
|
297
|
+
const fromPkg = eventProps.fromId
|
|
298
|
+
? await this.getNodePackagePath(eventProps.fromId)
|
|
299
|
+
: undefined;
|
|
300
|
+
const event = {
|
|
301
|
+
...propsToLog,
|
|
302
|
+
eventName: `${state}Object_${usageType}`,
|
|
303
|
+
pkg: pkg
|
|
304
|
+
? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
|
|
305
|
+
: undefined,
|
|
306
|
+
fromPkg: fromPkg
|
|
307
|
+
? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
|
|
308
|
+
: undefined,
|
|
309
|
+
};
|
|
310
|
+
|
|
311
|
+
if (state === UnreferencedState.Inactive) {
|
|
312
|
+
logger.sendTelemetryEvent(event);
|
|
313
|
+
} else {
|
|
314
|
+
logger.sendErrorEvent(event);
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
}
|
|
318
|
+
this.pendingEventsQueue = [];
|
|
319
|
+
}
|
|
320
|
+
|
|
321
|
+
/**
|
|
322
|
+
* For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
|
|
323
|
+
* this will give us a view into how much deleted content a container has.
|
|
324
|
+
*/
|
|
325
|
+
public logSweepEvents(
|
|
326
|
+
logger: ITelemetryLogger,
|
|
327
|
+
currentReferenceTimestampMs: number,
|
|
328
|
+
unreferencedNodesState: Map<string, UnreferencedStateTracker>,
|
|
329
|
+
completedGCRuns: number,
|
|
330
|
+
lastSummaryTime?: number,
|
|
331
|
+
) {
|
|
332
|
+
if (
|
|
333
|
+
this.mc.config.getBoolean(disableSweepLogKey) === true ||
|
|
334
|
+
this.configs.sweepTimeoutMs === undefined
|
|
335
|
+
) {
|
|
336
|
+
return;
|
|
337
|
+
}
|
|
338
|
+
|
|
339
|
+
for (const [nodeId, nodeStateTracker] of unreferencedNodesState) {
|
|
340
|
+
if (nodeStateTracker.state !== UnreferencedState.SweepReady) {
|
|
341
|
+
return;
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
const nodeType = this.getNodeType(nodeId);
|
|
345
|
+
if (nodeType !== GCNodeType.DataStore && nodeType !== GCNodeType.Blob) {
|
|
346
|
+
return;
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
// Log deleted event for each node only once to reduce noise in telemetry.
|
|
350
|
+
const uniqueEventId = `Deleted-${nodeId}`;
|
|
351
|
+
if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
355
|
+
logger.sendTelemetryEvent({
|
|
356
|
+
eventName: "GCObjectDeleted",
|
|
357
|
+
id: nodeId,
|
|
358
|
+
type: nodeType,
|
|
359
|
+
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
360
|
+
timeout: this.configs.sweepTimeoutMs,
|
|
361
|
+
completedGCRuns,
|
|
362
|
+
lastSummaryTime,
|
|
363
|
+
...this.createContainerMetadata,
|
|
364
|
+
});
|
|
365
|
+
}
|
|
366
|
+
}
|
|
367
|
+
}
|
|
368
|
+
|
|
369
|
+
/**
|
|
370
|
+
* Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
|
|
371
|
+
* tombstoned or deleted object is loaded.
|
|
372
|
+
*/
|
|
373
|
+
export function sendGCUnexpectedUsageEvent(
|
|
374
|
+
mc: MonitoringContext,
|
|
375
|
+
event: ITelemetryGenericEvent & {
|
|
376
|
+
category: "error" | "generic";
|
|
377
|
+
gcTombstoneEnforcementAllowed: boolean | undefined;
|
|
378
|
+
},
|
|
379
|
+
packagePath: readonly string[] | undefined,
|
|
380
|
+
error?: unknown,
|
|
381
|
+
) {
|
|
382
|
+
event.pkg = packagePathToTelemetryProperty(packagePath);
|
|
383
|
+
event.tombstoneFlags = JSON.stringify({
|
|
384
|
+
DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
|
|
385
|
+
ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
|
|
386
|
+
ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadKey),
|
|
387
|
+
});
|
|
388
|
+
event.sweepFlags = JSON.stringify({
|
|
389
|
+
EnableSweepFlag: mc.config.getBoolean(runSweepKey),
|
|
390
|
+
});
|
|
391
|
+
|
|
392
|
+
mc.logger.sendTelemetryEvent(event, error);
|
|
393
|
+
}
|
package/src/gc/index.ts
CHANGED
|
@@ -39,7 +39,6 @@ export {
|
|
|
39
39
|
cloneGCData,
|
|
40
40
|
concatGarbageCollectionStates,
|
|
41
41
|
getGCDataFromSnapshot,
|
|
42
|
-
sendGCUnexpectedUsageEvent,
|
|
43
42
|
shouldAllowGcTombstoneEnforcement,
|
|
44
43
|
shouldAllowGcSweep,
|
|
45
44
|
trimLeadingAndTrailingSlashes,
|
|
@@ -62,4 +61,5 @@ export {
|
|
|
62
61
|
closuresMapLocalStorageKey,
|
|
63
62
|
SweepReadyUsageDetectionHandler,
|
|
64
63
|
} from "./gcSweepReadyUsageDetection";
|
|
64
|
+
export { GCTelemetryTracker, sendGCUnexpectedUsageEvent } from "./gcTelemetry";
|
|
65
65
|
export { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
@@ -13,6 +13,7 @@ By default, the runtime is configured with a max batch size of `716800` bytes, w
|
|
|
13
13
|
- [Introduction](#introduction)
|
|
14
14
|
- [Compression](#compression)
|
|
15
15
|
- [Grouped batching](#grouped-batching)
|
|
16
|
+
- [Risks](#risks)
|
|
16
17
|
- [Chunking for compression](#chunking-for-compression)
|
|
17
18
|
- [Disabling in case of emergency](#disabling-in-case-of-emergency)
|
|
18
19
|
- [Example configs](#example-configs)
|
|
@@ -38,6 +39,18 @@ The purpose for enabling grouped batching on top of compression is that regular
|
|
|
38
39
|
|
|
39
40
|
See [below](#how-grouped-batching-works) for an example.
|
|
40
41
|
|
|
42
|
+
### Risks
|
|
43
|
+
|
|
44
|
+
This option is experimental and should not be enabled yet in production. This option should **ONLY** be enabled after observing that 99.9% of your application sessions contains these changes (runtime version "2.0.0-internal.4.1.0" or later). Containers created with this option may not open in future versions of the framework.
|
|
45
|
+
|
|
46
|
+
This option will change a couple of expectations around message structure and runtime layer expectations. Only enable this option after testing
|
|
47
|
+
and verifying that the following expectation changes won't have any effects:
|
|
48
|
+
|
|
49
|
+
- batch messages observed at the runtime layer will not match messages seen at the loader layer (i.e. grouped form at loader layer, ungrouped form at runtime layer)
|
|
50
|
+
- messages within the same batch will have the same sequence number
|
|
51
|
+
- client sequence numbers on batch messages can only be used to order messages with the same sequenceNumber
|
|
52
|
+
- requires all ops to be processed by runtime layer (version "2.0.0-internal.1.2.0" or later https://github.com/microsoft/FluidFramework/pull/11832)
|
|
53
|
+
|
|
41
54
|
## Chunking for compression
|
|
42
55
|
|
|
43
56
|
**Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.
|
|
@@ -39,7 +39,6 @@ export class OpGroupingManager {
|
|
|
39
39
|
}
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
// Need deserializedContent for back-compat
|
|
43
42
|
const deserializedContent = {
|
|
44
43
|
type: OpGroupingManager.groupedBatchOp,
|
|
45
44
|
contents: batch.content.map<IGroupedMessage>((message) => ({
|
|
@@ -56,6 +55,7 @@ export class OpGroupingManager {
|
|
|
56
55
|
localOpMetadata: undefined,
|
|
57
56
|
metadata: undefined,
|
|
58
57
|
referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
|
|
58
|
+
// Need deserializedContent for back-compat
|
|
59
59
|
deserializedContent: deserializedContent as ContainerRuntimeMessage,
|
|
60
60
|
contents: JSON.stringify(deserializedContent),
|
|
61
61
|
},
|
|
@@ -219,8 +219,8 @@ export class Outbox {
|
|
|
219
219
|
return this.params.groupingManager.groupBatch(batch);
|
|
220
220
|
}
|
|
221
221
|
|
|
222
|
-
const compressedBatch = this.params.
|
|
223
|
-
this.params.
|
|
222
|
+
const compressedBatch = this.params.compressor.compressBatch(
|
|
223
|
+
this.params.groupingManager.groupBatch(batch),
|
|
224
224
|
);
|
|
225
225
|
|
|
226
226
|
if (this.params.splitter.isBatchChunkingEnabled) {
|
|
@@ -27,40 +27,49 @@ export class RemoteMessageProcessor {
|
|
|
27
27
|
public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage[] {
|
|
28
28
|
const result: ISequencedDocumentMessage[] = [];
|
|
29
29
|
|
|
30
|
-
// Ungroup before
|
|
31
|
-
for (
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
// Ungroup the chunked message before decompressing
|
|
45
|
-
for (let ungroupedMessageAfterChunking of this.opGroupingManager.ungroupOp(
|
|
46
|
-
ungroupedMessage,
|
|
47
|
-
)) {
|
|
48
|
-
const decompressionAfterChunking = this.opDecompressor.processMessage(
|
|
49
|
-
ungroupedMessageAfterChunking,
|
|
50
|
-
);
|
|
51
|
-
ungroupedMessageAfterChunking = decompressionAfterChunking.message;
|
|
52
|
-
if (decompressionAfterChunking.state === "Skipped") {
|
|
53
|
-
// After chunking, if the original message was not compressed,
|
|
30
|
+
// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
|
|
31
|
+
for (const ungroupedMessage of this.opGroupingManager.ungroupOp(copy(remoteMessage))) {
|
|
32
|
+
const message = this.opDecompressor.processMessage(ungroupedMessage).message;
|
|
33
|
+
|
|
34
|
+
for (let ungroupedMessage2 of this.opGroupingManager.ungroupOp(message)) {
|
|
35
|
+
unpackRuntimeMessage(ungroupedMessage2);
|
|
36
|
+
|
|
37
|
+
const chunkProcessingResult =
|
|
38
|
+
this.opSplitter.processRemoteMessage(ungroupedMessage2);
|
|
39
|
+
ungroupedMessage2 = chunkProcessingResult.message;
|
|
40
|
+
if (chunkProcessingResult.state !== "Processed") {
|
|
41
|
+
// If the message is not chunked or if the splitter is still rebuilding the original message,
|
|
54
42
|
// there is no need to continue processing
|
|
55
|
-
result.push(
|
|
43
|
+
result.push(ungroupedMessage2);
|
|
56
44
|
continue;
|
|
57
45
|
}
|
|
58
46
|
|
|
59
|
-
//
|
|
60
|
-
|
|
61
|
-
|
|
47
|
+
// Ungroup before and after decompression for back-compat (cleanup tracked by AB#4371)
|
|
48
|
+
for (const ungroupedMessageAfterChunking of this.opGroupingManager.ungroupOp(
|
|
49
|
+
ungroupedMessage2,
|
|
50
|
+
)) {
|
|
51
|
+
const decompressionAfterChunking = this.opDecompressor.processMessage(
|
|
52
|
+
ungroupedMessageAfterChunking,
|
|
53
|
+
);
|
|
54
|
+
|
|
55
|
+
for (const ungroupedMessageAfterChunking2 of this.opGroupingManager.ungroupOp(
|
|
56
|
+
decompressionAfterChunking.message,
|
|
57
|
+
)) {
|
|
58
|
+
if (decompressionAfterChunking.state === "Skipped") {
|
|
59
|
+
// After chunking, if the original message was not compressed,
|
|
60
|
+
// there is no need to continue processing
|
|
61
|
+
result.push(ungroupedMessageAfterChunking2);
|
|
62
|
+
continue;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// The message needs to be unpacked after chunking + decompression
|
|
66
|
+
unpack(ungroupedMessageAfterChunking2);
|
|
67
|
+
result.push(ungroupedMessageAfterChunking2);
|
|
68
|
+
}
|
|
69
|
+
}
|
|
62
70
|
}
|
|
63
71
|
}
|
|
72
|
+
|
|
64
73
|
return result;
|
|
65
74
|
}
|
|
66
75
|
}
|
package/src/packageVersion.ts
CHANGED
|
@@ -49,10 +49,7 @@ export interface IRuntimeStateHandler {
|
|
|
49
49
|
connected(): boolean;
|
|
50
50
|
clientId(): string | undefined;
|
|
51
51
|
close(error?: ICriticalContainerError): void;
|
|
52
|
-
applyStashedOp: (
|
|
53
|
-
type: ContainerMessageType,
|
|
54
|
-
content: ISequencedDocumentMessage,
|
|
55
|
-
) => Promise<unknown>;
|
|
52
|
+
applyStashedOp: (type: ContainerMessageType, content: unknown) => Promise<unknown>;
|
|
56
53
|
reSubmit(
|
|
57
54
|
type: ContainerMessageType,
|
|
58
55
|
content: any,
|
|
@@ -416,14 +416,13 @@ export class OrderedClientElection
|
|
|
416
416
|
change = true;
|
|
417
417
|
}
|
|
418
418
|
if (change) {
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
426
|
-
}
|
|
419
|
+
this.logger.sendTelemetryEvent({
|
|
420
|
+
eventName: "SummarizerClientElected",
|
|
421
|
+
electedClientId: this._electedClient?.clientId,
|
|
422
|
+
electedParentId: this._electedParent?.clientId,
|
|
423
|
+
electionSequenceNumber: sequenceNumber,
|
|
424
|
+
isSummarizerClient,
|
|
425
|
+
});
|
|
427
426
|
this.emit("election", client, sequenceNumber, prevClient);
|
|
428
427
|
}
|
|
429
428
|
}
|
|
@@ -431,14 +430,12 @@ export class OrderedClientElection
|
|
|
431
430
|
private tryElectingParent(client: ILinkedClient | undefined, sequenceNumber: number): void {
|
|
432
431
|
if (this._electedParent !== client) {
|
|
433
432
|
this._electedParent = client;
|
|
434
|
-
|
|
435
|
-
|
|
436
|
-
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
|
|
440
|
-
});
|
|
441
|
-
}
|
|
433
|
+
this.logger.sendTelemetryEvent({
|
|
434
|
+
eventName: "SummarizerParentElected",
|
|
435
|
+
electedClientId: this._electedClient?.clientId,
|
|
436
|
+
electedParentId: this._electedParent?.clientId,
|
|
437
|
+
electionSequenceNumber: sequenceNumber,
|
|
438
|
+
});
|
|
442
439
|
this.emit("election", this._electedClient, sequenceNumber, this._electedClient);
|
|
443
440
|
}
|
|
444
441
|
}
|
|
@@ -473,7 +470,7 @@ export class OrderedClientElection
|
|
|
473
470
|
const newClientIsSummarizer = client.client.details.type === summarizerClientType;
|
|
474
471
|
const electedClientIsSummarizer =
|
|
475
472
|
this._electedClient?.client.details.type === summarizerClientType;
|
|
476
|
-
// Note that we allow a summarizer client to
|
|
473
|
+
// Note that we allow a summarizer client to supersede an interactive client as elected client.
|
|
477
474
|
if (
|
|
478
475
|
this._electedClient === undefined ||
|
|
479
476
|
(!electedClientIsSummarizer && newClientIsSummarizer)
|
|
@@ -25,6 +25,7 @@ import { RunningSummarizer } from "./runningSummarizer";
|
|
|
25
25
|
import {
|
|
26
26
|
IConnectableRuntime,
|
|
27
27
|
ISummarizer,
|
|
28
|
+
ISummarizeHeuristicData,
|
|
28
29
|
ISummarizerInternalsProvider,
|
|
29
30
|
ISummarizerRuntime,
|
|
30
31
|
ISummarizingWarning,
|
|
@@ -215,6 +216,8 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
215
216
|
return stopReason === "parentNotConnected";
|
|
216
217
|
}
|
|
217
218
|
|
|
219
|
+
private _heuristicData: ISummarizeHeuristicData | undefined;
|
|
220
|
+
|
|
218
221
|
/**
|
|
219
222
|
* Put the summarizer in a started state, including creating and initializing the RunningSummarizer.
|
|
220
223
|
* The start request can come either from the SummaryManager (in the auto-summarize case) or from the user
|
|
@@ -252,17 +255,22 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
252
255
|
throw new UsageError("clientId should be defined if connected.");
|
|
253
256
|
}
|
|
254
257
|
|
|
258
|
+
this._heuristicData = new SummarizeHeuristicData(
|
|
259
|
+
this.runtime.deltaManager.lastSequenceNumber,
|
|
260
|
+
{
|
|
261
|
+
/** summary attempt baseline for heuristics */
|
|
262
|
+
refSequenceNumber: this.runtime.deltaManager.initialSequenceNumber,
|
|
263
|
+
summaryTime: Date.now(),
|
|
264
|
+
} as const,
|
|
265
|
+
);
|
|
266
|
+
|
|
255
267
|
const runningSummarizer = await RunningSummarizer.start(
|
|
256
268
|
this.logger,
|
|
257
269
|
this.summaryCollection.createWatcher(clientId),
|
|
258
270
|
this.configurationGetter(),
|
|
259
271
|
async (...args) => this.internalsProvider.submitSummary(...args), // submitSummaryCallback
|
|
260
272
|
async (...args) => this.internalsProvider.refreshLatestSummaryAck(...args), // refreshLatestSummaryCallback
|
|
261
|
-
|
|
262
|
-
/** summary attempt baseline for heuristics */
|
|
263
|
-
refSequenceNumber: this.runtime.deltaManager.initialSequenceNumber,
|
|
264
|
-
summaryTime: Date.now(),
|
|
265
|
-
} as const),
|
|
273
|
+
this._heuristicData,
|
|
266
274
|
this.summaryCollection,
|
|
267
275
|
runCoordinator /* cancellationToken */,
|
|
268
276
|
(reason) => runCoordinator.stop(reason) /* stopSummarizerCallback */,
|
|
@@ -361,4 +369,8 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
361
369
|
}
|
|
362
370
|
return this.runningSummarizer.enqueueSummarize(...args);
|
|
363
371
|
};
|
|
372
|
+
|
|
373
|
+
public recordSummaryAttempt?(summaryRefSeqNum?: number) {
|
|
374
|
+
this._heuristicData?.recordAttempt(summaryRefSeqNum);
|
|
375
|
+
}
|
|
364
376
|
}
|