@fluidframework/container-runtime 2.1.0 → 2.3.0-288113
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +30 -0
- package/README.md +2 -2
- package/api-report/container-runtime.legacy.alpha.api.md +4 -3
- package/container-runtime.test-files.tar +0 -0
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +9 -0
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/channelCollection.d.ts +0 -14
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +2 -12
- package/dist/channelCollection.js.map +1 -1
- package/dist/containerRuntime.d.ts +34 -6
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +176 -81
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +9 -18
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +40 -78
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +0 -6
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +23 -66
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +11 -34
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +9 -52
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +3 -23
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +2 -6
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +4 -8
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +1 -9
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +3 -25
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -7
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -2
- package/dist/index.js.map +1 -1
- package/dist/messageTypes.d.ts +6 -5
- package/dist/messageTypes.d.ts.map +1 -1
- package/dist/messageTypes.js.map +1 -1
- package/dist/metadata.d.ts +9 -1
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +6 -1
- package/dist/metadata.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +8 -0
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +34 -2
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +1 -0
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +20 -0
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +34 -15
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +59 -46
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +28 -27
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +143 -112
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.js +5 -1
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +3 -4
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +16 -15
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +9 -0
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/channelCollection.d.ts +0 -14
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +2 -11
- package/lib/channelCollection.js.map +1 -1
- package/lib/containerRuntime.d.ts +34 -6
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +176 -81
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +9 -18
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +27 -65
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +0 -6
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +25 -68
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +12 -35
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +9 -52
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +2 -22
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +2 -6
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +4 -8
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +1 -9
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +3 -24
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -2
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/messageTypes.d.ts +6 -5
- package/lib/messageTypes.d.ts.map +1 -1
- package/lib/messageTypes.js.map +1 -1
- package/lib/metadata.d.ts +9 -1
- package/lib/metadata.d.ts.map +1 -1
- package/lib/metadata.js +4 -0
- package/lib/metadata.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +8 -0
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +34 -2
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +1 -0
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +20 -0
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +34 -15
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +59 -46
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +28 -27
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +144 -113
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.js +5 -1
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +3 -4
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +16 -15
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/package.json +22 -35
- package/src/batchTracker.ts +4 -2
- package/src/blobManager/blobManager.ts +9 -0
- package/src/channelCollection.ts +2 -11
- package/src/containerRuntime.ts +211 -109
- package/src/dataStoreContext.ts +29 -93
- package/src/gc/garbageCollection.ts +26 -79
- package/src/gc/gcConfigs.ts +12 -45
- package/src/gc/gcDefinitions.ts +10 -55
- package/src/gc/gcHelpers.ts +10 -8
- package/src/gc/gcSummaryStateTracker.ts +6 -9
- package/src/gc/gcTelemetry.ts +3 -38
- package/src/gc/index.ts +2 -6
- package/src/index.ts +0 -1
- package/src/messageTypes.ts +12 -11
- package/src/metadata.ts +16 -2
- package/src/opLifecycle/index.ts +1 -0
- package/src/opLifecycle/opGroupingManager.ts +42 -3
- package/src/opLifecycle/outbox.ts +30 -0
- package/src/opLifecycle/remoteMessageProcessor.ts +98 -62
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +199 -177
- package/src/summary/README.md +31 -28
- package/src/summary/summarizerNode/summarizerNode.ts +6 -1
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +20 -43
- package/src/summary/summaryFormats.md +25 -22
package/src/gc/gcHelpers.ts
CHANGED
|
@@ -240,10 +240,11 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
|
|
|
240
240
|
continue;
|
|
241
241
|
}
|
|
242
242
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
243
|
+
const childId = id.split("/")[1];
|
|
244
|
+
assert(
|
|
245
|
+
childId !== undefined,
|
|
246
|
+
0x9fe /* node id should be an absolute route with child id part */,
|
|
247
|
+
);
|
|
247
248
|
let childGCNodeId = id.slice(childId.length + 1);
|
|
248
249
|
// GC node id always begins with "/". Handle the special case where a child's id in the parent's GC nodes is
|
|
249
250
|
// of format `/root`. In this case, the childId is root and childGCNodeId is "". Make childGCNodeId = "/".
|
|
@@ -271,10 +272,11 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
|
|
|
271
272
|
// Remove the node's self used route, if any, and generate the children used routes.
|
|
272
273
|
const usedRoutes = gcDetails.usedRoutes.filter((route) => route !== "" && route !== "/");
|
|
273
274
|
for (const route of usedRoutes) {
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
275
|
+
const childId = route.split("/")[1];
|
|
276
|
+
assert(
|
|
277
|
+
childId !== undefined,
|
|
278
|
+
0x9ff /* used route should be an absolute route with child id part */,
|
|
279
|
+
);
|
|
278
280
|
const childUsedRoute = route.slice(childId.length + 1);
|
|
279
281
|
|
|
280
282
|
const childGCDetails = childGCDetailsMap.get(childId);
|
|
@@ -66,7 +66,7 @@ export class GCSummaryStateTracker {
|
|
|
66
66
|
// Tells whether GC should run or not.
|
|
67
67
|
private readonly configs: Pick<
|
|
68
68
|
IGarbageCollectorConfigs,
|
|
69
|
-
"
|
|
69
|
+
"gcAllowed" | "gcVersionInBaseSnapshot" | "gcVersionInEffect"
|
|
70
70
|
>,
|
|
71
71
|
) {}
|
|
72
72
|
|
|
@@ -100,7 +100,7 @@ export class GCSummaryStateTracker {
|
|
|
100
100
|
deletedNodes: Set<string>,
|
|
101
101
|
tombstones: string[],
|
|
102
102
|
): ISummarizeResult | undefined {
|
|
103
|
-
if (!this.configs.
|
|
103
|
+
if (!this.configs.gcAllowed) {
|
|
104
104
|
return;
|
|
105
105
|
}
|
|
106
106
|
|
|
@@ -109,12 +109,9 @@ export class GCSummaryStateTracker {
|
|
|
109
109
|
// to identify deleted nodes' usage.
|
|
110
110
|
const serializedDeletedNodes =
|
|
111
111
|
deletedNodes.size > 0 ? JSON.stringify(Array.from(deletedNodes).sort()) : undefined;
|
|
112
|
-
//
|
|
113
|
-
const serializedTombstones =
|
|
114
|
-
|
|
115
|
-
? JSON.stringify(tombstones.sort())
|
|
116
|
-
: undefined
|
|
117
|
-
: undefined;
|
|
112
|
+
// Serialize and write tombstones, if any.
|
|
113
|
+
const serializedTombstones =
|
|
114
|
+
tombstones.length > 0 ? JSON.stringify(tombstones.sort()) : undefined;
|
|
118
115
|
|
|
119
116
|
/**
|
|
120
117
|
* Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
|
|
@@ -231,7 +228,7 @@ export class GCSummaryStateTracker {
|
|
|
231
228
|
* Called to refresh the latest summary state. This happens when a pending summary is acked.
|
|
232
229
|
*/
|
|
233
230
|
public async refreshLatestSummary(result: IRefreshSummaryResult): Promise<void> {
|
|
234
|
-
if (!this.configs.
|
|
231
|
+
if (!this.configs.gcAllowed || !result.isSummaryTracked) {
|
|
235
232
|
return;
|
|
236
233
|
}
|
|
237
234
|
|
package/src/gc/gcTelemetry.ts
CHANGED
|
@@ -10,7 +10,6 @@ import {
|
|
|
10
10
|
MonitoringContext,
|
|
11
11
|
generateStack,
|
|
12
12
|
tagCodeArtifacts,
|
|
13
|
-
type ITelemetryGenericEventExt,
|
|
14
13
|
type ITelemetryPropertiesExt,
|
|
15
14
|
} from "@fluidframework/telemetry-utils/internal";
|
|
16
15
|
|
|
@@ -22,11 +21,7 @@ import {
|
|
|
22
21
|
GCNodeType,
|
|
23
22
|
IGarbageCollectorConfigs,
|
|
24
23
|
UnreferencedState,
|
|
25
|
-
disableTombstoneKey,
|
|
26
|
-
throwOnTombstoneLoadOverrideKey,
|
|
27
|
-
throwOnTombstoneUsageKey,
|
|
28
24
|
} from "./gcDefinitions.js";
|
|
29
|
-
import { getGCVersionInEffect } from "./gcHelpers.js";
|
|
30
25
|
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.js";
|
|
31
26
|
|
|
32
27
|
type NodeUsageType = "Changed" | "Loaded" | "Revived" | "Realized";
|
|
@@ -287,18 +282,12 @@ export class GCTelemetryTracker {
|
|
|
287
282
|
headers: { ...headers },
|
|
288
283
|
details: { ...detailedProps, ...additionalProps }, // Also includes some properties from INodeUsageProps type
|
|
289
284
|
gcConfigs,
|
|
290
|
-
tombstoneFlags: {
|
|
291
|
-
DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
|
|
292
|
-
ThrowOnTombstoneUsage: this.mc.config.getBoolean(throwOnTombstoneUsageKey),
|
|
293
|
-
ThrowOnTombstoneLoad: this.mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
|
|
294
|
-
},
|
|
295
285
|
};
|
|
296
286
|
|
|
297
287
|
if (
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
(usageType === "Changed" && this.configs.throwOnTombstoneUsage)
|
|
288
|
+
usageType === "Loaded" &&
|
|
289
|
+
this.configs.throwOnTombstoneLoad &&
|
|
290
|
+
!headers?.allowTombstone
|
|
302
291
|
) {
|
|
303
292
|
this.mc.logger.sendErrorEvent(event);
|
|
304
293
|
} else {
|
|
@@ -419,27 +408,3 @@ export class GCTelemetryTracker {
|
|
|
419
408
|
this.pendingEventsQueue = [];
|
|
420
409
|
}
|
|
421
410
|
}
|
|
422
|
-
|
|
423
|
-
/**
|
|
424
|
-
* Consolidates info / logic for logging when we encounter unexpected usage of GC'd objects. For example, when a
|
|
425
|
-
* tombstoned or deleted object is loaded.
|
|
426
|
-
*/
|
|
427
|
-
export function sendGCUnexpectedUsageEvent(
|
|
428
|
-
mc: MonitoringContext,
|
|
429
|
-
event: ITelemetryGenericEventExt & {
|
|
430
|
-
category: "error" | "generic";
|
|
431
|
-
gcTombstoneEnforcementAllowed: boolean | undefined;
|
|
432
|
-
},
|
|
433
|
-
packagePath: readonly string[] | undefined,
|
|
434
|
-
error?: unknown,
|
|
435
|
-
) {
|
|
436
|
-
event.pkg = tagCodeArtifacts({ pkg: packagePath?.join("/") })?.pkg;
|
|
437
|
-
event.tombstoneFlags = JSON.stringify({
|
|
438
|
-
DisableTombstone: mc.config.getBoolean(disableTombstoneKey),
|
|
439
|
-
ThrowOnTombstoneUsage: mc.config.getBoolean(throwOnTombstoneUsageKey),
|
|
440
|
-
ThrowOnTombstoneLoad: mc.config.getBoolean(throwOnTombstoneLoadOverrideKey),
|
|
441
|
-
});
|
|
442
|
-
event.gcVersion = getGCVersionInEffect(mc.config);
|
|
443
|
-
|
|
444
|
-
mc.logger.sendTelemetryEvent(event, error);
|
|
445
|
-
}
|
package/src/gc/index.ts
CHANGED
|
@@ -11,8 +11,6 @@ export {
|
|
|
11
11
|
defaultSessionExpiryDurationMs,
|
|
12
12
|
GCNodeType,
|
|
13
13
|
gcTestModeKey,
|
|
14
|
-
gcDisableDataStoreSweepOptionName,
|
|
15
|
-
gcDisableThrowOnTombstoneLoadOptionName,
|
|
16
14
|
gcGenerationOptionName,
|
|
17
15
|
GCFeatureMatrix,
|
|
18
16
|
GCVersion,
|
|
@@ -32,10 +30,8 @@ export {
|
|
|
32
30
|
oneDayMs,
|
|
33
31
|
runSessionExpiryKey,
|
|
34
32
|
stableGCVersion,
|
|
35
|
-
disableAutoRecoveryKey,
|
|
36
|
-
disableDatastoreSweepKey,
|
|
37
33
|
UnreferencedState,
|
|
38
|
-
|
|
34
|
+
disableThrowOnTombstoneLoadKey,
|
|
39
35
|
GarbageCollectionMessage,
|
|
40
36
|
GarbageCollectionMessageType,
|
|
41
37
|
ISweepMessage,
|
|
@@ -59,5 +55,5 @@ export {
|
|
|
59
55
|
GCSummaryStateTracker,
|
|
60
56
|
IGCSummaryTrackingData,
|
|
61
57
|
} from "./gcSummaryStateTracker.js";
|
|
62
|
-
export { GCTelemetryTracker
|
|
58
|
+
export { GCTelemetryTracker } from "./gcTelemetry.js";
|
|
63
59
|
export { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.js";
|
package/src/index.ts
CHANGED
package/src/messageTypes.ts
CHANGED
|
@@ -88,12 +88,18 @@ export interface IContainerRuntimeMessageCompatDetails {
|
|
|
88
88
|
* IMPORTANT: when creating one to be serialized, set the properties in the order they appear here.
|
|
89
89
|
* This way stringified values can be compared.
|
|
90
90
|
*/
|
|
91
|
-
|
|
91
|
+
type TypedContainerRuntimeMessage<
|
|
92
|
+
TType extends ContainerMessageType,
|
|
93
|
+
TContents,
|
|
94
|
+
TUSedCompatDetails extends boolean = false,
|
|
95
|
+
> = {
|
|
92
96
|
/** Type of the op, within the ContainerRuntime's domain */
|
|
93
97
|
type: TType;
|
|
94
98
|
/** Domain-specific contents, interpreted according to the type */
|
|
95
99
|
contents: TContents;
|
|
96
|
-
}
|
|
100
|
+
} & (TUSedCompatDetails extends true
|
|
101
|
+
? Partial<RecentlyAddedContainerRuntimeMessageDetails>
|
|
102
|
+
: { compatDetails?: undefined });
|
|
97
103
|
|
|
98
104
|
/**
|
|
99
105
|
* Additional details expected for any recently added message.
|
|
@@ -139,10 +145,9 @@ export type ContainerRuntimeIdAllocationMessage = TypedContainerRuntimeMessage<
|
|
|
139
145
|
>;
|
|
140
146
|
export type ContainerRuntimeGCMessage = TypedContainerRuntimeMessage<
|
|
141
147
|
ContainerMessageType.GC,
|
|
142
|
-
GarbageCollectionMessage
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
Partial<RecentlyAddedContainerRuntimeMessageDetails>;
|
|
148
|
+
GarbageCollectionMessage,
|
|
149
|
+
true // TUsedCompatDetails
|
|
150
|
+
>;
|
|
146
151
|
export type ContainerRuntimeDocumentSchemaMessage = TypedContainerRuntimeMessage<
|
|
147
152
|
ContainerMessageType.DocumentSchemaChange,
|
|
148
153
|
IDocumentSchemaChangeMessage
|
|
@@ -223,15 +228,11 @@ export type InboundSequencedContainerRuntimeMessage = Omit<
|
|
|
223
228
|
* There should never be a runtime value of "__not_a_...".
|
|
224
229
|
* Currently additionally replaces `contents` type until protocol-definitions update is taken with `unknown` instead of `any`.
|
|
225
230
|
*/
|
|
226
|
-
type InboundSequencedNonContainerRuntimeMessage = Omit<
|
|
231
|
+
export type InboundSequencedNonContainerRuntimeMessage = Omit<
|
|
227
232
|
ISequencedDocumentMessage,
|
|
228
233
|
"type" | "contents"
|
|
229
234
|
> & { type: "__not_a_container_runtime_message_type__"; contents: unknown };
|
|
230
235
|
|
|
231
|
-
export type InboundSequencedContainerRuntimeMessageOrSystemMessage =
|
|
232
|
-
| InboundSequencedContainerRuntimeMessage
|
|
233
|
-
| InboundSequencedNonContainerRuntimeMessage;
|
|
234
|
-
|
|
235
236
|
/** A [loose] InboundSequencedContainerRuntimeMessage that is recent and may contain compat details.
|
|
236
237
|
* It exists solely to to provide access to those details.
|
|
237
238
|
*/
|
package/src/metadata.ts
CHANGED
|
@@ -6,10 +6,24 @@
|
|
|
6
6
|
import type { BatchId } from "./opLifecycle/index.js";
|
|
7
7
|
|
|
8
8
|
/** Syntactic sugar for casting */
|
|
9
|
-
export function asBatchMetadata(metadata: unknown): IBatchMetadata | undefined {
|
|
10
|
-
return metadata as IBatchMetadata | undefined;
|
|
9
|
+
export function asBatchMetadata(metadata: unknown): Partial<IBatchMetadata> | undefined {
|
|
10
|
+
return metadata as Partial<IBatchMetadata> | undefined;
|
|
11
11
|
}
|
|
12
12
|
|
|
13
|
+
/** Syntactic sugar for casting */
|
|
14
|
+
export function asEmptyBatchLocalOpMetadata(
|
|
15
|
+
localOpMetadata: unknown,
|
|
16
|
+
): IEmptyBatchMetadata | undefined {
|
|
17
|
+
return localOpMetadata as IEmptyBatchMetadata | undefined;
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
/**
|
|
21
|
+
* Properties put on the localOpMetadata object for empty batches
|
|
22
|
+
*/
|
|
23
|
+
export interface IEmptyBatchMetadata {
|
|
24
|
+
// Set to true on localOpMetadata for empty batches
|
|
25
|
+
emptyBatch?: boolean;
|
|
26
|
+
}
|
|
13
27
|
/**
|
|
14
28
|
* Properties put on the op metadata object for batch tracking
|
|
15
29
|
*/
|
package/src/opLifecycle/index.ts
CHANGED
|
@@ -18,6 +18,7 @@ export { OpDecompressor } from "./opDecompressor.js";
|
|
|
18
18
|
export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
|
|
19
19
|
export {
|
|
20
20
|
ensureContentsDeserialized,
|
|
21
|
+
InboundBatch,
|
|
21
22
|
RemoteMessageProcessor,
|
|
22
23
|
unpackRuntimeMessage,
|
|
23
24
|
} from "./remoteMessageProcessor.js";
|
|
@@ -49,6 +49,40 @@ export class OpGroupingManager {
|
|
|
49
49
|
this.logger = createChildLogger({ logger, namespace: "OpGroupingManager" });
|
|
50
50
|
}
|
|
51
51
|
|
|
52
|
+
/**
|
|
53
|
+
* Creates a new batch with a single message of type "groupedBatch" and empty contents.
|
|
54
|
+
* This is needed as a placeholder if a batch becomes empty on resubmit, but we are tracking batch IDs.
|
|
55
|
+
* @param resubmittingBatchId - batch ID of the resubmitting batch
|
|
56
|
+
* @param referenceSequenceNumber - reference sequence number
|
|
57
|
+
* @returns - IBatch containing a single empty Grouped Batch op
|
|
58
|
+
*/
|
|
59
|
+
public createEmptyGroupedBatch(
|
|
60
|
+
resubmittingBatchId: string,
|
|
61
|
+
referenceSequenceNumber: number,
|
|
62
|
+
): IBatch<[BatchMessage]> {
|
|
63
|
+
assert(
|
|
64
|
+
this.config.groupedBatchingEnabled,
|
|
65
|
+
0xa00 /* cannot create empty grouped batch when grouped batching is disabled */,
|
|
66
|
+
);
|
|
67
|
+
const serializedContent = JSON.stringify({
|
|
68
|
+
type: OpGroupingManager.groupedBatchOp,
|
|
69
|
+
contents: [],
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
return {
|
|
73
|
+
contentSizeInBytes: 0,
|
|
74
|
+
messages: [
|
|
75
|
+
{
|
|
76
|
+
metadata: { batchId: resubmittingBatchId },
|
|
77
|
+
localOpMetadata: { emptyBatch: true },
|
|
78
|
+
referenceSequenceNumber,
|
|
79
|
+
contents: serializedContent,
|
|
80
|
+
},
|
|
81
|
+
],
|
|
82
|
+
referenceSequenceNumber,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
|
|
52
86
|
/**
|
|
53
87
|
* Converts the given batch into a "grouped batch" - a batch with a single message of type "groupedBatch",
|
|
54
88
|
* with contents being an array of the original batch's messages.
|
|
@@ -70,10 +104,14 @@ export class OpGroupingManager {
|
|
|
70
104
|
referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
|
|
71
105
|
});
|
|
72
106
|
}
|
|
73
|
-
|
|
107
|
+
// We expect this will be on the first message, if present at all.
|
|
108
|
+
let groupedBatchId;
|
|
74
109
|
for (const message of batch.messages) {
|
|
75
110
|
if (message.metadata) {
|
|
76
111
|
const { batch: _batch, batchId, ...rest } = message.metadata;
|
|
112
|
+
if (batchId) {
|
|
113
|
+
groupedBatchId = batchId;
|
|
114
|
+
}
|
|
77
115
|
assert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);
|
|
78
116
|
}
|
|
79
117
|
}
|
|
@@ -91,7 +129,7 @@ export class OpGroupingManager {
|
|
|
91
129
|
...batch,
|
|
92
130
|
messages: [
|
|
93
131
|
{
|
|
94
|
-
metadata:
|
|
132
|
+
metadata: { batchId: groupedBatchId },
|
|
95
133
|
// TODO why are we non null asserting here?
|
|
96
134
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
97
135
|
referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
|
|
@@ -121,7 +159,8 @@ export class OpGroupingManager {
|
|
|
121
159
|
// Grouped batching must be enabled
|
|
122
160
|
this.config.groupedBatchingEnabled &&
|
|
123
161
|
// The number of ops in the batch must surpass the configured threshold
|
|
124
|
-
|
|
162
|
+
// or be empty (to allow for empty batches to be grouped)
|
|
163
|
+
(batch.messages.length === 0 || batch.messages.length >= this.config.opCountThreshold) &&
|
|
125
164
|
// Support for reentrant batches must be explicitly enabled
|
|
126
165
|
(this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
|
|
127
166
|
);
|
|
@@ -254,6 +254,14 @@ export class Outbox {
|
|
|
254
254
|
// Don't use resubmittingBatchId for idAllocationBatch.
|
|
255
255
|
// ID Allocation messages are not directly resubmitted so we don't want to reuse the batch ID.
|
|
256
256
|
this.flushInternal(this.idAllocationBatch);
|
|
257
|
+
// We need to flush an empty batch if the main batch *becomes* empty on resubmission.
|
|
258
|
+
// When resubmitting the main batch, the blobAttach batch will always be empty since we don't resubmit them simultaneously.
|
|
259
|
+
// And conversely, the blobAttach will never *become* empty on resubmit.
|
|
260
|
+
// So if both blobAttachBatch and mainBatch are empty, we must submit an empty main batch.
|
|
261
|
+
if (resubmittingBatchId && this.blobAttachBatch.empty && this.mainBatch.empty) {
|
|
262
|
+
this.flushEmptyBatch(resubmittingBatchId);
|
|
263
|
+
return;
|
|
264
|
+
}
|
|
257
265
|
this.flushInternal(
|
|
258
266
|
this.blobAttachBatch,
|
|
259
267
|
true /* disableGroupedBatching */,
|
|
@@ -266,6 +274,28 @@ export class Outbox {
|
|
|
266
274
|
);
|
|
267
275
|
}
|
|
268
276
|
|
|
277
|
+
private flushEmptyBatch(resubmittingBatchId: BatchId) {
|
|
278
|
+
const referenceSequenceNumber =
|
|
279
|
+
this.params.getCurrentSequenceNumbers().referenceSequenceNumber;
|
|
280
|
+
assert(
|
|
281
|
+
referenceSequenceNumber !== undefined,
|
|
282
|
+
0xa01 /* reference sequence number should be defined */,
|
|
283
|
+
);
|
|
284
|
+
const emptyGroupedBatch = this.params.groupingManager.createEmptyGroupedBatch(
|
|
285
|
+
resubmittingBatchId,
|
|
286
|
+
referenceSequenceNumber,
|
|
287
|
+
);
|
|
288
|
+
let clientSequenceNumber: number | undefined;
|
|
289
|
+
if (this.params.shouldSend()) {
|
|
290
|
+
clientSequenceNumber = this.sendBatch(emptyGroupedBatch);
|
|
291
|
+
}
|
|
292
|
+
this.params.pendingStateManager.onFlushBatch(
|
|
293
|
+
emptyGroupedBatch.messages, // This is the single empty Grouped Batch message
|
|
294
|
+
clientSequenceNumber,
|
|
295
|
+
);
|
|
296
|
+
return;
|
|
297
|
+
}
|
|
298
|
+
|
|
269
299
|
private flushInternal(
|
|
270
300
|
batchManager: BatchManager,
|
|
271
301
|
disableGroupedBatching: boolean = false,
|
|
@@ -21,6 +21,35 @@ import { OpDecompressor } from "./opDecompressor.js";
|
|
|
21
21
|
import { OpGroupingManager, isGroupedBatch } from "./opGroupingManager.js";
|
|
22
22
|
import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
|
|
23
23
|
|
|
24
|
+
/** Messages being received as a batch, with details needed to process the batch */
|
|
25
|
+
export interface InboundBatch {
|
|
26
|
+
/** Messages in this batch */
|
|
27
|
+
readonly messages: InboundSequencedContainerRuntimeMessage[];
|
|
28
|
+
/** Batch ID, if present */
|
|
29
|
+
readonly batchId: string | undefined;
|
|
30
|
+
/** clientId that sent this batch. Used to compute Batch ID if needed */
|
|
31
|
+
readonly clientId: string;
|
|
32
|
+
/**
|
|
33
|
+
* Client Sequence Number of the first message in the batch.
|
|
34
|
+
* Used to compute Batch ID if needed
|
|
35
|
+
*
|
|
36
|
+
* @remarks For chunked batches, this is the CSN of the "representative" chunk (the final chunk).
|
|
37
|
+
* For grouped batches, clientSequenceNumber on messages is overwritten, so we track this original value here.
|
|
38
|
+
*/
|
|
39
|
+
readonly batchStartCsn: number;
|
|
40
|
+
/** For an empty batch (with no messages), we need to remember the empty grouped batch's sequence number */
|
|
41
|
+
readonly emptyBatchSequenceNumber?: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
function assertHasClientId(
|
|
45
|
+
message: ISequencedDocumentMessage,
|
|
46
|
+
): asserts message is ISequencedDocumentMessage & { clientId: string } {
|
|
47
|
+
assert(
|
|
48
|
+
message.clientId !== null,
|
|
49
|
+
0xa02 /* Server-generated message should not reach RemoteMessageProcessor */,
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
24
53
|
/**
|
|
25
54
|
* Stateful class for processing incoming remote messages as the virtualization measures are unwrapped,
|
|
26
55
|
* potentially across numerous inbound ops.
|
|
@@ -29,13 +58,11 @@ import { OpSplitter, isChunkedMessage } from "./opSplitter.js";
|
|
|
29
58
|
*/
|
|
30
59
|
export class RemoteMessageProcessor {
|
|
31
60
|
/**
|
|
32
|
-
*
|
|
33
|
-
* If undefined, we are expecting the next message to start a new batch.
|
|
61
|
+
* The current batch being received, with details needed to process it.
|
|
34
62
|
*
|
|
35
|
-
* @remarks
|
|
63
|
+
* @remarks If undefined, we are expecting the next message to start a new batch.
|
|
36
64
|
*/
|
|
37
|
-
private
|
|
38
|
-
private readonly processorBatch: InboundSequencedContainerRuntimeMessage[] = [];
|
|
65
|
+
private batchInProgress: InboundBatch | undefined;
|
|
39
66
|
|
|
40
67
|
constructor(
|
|
41
68
|
private readonly opSplitter: OpSplitter,
|
|
@@ -73,19 +100,17 @@ export class RemoteMessageProcessor {
|
|
|
73
100
|
public process(
|
|
74
101
|
remoteMessageCopy: ISequencedDocumentMessage,
|
|
75
102
|
logLegacyCase: (codePath: string) => void,
|
|
76
|
-
):
|
|
77
|
-
| {
|
|
78
|
-
messages: InboundSequencedContainerRuntimeMessage[];
|
|
79
|
-
batchStartCsn: number;
|
|
80
|
-
}
|
|
81
|
-
| undefined {
|
|
103
|
+
): InboundBatch | undefined {
|
|
82
104
|
let message = remoteMessageCopy;
|
|
83
105
|
|
|
106
|
+
assertHasClientId(message);
|
|
107
|
+
const clientId = message.clientId;
|
|
108
|
+
|
|
84
109
|
if (isChunkedMessage(message)) {
|
|
85
110
|
const chunkProcessingResult = this.opSplitter.processChunk(message);
|
|
86
111
|
// Only continue further if current chunk is the final chunk
|
|
87
112
|
if (!chunkProcessingResult.isFinalChunk) {
|
|
88
|
-
return;
|
|
113
|
+
return undefined;
|
|
89
114
|
}
|
|
90
115
|
// This message will always be compressed
|
|
91
116
|
message = chunkProcessingResult.message;
|
|
@@ -104,100 +129,111 @@ export class RemoteMessageProcessor {
|
|
|
104
129
|
}
|
|
105
130
|
|
|
106
131
|
if (isGroupedBatch(message)) {
|
|
107
|
-
// We should be awaiting a new batch (
|
|
132
|
+
// We should be awaiting a new batch (batchInProgress undefined)
|
|
108
133
|
assert(
|
|
109
|
-
this.
|
|
134
|
+
this.batchInProgress === undefined,
|
|
110
135
|
0x9d3 /* Grouped batch interrupting another batch */,
|
|
111
136
|
);
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
0x9d4 /* Processor batch should be empty on grouped batch */,
|
|
115
|
-
);
|
|
137
|
+
const batchId = asBatchMetadata(message.metadata)?.batchId;
|
|
138
|
+
const groupedMessages = this.opGroupingManager.ungroupOp(message).map(unpack);
|
|
116
139
|
return {
|
|
117
|
-
messages:
|
|
140
|
+
messages: groupedMessages, // Will be [] for an empty batch
|
|
118
141
|
batchStartCsn: message.clientSequenceNumber,
|
|
142
|
+
clientId,
|
|
143
|
+
batchId,
|
|
144
|
+
// If the batch is empty, we need to return the sequence number aside
|
|
145
|
+
emptyBatchSequenceNumber:
|
|
146
|
+
groupedMessages.length === 0 ? message.sequenceNumber : undefined,
|
|
119
147
|
};
|
|
120
148
|
}
|
|
121
149
|
|
|
122
|
-
const batchStartCsn = this.getAndUpdateBatchStartCsn(message);
|
|
123
|
-
|
|
124
150
|
// Do a final unpack of runtime messages in case the message was not grouped, compressed, or chunked
|
|
125
151
|
unpackRuntimeMessage(message, logLegacyCase);
|
|
126
|
-
this.processorBatch.push(message as InboundSequencedContainerRuntimeMessage);
|
|
127
152
|
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
153
|
+
const { batchEnded } = this.addMessageToBatch(
|
|
154
|
+
message as InboundSequencedContainerRuntimeMessage & { clientId: string },
|
|
155
|
+
);
|
|
156
|
+
|
|
157
|
+
if (!batchEnded) {
|
|
132
158
|
// batch not yet complete
|
|
133
159
|
return undefined;
|
|
134
160
|
}
|
|
135
161
|
|
|
136
|
-
const
|
|
137
|
-
this.
|
|
138
|
-
return
|
|
139
|
-
messages,
|
|
140
|
-
batchStartCsn,
|
|
141
|
-
};
|
|
162
|
+
const completedBatch = this.batchInProgress;
|
|
163
|
+
this.batchInProgress = undefined;
|
|
164
|
+
return completedBatch;
|
|
142
165
|
}
|
|
143
166
|
|
|
144
167
|
/**
|
|
145
|
-
*
|
|
146
|
-
*
|
|
147
|
-
* the batch
|
|
168
|
+
* Add the given message to the current batch, and indicate whether the batch is now complete.
|
|
169
|
+
*
|
|
170
|
+
* @returns batchEnded: true if the batch is now complete, batchEnded: false if more messages are expected
|
|
148
171
|
*/
|
|
149
|
-
private
|
|
172
|
+
private addMessageToBatch(
|
|
173
|
+
message: InboundSequencedContainerRuntimeMessage & { clientId: string },
|
|
174
|
+
): { batchEnded: boolean } {
|
|
150
175
|
const batchMetadataFlag = asBatchMetadata(message.metadata)?.batch;
|
|
151
|
-
if (this.
|
|
176
|
+
if (this.batchInProgress === undefined) {
|
|
152
177
|
// We are waiting for a new batch
|
|
153
178
|
assert(batchMetadataFlag !== false, 0x9d5 /* Unexpected batch end marker */);
|
|
154
179
|
|
|
155
180
|
// Start of a new multi-message batch
|
|
156
181
|
if (batchMetadataFlag === true) {
|
|
157
|
-
this.
|
|
158
|
-
|
|
182
|
+
this.batchInProgress = {
|
|
183
|
+
messages: [message],
|
|
184
|
+
batchId: asBatchMetadata(message.metadata)?.batchId,
|
|
185
|
+
clientId: message.clientId,
|
|
186
|
+
batchStartCsn: message.clientSequenceNumber,
|
|
187
|
+
};
|
|
188
|
+
|
|
189
|
+
return { batchEnded: false };
|
|
159
190
|
}
|
|
160
191
|
|
|
161
192
|
// Single-message batch (Since metadata flag is undefined)
|
|
162
|
-
|
|
163
|
-
|
|
193
|
+
this.batchInProgress = {
|
|
194
|
+
messages: [message],
|
|
195
|
+
batchStartCsn: message.clientSequenceNumber,
|
|
196
|
+
clientId: message.clientId,
|
|
197
|
+
batchId: asBatchMetadata(message.metadata)?.batchId,
|
|
198
|
+
};
|
|
199
|
+
return { batchEnded: true };
|
|
164
200
|
}
|
|
165
|
-
|
|
166
|
-
// We are in the middle or end of an existing multi-message batch. Return the current batchStartCsn
|
|
167
|
-
const batchStartCsn = this.batchStartCsn;
|
|
168
|
-
|
|
169
201
|
assert(batchMetadataFlag !== true, 0x9d6 /* Unexpected batch start marker */);
|
|
170
|
-
if (batchMetadataFlag === false) {
|
|
171
|
-
// Batch end? Then get ready for the next batch to start
|
|
172
|
-
this.batchStartCsn = undefined;
|
|
173
|
-
}
|
|
174
202
|
|
|
175
|
-
|
|
203
|
+
this.batchInProgress.messages.push(message);
|
|
204
|
+
|
|
205
|
+
return { batchEnded: batchMetadataFlag === false };
|
|
176
206
|
}
|
|
177
207
|
}
|
|
178
208
|
|
|
179
|
-
/**
|
|
209
|
+
/**
|
|
210
|
+
* Takes an incoming message and if the contents is a string, JSON.parse's it in place
|
|
211
|
+
* @param mutableMessage - op message received
|
|
212
|
+
* @param hasModernRuntimeMessageEnvelope - false if the message does not contain the modern op envelop where message.type is MessageType.Operation
|
|
213
|
+
* @param logLegacyCase - callback to log when legacy op is encountered
|
|
214
|
+
*/
|
|
180
215
|
export function ensureContentsDeserialized(
|
|
181
216
|
mutableMessage: ISequencedDocumentMessage,
|
|
182
|
-
|
|
217
|
+
hasModernRuntimeMessageEnvelope: boolean,
|
|
183
218
|
logLegacyCase: (codePath: string) => void,
|
|
184
219
|
): void {
|
|
185
|
-
//
|
|
186
|
-
//
|
|
187
|
-
//
|
|
188
|
-
|
|
220
|
+
// Currently the loader layer is parsing the contents of the message as JSON if it is a string,
|
|
221
|
+
// so we never expect to see this case.
|
|
222
|
+
// We intend to remove that logic from the Loader, at which point we will have it here.
|
|
223
|
+
// Only hasModernRuntimeMessageEnvelope true will be expected to have JSON contents.
|
|
224
|
+
let didParseJsonContents: boolean;
|
|
189
225
|
if (typeof mutableMessage.contents === "string" && mutableMessage.contents !== "") {
|
|
190
226
|
mutableMessage.contents = JSON.parse(mutableMessage.contents);
|
|
191
|
-
|
|
227
|
+
didParseJsonContents = true;
|
|
192
228
|
} else {
|
|
193
|
-
|
|
229
|
+
didParseJsonContents = false;
|
|
194
230
|
}
|
|
195
231
|
|
|
196
|
-
//
|
|
197
|
-
//
|
|
232
|
+
// The DeltaManager parses the contents of the message as JSON if it is a string,
|
|
233
|
+
// so we should never end up parsing it here.
|
|
198
234
|
// Let's observe if we are wrong about this to learn about these cases.
|
|
199
|
-
if (
|
|
200
|
-
logLegacyCase("
|
|
235
|
+
if (didParseJsonContents) {
|
|
236
|
+
logLegacyCase("ensureContentsDeserialized_foundJsonContents");
|
|
201
237
|
}
|
|
202
238
|
}
|
|
203
239
|
|
package/src/packageVersion.ts
CHANGED