@fluidframework/container-runtime 2.1.0-276985 → 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (199) hide show
  1. package/CHANGELOG.md +4 -0
  2. package/README.md +71 -18
  3. package/api-extractor/api-extractor.current.json +5 -0
  4. package/api-extractor/api-extractor.legacy.json +1 -1
  5. package/api-extractor.json +1 -1
  6. package/api-report/container-runtime.legacy.public.api.md +9 -0
  7. package/container-runtime.test-files.tar +0 -0
  8. package/dist/blobManager/blobManager.d.ts +10 -0
  9. package/dist/blobManager/blobManager.d.ts.map +1 -1
  10. package/dist/blobManager/blobManager.js +19 -0
  11. package/dist/blobManager/blobManager.js.map +1 -1
  12. package/dist/channelCollection.d.ts +1 -1
  13. package/dist/channelCollection.d.ts.map +1 -1
  14. package/dist/channelCollection.js +40 -8
  15. package/dist/channelCollection.js.map +1 -1
  16. package/dist/containerRuntime.d.ts +14 -5
  17. package/dist/containerRuntime.d.ts.map +1 -1
  18. package/dist/containerRuntime.js +151 -99
  19. package/dist/containerRuntime.js.map +1 -1
  20. package/dist/dataStoreContext.d.ts +4 -0
  21. package/dist/dataStoreContext.d.ts.map +1 -1
  22. package/dist/dataStoreContext.js +9 -3
  23. package/dist/dataStoreContext.js.map +1 -1
  24. package/dist/gc/garbageCollection.d.ts +1 -1
  25. package/dist/gc/garbageCollection.d.ts.map +1 -1
  26. package/dist/gc/garbageCollection.js +14 -8
  27. package/dist/gc/garbageCollection.js.map +1 -1
  28. package/dist/gc/gcDefinitions.d.ts +4 -2
  29. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  30. package/dist/gc/gcDefinitions.js.map +1 -1
  31. package/dist/gc/gcHelpers.d.ts.map +1 -1
  32. package/dist/gc/gcHelpers.js +12 -0
  33. package/dist/gc/gcHelpers.js.map +1 -1
  34. package/dist/gc/gcTelemetry.d.ts +3 -2
  35. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  36. package/dist/gc/gcTelemetry.js +6 -6
  37. package/dist/gc/gcTelemetry.js.map +1 -1
  38. package/dist/legacy.d.ts +1 -1
  39. package/dist/metadata.d.ts +7 -1
  40. package/dist/metadata.d.ts.map +1 -1
  41. package/dist/metadata.js +6 -0
  42. package/dist/metadata.js.map +1 -1
  43. package/dist/opLifecycle/batchManager.d.ts +8 -1
  44. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  45. package/dist/opLifecycle/batchManager.js +37 -16
  46. package/dist/opLifecycle/batchManager.js.map +1 -1
  47. package/dist/opLifecycle/definitions.d.ts +1 -1
  48. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  49. package/dist/opLifecycle/definitions.js.map +1 -1
  50. package/dist/opLifecycle/index.d.ts +2 -2
  51. package/dist/opLifecycle/index.d.ts.map +1 -1
  52. package/dist/opLifecycle/index.js +3 -1
  53. package/dist/opLifecycle/index.js.map +1 -1
  54. package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
  55. package/dist/opLifecycle/opCompressor.js +12 -8
  56. package/dist/opLifecycle/opCompressor.js.map +1 -1
  57. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  58. package/dist/opLifecycle/opGroupingManager.js +14 -11
  59. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  60. package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
  61. package/dist/opLifecycle/opSplitter.js +11 -6
  62. package/dist/opLifecycle/opSplitter.js.map +1 -1
  63. package/dist/opLifecycle/outbox.d.ts +22 -6
  64. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  65. package/dist/opLifecycle/outbox.js +43 -21
  66. package/dist/opLifecycle/outbox.js.map +1 -1
  67. package/dist/opLifecycle/remoteMessageProcessor.d.ts +10 -8
  68. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  69. package/dist/opLifecycle/remoteMessageProcessor.js +39 -15
  70. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  71. package/dist/packageVersion.d.ts +1 -1
  72. package/dist/packageVersion.d.ts.map +1 -1
  73. package/dist/packageVersion.js +1 -1
  74. package/dist/packageVersion.js.map +1 -1
  75. package/dist/pendingStateManager.d.ts +37 -13
  76. package/dist/pendingStateManager.d.ts.map +1 -1
  77. package/dist/pendingStateManager.js +95 -45
  78. package/dist/pendingStateManager.js.map +1 -1
  79. package/dist/public.d.ts +1 -1
  80. package/dist/scheduleManager.js +4 -0
  81. package/dist/scheduleManager.js.map +1 -1
  82. package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
  83. package/dist/summary/summarizerNode/summarizerNodeUtils.js +2 -0
  84. package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
  85. package/dist/summary/summaryFormat.d.ts.map +1 -1
  86. package/dist/summary/summaryFormat.js +4 -1
  87. package/dist/summary/summaryFormat.js.map +1 -1
  88. package/internal.d.ts +1 -1
  89. package/legacy.d.ts +1 -1
  90. package/lib/blobManager/blobManager.d.ts +10 -0
  91. package/lib/blobManager/blobManager.d.ts.map +1 -1
  92. package/lib/blobManager/blobManager.js +19 -0
  93. package/lib/blobManager/blobManager.js.map +1 -1
  94. package/lib/channelCollection.d.ts +1 -1
  95. package/lib/channelCollection.d.ts.map +1 -1
  96. package/lib/channelCollection.js +40 -8
  97. package/lib/channelCollection.js.map +1 -1
  98. package/lib/containerRuntime.d.ts +14 -5
  99. package/lib/containerRuntime.d.ts.map +1 -1
  100. package/lib/containerRuntime.js +152 -100
  101. package/lib/containerRuntime.js.map +1 -1
  102. package/lib/dataStoreContext.d.ts +4 -0
  103. package/lib/dataStoreContext.d.ts.map +1 -1
  104. package/lib/dataStoreContext.js +10 -4
  105. package/lib/dataStoreContext.js.map +1 -1
  106. package/lib/gc/garbageCollection.d.ts +1 -1
  107. package/lib/gc/garbageCollection.d.ts.map +1 -1
  108. package/lib/gc/garbageCollection.js +14 -8
  109. package/lib/gc/garbageCollection.js.map +1 -1
  110. package/lib/gc/gcDefinitions.d.ts +4 -2
  111. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  112. package/lib/gc/gcDefinitions.js.map +1 -1
  113. package/lib/gc/gcHelpers.d.ts.map +1 -1
  114. package/lib/gc/gcHelpers.js +12 -0
  115. package/lib/gc/gcHelpers.js.map +1 -1
  116. package/lib/gc/gcTelemetry.d.ts +3 -2
  117. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  118. package/lib/gc/gcTelemetry.js +6 -6
  119. package/lib/gc/gcTelemetry.js.map +1 -1
  120. package/lib/legacy.d.ts +1 -1
  121. package/lib/metadata.d.ts +7 -1
  122. package/lib/metadata.d.ts.map +1 -1
  123. package/lib/metadata.js +4 -1
  124. package/lib/metadata.js.map +1 -1
  125. package/lib/opLifecycle/batchManager.d.ts +8 -1
  126. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  127. package/lib/opLifecycle/batchManager.js +35 -15
  128. package/lib/opLifecycle/batchManager.js.map +1 -1
  129. package/lib/opLifecycle/definitions.d.ts +1 -1
  130. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  131. package/lib/opLifecycle/definitions.js.map +1 -1
  132. package/lib/opLifecycle/index.d.ts +2 -2
  133. package/lib/opLifecycle/index.d.ts.map +1 -1
  134. package/lib/opLifecycle/index.js +2 -2
  135. package/lib/opLifecycle/index.js.map +1 -1
  136. package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
  137. package/lib/opLifecycle/opCompressor.js +12 -8
  138. package/lib/opLifecycle/opCompressor.js.map +1 -1
  139. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  140. package/lib/opLifecycle/opGroupingManager.js +14 -11
  141. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  142. package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
  143. package/lib/opLifecycle/opSplitter.js +11 -6
  144. package/lib/opLifecycle/opSplitter.js.map +1 -1
  145. package/lib/opLifecycle/outbox.d.ts +22 -6
  146. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  147. package/lib/opLifecycle/outbox.js +44 -22
  148. package/lib/opLifecycle/outbox.js.map +1 -1
  149. package/lib/opLifecycle/remoteMessageProcessor.d.ts +10 -8
  150. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  151. package/lib/opLifecycle/remoteMessageProcessor.js +37 -14
  152. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  153. package/lib/packageVersion.d.ts +1 -1
  154. package/lib/packageVersion.d.ts.map +1 -1
  155. package/lib/packageVersion.js +1 -1
  156. package/lib/packageVersion.js.map +1 -1
  157. package/lib/pendingStateManager.d.ts +37 -13
  158. package/lib/pendingStateManager.d.ts.map +1 -1
  159. package/lib/pendingStateManager.js +95 -45
  160. package/lib/pendingStateManager.js.map +1 -1
  161. package/lib/public.d.ts +1 -1
  162. package/lib/scheduleManager.js +4 -0
  163. package/lib/scheduleManager.js.map +1 -1
  164. package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
  165. package/lib/summary/summarizerNode/summarizerNodeUtils.js +2 -0
  166. package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
  167. package/lib/summary/summaryFormat.d.ts.map +1 -1
  168. package/lib/summary/summaryFormat.js +4 -1
  169. package/lib/summary/summaryFormat.js.map +1 -1
  170. package/package.json +46 -31
  171. package/src/blobManager/blobManager.ts +19 -0
  172. package/src/channelCollection.ts +48 -11
  173. package/src/containerRuntime.ts +203 -133
  174. package/src/dataStoreContext.ts +22 -4
  175. package/src/gc/garbageCollection.ts +15 -10
  176. package/src/gc/gcDefinitions.ts +7 -2
  177. package/src/gc/gcHelpers.ts +18 -6
  178. package/src/gc/gcTelemetry.ts +20 -8
  179. package/src/metadata.ts +11 -1
  180. package/src/opLifecycle/README.md +0 -8
  181. package/src/opLifecycle/batchManager.ts +49 -16
  182. package/src/opLifecycle/definitions.ts +1 -1
  183. package/src/opLifecycle/index.ts +13 -2
  184. package/src/opLifecycle/opCompressor.ts +12 -8
  185. package/src/opLifecycle/opGroupingManager.ts +14 -11
  186. package/src/opLifecycle/opSplitter.ts +10 -6
  187. package/src/opLifecycle/outbox.ts +64 -26
  188. package/src/opLifecycle/remoteMessageProcessor.ts +56 -17
  189. package/src/packageVersion.ts +1 -1
  190. package/src/pendingStateManager.ts +173 -74
  191. package/src/scheduleManager.ts +6 -2
  192. package/src/summary/README.md +81 -0
  193. package/src/summary/summarizerNode/summarizerNodeUtils.ts +3 -1
  194. package/src/summary/summaryFormat.ts +3 -1
  195. package/src/summary/summaryFormats.md +69 -8
  196. package/tsconfig.json +0 -1
  197. package/src/summary/images/appTree.png +0 -0
  198. package/src/summary/images/protocolAndAppTree.png +0 -0
  199. package/src/summary/images/summaryTree.png +0 -0
@@ -403,20 +403,21 @@ export class GarbageCollector implements IGarbageCollector {
403
403
  this.mc.logger,
404
404
  {
405
405
  eventName: "InitializeOrUpdateGCState",
406
- details: { initialized, unrefNodeCount: this.unreferencedNodesState.size },
407
406
  },
408
- async () => {
407
+ async (event) => {
409
408
  // If the GC state hasn't been initialized yet, initialize it and return.
410
409
  if (!initialized) {
411
410
  await this.initializeGCStateFromBaseSnapshotP;
412
- return;
413
- }
414
-
415
- // If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
416
- // reference timestamp.
417
- for (const [, nodeStateTracker] of this.unreferencedNodesState) {
418
- nodeStateTracker.updateTracking(currentReferenceTimestampMs);
411
+ } else {
412
+ // If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
413
+ // reference timestamp.
414
+ for (const [, nodeStateTracker] of this.unreferencedNodesState) {
415
+ nodeStateTracker.updateTracking(currentReferenceTimestampMs);
416
+ }
419
417
  }
418
+ event.end({
419
+ details: { initialized, unrefNodeCount: this.unreferencedNodesState.size },
420
+ });
420
421
  },
421
422
  );
422
423
  }
@@ -819,7 +820,9 @@ export class GarbageCollector implements IGarbageCollector {
819
820
  if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
820
821
  gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
821
822
  } else {
822
- gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
823
+ // Non null asserting here because we are checking if it is undefined above.
824
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
825
+ gcDataSuperSet.gcNodes[sourceNodeId]!.push(...outboundRoutes);
823
826
  }
824
827
  newOutboundRoutesSinceLastRun.push(...outboundRoutes);
825
828
  },
@@ -1001,6 +1004,7 @@ export class GarbageCollector implements IGarbageCollector {
1001
1004
  packagePath,
1002
1005
  request,
1003
1006
  headerData,
1007
+ additionalProps,
1004
1008
  }: IGCNodeUpdatedProps) {
1005
1009
  // If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
1006
1010
  // logging as nothing interesting would have happened worth logging.
@@ -1027,6 +1031,7 @@ export class GarbageCollector implements IGarbageCollector {
1027
1031
  headers: headerData,
1028
1032
  requestUrl: request?.url,
1029
1033
  requestHeaders: JSON.stringify(request?.headers),
1034
+ additionalProps,
1030
1035
  });
1031
1036
 
1032
1037
  // Any time we log a Tombstone Loaded error (via Telemetry Tracker),
@@ -13,7 +13,10 @@ import {
13
13
  ISummarizeResult,
14
14
  } from "@fluidframework/runtime-definitions/internal";
15
15
  import { ReadAndParseBlob } from "@fluidframework/runtime-utils/internal";
16
- import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/internal";
16
+ import {
17
+ ITelemetryLoggerExt,
18
+ type ITelemetryPropertiesExt,
19
+ } from "@fluidframework/telemetry-utils/internal";
17
20
 
18
21
  import { RuntimeHeaderData } from "../containerRuntime.js";
19
22
  import { ContainerRuntimeGCMessage } from "../messageTypes.js";
@@ -397,7 +400,7 @@ export interface IGCNodeUpdatedProps {
397
400
  /** Type and path of the updated node */
398
401
  node: { type: (typeof GCNodeType)["DataStore" | "Blob"]; path: string };
399
402
  /** Whether the node (or a subpath) was loaded or changed. */
400
- reason: "Loaded" | "Changed";
403
+ reason: "Loaded" | "Changed" | "Realized";
401
404
  /**
402
405
  * The op-based timestamp when the node changed. If the update is from receiving an op, this should
403
406
  * be the timestamp of the op. If not, this should be the timestamp of the last op processed.
@@ -409,6 +412,8 @@ export interface IGCNodeUpdatedProps {
409
412
  request?: IRequest;
410
413
  /** If the node was loaded via request path, the header data. May be modified from the original request */
411
414
  headerData?: RuntimeHeaderData;
415
+ /** Any other properties to be logged. */
416
+ additionalProps?: ITelemetryPropertiesExt;
412
417
  }
413
418
 
414
419
  /** Parameters necessary for creating a GarbageCollector. */
@@ -166,7 +166,9 @@ export function concatGarbageCollectionData(
166
166
  if (combinedGCData.gcNodes[id] === undefined) {
167
167
  combinedGCData.gcNodes[id] = Array.from(routes);
168
168
  } else {
169
- const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]];
169
+ // Non null asserting here since we are checking if combinedGCData.gcNodes[id] is not undefined above.
170
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
171
+ const combinedRoutes = [...routes, ...combinedGCData.gcNodes[id]!];
170
172
  combinedGCData.gcNodes[id] = [...new Set(combinedRoutes)];
171
173
  }
172
174
  }
@@ -187,13 +189,17 @@ export async function getGCDataFromSnapshot(
187
189
  for (const key of Object.keys(gcSnapshotTree.blobs)) {
188
190
  // Update deleted nodes blob.
189
191
  if (key === gcDeletedBlobKey) {
190
- deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
192
+ // Non null asserting here, we can change this to Object.entries later
193
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
194
+ deletedNodes = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
191
195
  continue;
192
196
  }
193
197
 
194
198
  // Update tombstone blob.
195
199
  if (key === gcTombstoneBlobKey) {
196
- tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]);
200
+ // Non null asserting here, we can change this to Object.entries later
201
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
202
+ tombstones = await readAndParseBlob<string[]>(gcSnapshotTree.blobs[key]!);
197
203
  continue;
198
204
  }
199
205
 
@@ -235,7 +241,9 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
235
241
  }
236
242
 
237
243
  assert(id.startsWith("/"), 0x5d9 /* node id should always be an absolute route */);
238
- const childId = id.split("/")[1];
244
+ // TODO Why are we non null asserting here
245
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
246
+ const childId = id.split("/")[1]!;
239
247
  let childGCNodeId = id.slice(childId.length + 1);
240
248
  // GC node id always begins with "/". Handle the special case where a child's id in the parent's GC nodes is
241
249
  // of format `/root`. In this case, the childId is root and childGCNodeId is "". Make childGCNodeId = "/".
@@ -264,7 +272,9 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
264
272
  const usedRoutes = gcDetails.usedRoutes.filter((route) => route !== "" && route !== "/");
265
273
  for (const route of usedRoutes) {
266
274
  assert(route.startsWith("/"), 0x5db /* Used route should always be an absolute route */);
267
- const childId = route.split("/")[1];
275
+ // TODO Why are we non null asserting here
276
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
277
+ const childId = route.split("/")[1]!;
268
278
  const childUsedRoute = route.slice(childId.length + 1);
269
279
 
270
280
  const childGCDetails = childGCDetailsMap.get(childId);
@@ -290,7 +300,9 @@ function trimLeadingAndTrailingSlashes(str: string) {
290
300
 
291
301
  /** Reformats a request URL to match expected format for a GC node path */
292
302
  export function urlToGCNodePath(url: string): string {
293
- return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
303
+ // TODO Why are we non null asserting here
304
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
305
+ return `/${trimLeadingAndTrailingSlashes(url.split("?")[0]!)}`;
294
306
  }
295
307
 
296
308
  /**
@@ -11,6 +11,7 @@ import {
11
11
  generateStack,
12
12
  tagCodeArtifacts,
13
13
  type ITelemetryGenericEventExt,
14
+ type ITelemetryPropertiesExt,
14
15
  } from "@fluidframework/telemetry-utils/internal";
15
16
 
16
17
  import { RuntimeHeaderData } from "../containerRuntime.js";
@@ -28,7 +29,7 @@ import {
28
29
  import { getGCVersionInEffect } from "./gcHelpers.js";
29
30
  import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker.js";
30
31
 
31
- type NodeUsageType = "Changed" | "Loaded" | "Revived";
32
+ type NodeUsageType = "Changed" | "Loaded" | "Revived" | "Realized";
32
33
 
33
34
  /** Properties that are common to IUnreferencedEventProps and INodeUsageProps */
34
35
  interface ICommonProps {
@@ -37,6 +38,7 @@ interface ICommonProps {
37
38
  isTombstoned: boolean;
38
39
  lastSummaryTime?: number;
39
40
  headers?: RuntimeHeaderData;
41
+ additionalProps?: ITelemetryPropertiesExt;
40
42
  }
41
43
 
42
44
  /** The event that is logged when unreferenced node is used after a certain time. */
@@ -240,7 +242,8 @@ export class GCTelemetryTracker {
240
242
  // Events generated:
241
243
  // InactiveObject_Loaded, SweepReadyObject_Loaded
242
244
  if (usageType === "Loaded") {
243
- const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
245
+ const { id, fromId, headers, gcConfigs, additionalProps, ...detailedProps } =
246
+ unrefEventProps;
244
247
  const event = {
245
248
  eventName: `${state}Object_${usageType}`,
246
249
  ...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
@@ -248,7 +251,7 @@ export class GCTelemetryTracker {
248
251
  id,
249
252
  fromId,
250
253
  headers: { ...headers },
251
- details: detailedProps,
254
+ details: { ...detailedProps, ...additionalProps },
252
255
  gcConfigs,
253
256
  };
254
257
 
@@ -272,7 +275,8 @@ export class GCTelemetryTracker {
272
275
  // GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
273
276
  // GC_Tombstone_SubDataStore_Requested, GC_Tombstone_SubDataStore_Changed, GC_Tombstone_SubDataStore_Revived
274
277
  // GC_Tombstone_Blob_Requested, GC_Tombstone_Blob_Changed, GC_Tombstone_Blob_Revived
275
- const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
278
+ const { id, fromId, headers, gcConfigs, additionalProps, ...detailedProps } =
279
+ unrefEventProps;
276
280
  const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
277
281
  const event = {
278
282
  eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
@@ -281,7 +285,7 @@ export class GCTelemetryTracker {
281
285
  id,
282
286
  fromId,
283
287
  headers: { ...headers },
284
- details: detailedProps, // Also includes some properties from INodeUsageProps type
288
+ details: { ...detailedProps, ...additionalProps }, // Also includes some properties from INodeUsageProps type
285
289
  gcConfigs,
286
290
  tombstoneFlags: {
287
291
  DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
@@ -370,8 +374,16 @@ export class GCTelemetryTracker {
370
374
  // InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
371
375
  // SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
372
376
  for (const eventProps of this.pendingEventsQueue) {
373
- const { usageType, state, id, fromId, headers, gcConfigs, ...detailedProps } =
374
- eventProps;
377
+ const {
378
+ usageType,
379
+ state,
380
+ id,
381
+ fromId,
382
+ headers,
383
+ gcConfigs,
384
+ additionalProps,
385
+ ...detailedProps
386
+ } = eventProps;
375
387
  /**
376
388
  * Revived event is logged only if the node is active. If the node is not active, the reference to it was
377
389
  * from another unreferenced node and this scenario is not interesting to log.
@@ -391,7 +403,7 @@ export class GCTelemetryTracker {
391
403
  id,
392
404
  fromId,
393
405
  headers: { ...headers },
394
- details: detailedProps,
406
+ details: { ...detailedProps, ...additionalProps },
395
407
  gcConfigs,
396
408
  ...tagCodeArtifacts({
397
409
  pkg: pkg?.join("/"),
package/src/metadata.ts CHANGED
@@ -3,11 +3,21 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
+ import type { BatchId } from "./opLifecycle/index.js";
7
+
8
+ /** Syntactic sugar for casting */
9
+ export function asBatchMetadata(metadata: unknown): IBatchMetadata | undefined {
10
+ return metadata as IBatchMetadata | undefined;
11
+ }
12
+
6
13
  /**
7
- * Batching makes assumptions about what might be on the metadata. This interface codifies those assumptions, but does not validate them.
14
+ * Properties put on the op metadata object for batch tracking
8
15
  */
9
16
  export interface IBatchMetadata {
17
+ /** Set on first/last messages of a multi-message batch, to true/false respectively */
10
18
  batch?: boolean;
19
+ /** Maybe set on first message of a batch, to the batchId generated when resubmitting (set/fixed on first resubmit) */
20
+ batchId?: BatchId;
11
21
  }
12
22
 
13
23
  /**
@@ -10,7 +10,6 @@
10
10
  - [Grouped batching](#grouped-batching)
11
11
  - [Changes in op semantics](#changes-in-op-semantics)
12
12
  - [Chunking for compression](#chunking-for-compression)
13
- - [Disabling in case of emergency](#disabling-in-case-of-emergency)
14
13
  - [Configuration](#configuration)
15
14
  - [Note about performance and latency](#note-about-performance-and-latency)
16
15
  - [How it works](#how-it-works)
@@ -103,13 +102,6 @@ This config would govern chunking compressed batches only. We will not be enabli
103
102
 
104
103
  Chunking is relevant for both `FlushMode.TurnBased` and `FlushMode.Immediate` as it only targets the contents of the ops and not the number of ops in a batch. Chunking is opaque to the server and implementations of the Fluid protocol do not need to alter their behavior to support this client feature.
105
104
 
106
- ## Disabling in case of emergency
107
-
108
- Compression and Chunking configuration can be overridden via feature gates to force-disable them:
109
-
110
- - `Fluid.ContainerRuntime.CompressionDisabled` - if set to true, will disable compression (this has a side effect of also disabling chunking, as chunking is invoked only for compressed payloads).
111
- - `Fluid.ContainerRuntime.CompressionChunkingDisabled` - if set to true, will disable chunking for compression.
112
-
113
105
  ## Configuration
114
106
 
115
107
  These features are configured via `IContainerRuntimeOptions`, passed to the `ContainerRuntime.loadRuntime` function.
@@ -3,7 +3,10 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
+ import { assert } from "@fluidframework/core-utils/internal";
7
+
6
8
  import { ICompressionRuntimeOptions } from "../containerRuntime.js";
9
+ import { type IBatchMetadata } from "../metadata.js";
7
10
 
8
11
  import { BatchMessage, IBatch, IBatchCheckpoint } from "./definitions.js";
9
12
 
@@ -22,6 +25,14 @@ export interface BatchSequenceNumbers {
22
25
  clientSequenceNumber?: number;
23
26
  }
24
27
 
28
+ /** Type alias for the batchId stored in batch metadata */
29
+ export type BatchId = string;
30
+
31
+ /** Compose original client ID and client sequence number into BatchId to stamp on the message during reconnect */
32
+ export function generateBatchId(originalClientId: string, batchStartCsn: number): BatchId {
33
+ return `${originalClientId}_[${batchStartCsn}]`;
34
+ }
35
+
25
36
  /**
26
37
  * Estimated size of the stringification overhead for an op accumulated
27
38
  * from runtime to loader to the service.
@@ -53,7 +64,9 @@ export class BatchManager {
53
64
  private get referenceSequenceNumber(): number | undefined {
54
65
  return this.pendingBatch.length === 0
55
66
  ? undefined
56
- : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
67
+ : // Non null asserting here since we are checking the length above
68
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
69
+ this.pendingBatch[this.pendingBatch.length - 1]!.referenceSequenceNumber;
57
70
  }
58
71
 
59
72
  /**
@@ -97,9 +110,12 @@ export class BatchManager {
97
110
  return this.pendingBatch.length === 0;
98
111
  }
99
112
 
100
- public popBatch(): IBatch {
113
+ /**
114
+ * Gets the pending batch and clears state for the next batch.
115
+ */
116
+ public popBatch(batchId?: BatchId): IBatch {
101
117
  const batch: IBatch = {
102
- content: this.pendingBatch,
118
+ messages: this.pendingBatch,
103
119
  contentSizeInBytes: this.batchContentSize,
104
120
  referenceSequenceNumber: this.referenceSequenceNumber,
105
121
  hasReentrantOps: this.hasReentrantOps,
@@ -110,7 +126,7 @@ export class BatchManager {
110
126
  this.clientSequenceNumber = undefined;
111
127
  this.hasReentrantOps = false;
112
128
 
113
- return addBatchMetadata(batch);
129
+ return addBatchMetadata(batch, batchId);
114
130
  }
115
131
 
116
132
  /**
@@ -122,7 +138,9 @@ export class BatchManager {
122
138
  rollback: (process: (message: BatchMessage) => void) => {
123
139
  for (let i = this.pendingBatch.length; i > startPoint; ) {
124
140
  i--;
125
- const message = this.pendingBatch[i];
141
+ // Non null asserting here since we are iterating though pendingBatch
142
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
143
+ const message = this.pendingBatch[i]!;
126
144
  this.batchContentSize -= message.contents?.length ?? 0;
127
145
  process(message);
128
146
  }
@@ -133,16 +151,31 @@ export class BatchManager {
133
151
  }
134
152
  }
135
153
 
136
- const addBatchMetadata = (batch: IBatch): IBatch => {
137
- if (batch.content.length > 1) {
138
- batch.content[0].metadata = {
139
- ...batch.content[0].metadata,
140
- batch: true,
141
- };
142
- batch.content[batch.content.length - 1].metadata = {
143
- ...batch.content[batch.content.length - 1].metadata,
144
- batch: false,
145
- };
154
+ const addBatchMetadata = (batch: IBatch, batchId?: BatchId): IBatch => {
155
+ const batchEnd = batch.messages.length - 1;
156
+
157
+ const firstMsg = batch.messages[0];
158
+ const lastMsg = batch.messages[batchEnd];
159
+ assert(
160
+ firstMsg !== undefined && lastMsg !== undefined,
161
+ 0x9d1 /* expected non-empty batch */,
162
+ );
163
+
164
+ const firstMetadata: Partial<IBatchMetadata> = firstMsg.metadata ?? {};
165
+ const lastMetadata: Partial<IBatchMetadata> = lastMsg.metadata ?? {};
166
+
167
+ // Multi-message batches: mark the first and last messages with the "batch" flag indicating batch start/end
168
+ if (batch.messages.length > 1) {
169
+ firstMetadata.batch = true;
170
+ lastMetadata.batch = false;
171
+ firstMsg.metadata = firstMetadata;
172
+ lastMsg.metadata = lastMetadata;
173
+ }
174
+
175
+ // If batchId is provided (e.g. in case of resubmit): stamp it on the first message
176
+ if (batchId !== undefined) {
177
+ firstMetadata.batchId = batchId;
178
+ firstMsg.metadata = firstMetadata;
146
179
  }
147
180
 
148
181
  return batch;
@@ -157,7 +190,7 @@ const addBatchMetadata = (batch: IBatch): IBatch => {
157
190
  * @returns An estimate of the payload size in bytes which will be produced when the batch is sent over the wire
158
191
  */
159
192
  export const estimateSocketSize = (batch: IBatch): number => {
160
- return batch.contentSizeInBytes + opOverhead * batch.content.length;
193
+ return batch.contentSizeInBytes + opOverhead * batch.messages.length;
161
194
  };
162
195
 
163
196
  export const sequenceNumbersMatch = (
@@ -28,7 +28,7 @@ export interface IBatch<TMessages extends BatchMessage[] = BatchMessage[]> {
28
28
  /**
29
29
  * All the messages in the batch
30
30
  */
31
- readonly content: TMessages;
31
+ readonly messages: TMessages;
32
32
  /**
33
33
  * The reference sequence number for the batch
34
34
  */
@@ -3,13 +3,24 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
- export { BatchManager, estimateSocketSize, BatchSequenceNumbers } from "./batchManager.js";
6
+ export {
7
+ BatchId,
8
+ BatchManager,
9
+ BatchSequenceNumbers,
10
+ estimateSocketSize,
11
+ generateBatchId,
12
+ IBatchManagerOptions,
13
+ } from "./batchManager.js";
7
14
  export { BatchMessage, IBatch, IBatchCheckpoint, IChunkedOp } from "./definitions.js";
8
15
  export { Outbox, getLongStack } from "./outbox.js";
9
16
  export { OpCompressor } from "./opCompressor.js";
10
17
  export { OpDecompressor } from "./opDecompressor.js";
11
18
  export { OpSplitter, splitOp, isChunkedMessage } from "./opSplitter.js";
12
- export { RemoteMessageProcessor, unpackRuntimeMessage } from "./remoteMessageProcessor.js";
19
+ export {
20
+ ensureContentsDeserialized,
21
+ RemoteMessageProcessor,
22
+ unpackRuntimeMessage,
23
+ } from "./remoteMessageProcessor.js";
13
24
  export {
14
25
  OpGroupingManager,
15
26
  OpGroupingManagerConfig,
@@ -35,7 +35,7 @@ export class OpCompressor {
35
35
  */
36
36
  public compressBatch(batch: IBatch): IBatch {
37
37
  assert(
38
- batch.contentSizeInBytes > 0 && batch.content.length > 0,
38
+ batch.contentSizeInBytes > 0 && batch.messages.length > 0,
39
39
  0x5a4 /* Batch should not be empty */,
40
40
  );
41
41
 
@@ -47,14 +47,18 @@ export class OpCompressor {
47
47
 
48
48
  const messages: BatchMessage[] = [];
49
49
  messages.push({
50
- ...batch.content[0],
50
+ // Non null asserting here because of the length assert above
51
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
52
+ ...batch.messages[0]!,
51
53
  contents: JSON.stringify({ packedContents: compressedContent }),
52
- metadata: batch.content[0].metadata,
54
+ // Non null asserting here because of the length assert above
55
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
56
+ metadata: batch.messages[0]!.metadata,
53
57
  compression: CompressionAlgorithms.lz4,
54
58
  });
55
59
 
56
60
  // Add empty placeholder messages to reserve the sequence numbers
57
- for (const message of batch.content.slice(1)) {
61
+ for (const message of batch.messages.slice(1)) {
58
62
  messages.push({
59
63
  localOpMetadata: message.localOpMetadata,
60
64
  metadata: message.metadata,
@@ -64,7 +68,7 @@ export class OpCompressor {
64
68
 
65
69
  const compressedBatch: IBatch = {
66
70
  contentSizeInBytes: compressedContent.length,
67
- content: messages,
71
+ messages,
68
72
  referenceSequenceNumber: batch.referenceSequenceNumber,
69
73
  };
70
74
 
@@ -74,7 +78,7 @@ export class OpCompressor {
74
78
  duration,
75
79
  sizeBeforeCompression: batch.contentSizeInBytes,
76
80
  sizeAfterCompression: compressedBatch.contentSizeInBytes,
77
- opCount: compressedBatch.content.length,
81
+ opCount: compressedBatch.messages.length,
78
82
  socketSize: estimateSocketSize(compressedBatch),
79
83
  });
80
84
  }
@@ -88,7 +92,7 @@ export class OpCompressor {
88
92
  private serializeBatchContents(batch: IBatch): string {
89
93
  try {
90
94
  // Yields a valid JSON array, since each message.contents is already serialized to JSON
91
- return `[${batch.content.map((message) => message.contents).join(",")}]`;
95
+ return `[${batch.messages.map(({ contents }) => contents).join(",")}]`;
92
96
  } catch (e: any) {
93
97
  if (e.message === "Invalid string length") {
94
98
  // This is how JSON.stringify signals that
@@ -98,7 +102,7 @@ export class OpCompressor {
98
102
  {
99
103
  eventName: "BatchTooLarge",
100
104
  size: batch.contentSizeInBytes,
101
- length: batch.content.length,
105
+ length: batch.messages.length,
102
106
  },
103
107
  error,
104
108
  );
@@ -59,27 +59,28 @@ export class OpGroupingManager {
59
59
  public groupBatch(batch: IBatch): IBatch<[BatchMessage]> {
60
60
  assert(this.shouldGroup(batch), 0x946 /* cannot group the provided batch */);
61
61
 
62
- if (batch.content.length >= 1000) {
62
+ if (batch.messages.length >= 1000) {
63
63
  this.logger.sendTelemetryEvent({
64
64
  eventName: "GroupLargeBatch",
65
- length: batch.content.length,
65
+ length: batch.messages.length,
66
66
  threshold: this.config.opCountThreshold,
67
67
  reentrant: batch.hasReentrantOps,
68
- referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
68
+ // Non null asserting here because of the length check above
69
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
70
+ referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
69
71
  });
70
72
  }
71
73
 
72
- for (const message of batch.content) {
74
+ for (const message of batch.messages) {
73
75
  if (message.metadata) {
74
- const keys = Object.keys(message.metadata);
75
- assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
76
- assert(keys.length === 0 || keys[0] === "batch", 0x5de /* unexpected op metadata */);
76
+ const { batch: _batch, batchId, ...rest } = message.metadata;
77
+ assert(Object.keys(rest).length === 0, 0x5dd /* cannot group ops with metadata */);
77
78
  }
78
79
  }
79
80
 
80
81
  const serializedContent = JSON.stringify({
81
82
  type: OpGroupingManager.groupedBatchOp,
82
- contents: batch.content.map<IGroupedMessage>((message) => ({
83
+ contents: batch.messages.map<IGroupedMessage>((message) => ({
83
84
  contents: message.contents === undefined ? undefined : JSON.parse(message.contents),
84
85
  metadata: message.metadata,
85
86
  compression: message.compression,
@@ -88,10 +89,12 @@ export class OpGroupingManager {
88
89
 
89
90
  const groupedBatch: IBatch<[BatchMessage]> = {
90
91
  ...batch,
91
- content: [
92
+ messages: [
92
93
  {
93
94
  metadata: undefined,
94
- referenceSequenceNumber: batch.content[0].referenceSequenceNumber,
95
+ // TODO why are we non null asserting here?
96
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
97
+ referenceSequenceNumber: batch.messages[0]!.referenceSequenceNumber,
95
98
  contents: serializedContent,
96
99
  },
97
100
  ],
@@ -118,7 +121,7 @@ export class OpGroupingManager {
118
121
  // Grouped batching must be enabled
119
122
  this.config.groupedBatchingEnabled &&
120
123
  // The number of ops in the batch must surpass the configured threshold
121
- batch.content.length >= this.config.opCountThreshold &&
124
+ batch.messages.length >= this.config.opCountThreshold &&
122
125
  // Support for reentrant batches must be explicitly enabled
123
126
  (this.config.reentrantBatchGroupingEnabled || batch.hasReentrantOps !== true)
124
127
  );
@@ -121,7 +121,7 @@ export class OpSplitter {
121
121
  public splitFirstBatchMessage(batch: IBatch): IBatch {
122
122
  assert(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
123
123
  assert(
124
- batch.contentSizeInBytes > 0 && batch.content.length > 0,
124
+ batch.contentSizeInBytes > 0 && batch.messages.length > 0,
125
125
  0x514 /* Batch needs to be non-empty */,
126
126
  );
127
127
  assert(
@@ -134,13 +134,15 @@ export class OpSplitter {
134
134
  0x516 /* Chunk size needs to be smaller than the max batch size */,
135
135
  );
136
136
 
137
- const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
137
+ // Non null asserting here because of the length check above
138
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
139
+ const firstMessage = batch.messages[0]!; // we expect this to be the large compressed op, which needs to be split
138
140
  assert(
139
141
  (firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes,
140
142
  0x518 /* First message in the batch needs to be chunkable */,
141
143
  );
142
144
 
143
- const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
145
+ const restOfMessages = batch.messages.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
144
146
  const socketSize = estimateSocketSize(batch);
145
147
  const chunks = splitOp(
146
148
  firstMessage,
@@ -163,7 +165,9 @@ export class OpSplitter {
163
165
  // The last chunk will be part of the new batch and needs to
164
166
  // preserve the batch metadata of the original batch
165
167
  const lastChunk = chunkToBatchMessage(
166
- chunks[chunks.length - 1],
168
+ // Non null asserting here because of the length assert above
169
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
170
+ chunks[chunks.length - 1]!,
167
171
  batch.referenceSequenceNumber,
168
172
  { batch: firstMessage.metadata?.batch },
169
173
  );
@@ -171,7 +175,7 @@ export class OpSplitter {
171
175
  this.logger.sendPerformanceEvent({
172
176
  // Used to be "Chunked compressed batch"
173
177
  eventName: "CompressedChunkedBatch",
174
- length: batch.content.length,
178
+ length: batch.messages.length,
175
179
  sizeInBytes: batch.contentSizeInBytes,
176
180
  chunks: chunks.length,
177
181
  chunkSizeInBytes: this.chunkSizeInBytes,
@@ -179,7 +183,7 @@ export class OpSplitter {
179
183
  });
180
184
 
181
185
  return {
182
- content: [lastChunk, ...restOfMessages],
186
+ messages: [lastChunk, ...restOfMessages],
183
187
  contentSizeInBytes: lastChunk.contents?.length ?? 0,
184
188
  referenceSequenceNumber: batch.referenceSequenceNumber,
185
189
  };