@fluidframework/container-runtime 2.41.0-338401 → 2.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/CHANGELOG.md +8 -0
  2. package/container-runtime.test-files.tar +0 -0
  3. package/dist/channelCollection.d.ts +1 -1
  4. package/dist/channelCollection.d.ts.map +1 -1
  5. package/dist/channelCollection.js +4 -4
  6. package/dist/channelCollection.js.map +1 -1
  7. package/dist/compatUtils.d.ts +22 -1
  8. package/dist/compatUtils.d.ts.map +1 -1
  9. package/dist/compatUtils.js +109 -7
  10. package/dist/compatUtils.js.map +1 -1
  11. package/dist/containerRuntime.d.ts +67 -28
  12. package/dist/containerRuntime.d.ts.map +1 -1
  13. package/dist/containerRuntime.js +332 -186
  14. package/dist/containerRuntime.js.map +1 -1
  15. package/dist/dataStore.d.ts.map +1 -1
  16. package/dist/dataStore.js +5 -0
  17. package/dist/dataStore.js.map +1 -1
  18. package/dist/gc/garbageCollection.d.ts.map +1 -1
  19. package/dist/gc/garbageCollection.js +2 -0
  20. package/dist/gc/garbageCollection.js.map +1 -1
  21. package/dist/gc/gcDefinitions.d.ts +1 -1
  22. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  23. package/dist/gc/gcDefinitions.js.map +1 -1
  24. package/dist/index.d.ts +1 -1
  25. package/dist/index.d.ts.map +1 -1
  26. package/dist/index.js.map +1 -1
  27. package/dist/messageTypes.d.ts +5 -4
  28. package/dist/messageTypes.d.ts.map +1 -1
  29. package/dist/messageTypes.js.map +1 -1
  30. package/dist/metadata.d.ts +1 -1
  31. package/dist/metadata.d.ts.map +1 -1
  32. package/dist/metadata.js.map +1 -1
  33. package/dist/opLifecycle/batchManager.d.ts +4 -0
  34. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  35. package/dist/opLifecycle/batchManager.js +7 -0
  36. package/dist/opLifecycle/batchManager.js.map +1 -1
  37. package/dist/opLifecycle/definitions.d.ts +6 -5
  38. package/dist/opLifecycle/definitions.d.ts.map +1 -1
  39. package/dist/opLifecycle/definitions.js.map +1 -1
  40. package/dist/opLifecycle/index.d.ts +1 -1
  41. package/dist/opLifecycle/index.d.ts.map +1 -1
  42. package/dist/opLifecycle/index.js.map +1 -1
  43. package/dist/opLifecycle/opGroupingManager.d.ts +9 -0
  44. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  45. package/dist/opLifecycle/opGroupingManager.js +6 -4
  46. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  47. package/dist/opLifecycle/opSerialization.d.ts +2 -1
  48. package/dist/opLifecycle/opSerialization.d.ts.map +1 -1
  49. package/dist/opLifecycle/opSerialization.js.map +1 -1
  50. package/dist/opLifecycle/outbox.d.ts +1 -0
  51. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  52. package/dist/opLifecycle/outbox.js +6 -1
  53. package/dist/opLifecycle/outbox.js.map +1 -1
  54. package/dist/packageVersion.d.ts +1 -1
  55. package/dist/packageVersion.d.ts.map +1 -1
  56. package/dist/packageVersion.js +1 -1
  57. package/dist/packageVersion.js.map +1 -1
  58. package/dist/pendingStateManager.d.ts +22 -5
  59. package/dist/pendingStateManager.d.ts.map +1 -1
  60. package/dist/pendingStateManager.js +34 -11
  61. package/dist/pendingStateManager.js.map +1 -1
  62. package/dist/runCounter.d.ts.map +1 -1
  63. package/dist/runCounter.js +1 -1
  64. package/dist/runCounter.js.map +1 -1
  65. package/dist/summary/documentSchema.d.ts +42 -18
  66. package/dist/summary/documentSchema.d.ts.map +1 -1
  67. package/dist/summary/documentSchema.js +62 -52
  68. package/dist/summary/documentSchema.js.map +1 -1
  69. package/dist/summary/index.d.ts +1 -1
  70. package/dist/summary/index.d.ts.map +1 -1
  71. package/dist/summary/index.js.map +1 -1
  72. package/lib/channelCollection.d.ts +1 -1
  73. package/lib/channelCollection.d.ts.map +1 -1
  74. package/lib/channelCollection.js +4 -4
  75. package/lib/channelCollection.js.map +1 -1
  76. package/lib/compatUtils.d.ts +22 -1
  77. package/lib/compatUtils.d.ts.map +1 -1
  78. package/lib/compatUtils.js +102 -3
  79. package/lib/compatUtils.js.map +1 -1
  80. package/lib/containerRuntime.d.ts +67 -28
  81. package/lib/containerRuntime.d.ts.map +1 -1
  82. package/lib/containerRuntime.js +333 -188
  83. package/lib/containerRuntime.js.map +1 -1
  84. package/lib/dataStore.d.ts.map +1 -1
  85. package/lib/dataStore.js +5 -0
  86. package/lib/dataStore.js.map +1 -1
  87. package/lib/gc/garbageCollection.d.ts.map +1 -1
  88. package/lib/gc/garbageCollection.js +2 -0
  89. package/lib/gc/garbageCollection.js.map +1 -1
  90. package/lib/gc/gcDefinitions.d.ts +1 -1
  91. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  92. package/lib/gc/gcDefinitions.js.map +1 -1
  93. package/lib/index.d.ts +1 -1
  94. package/lib/index.d.ts.map +1 -1
  95. package/lib/index.js.map +1 -1
  96. package/lib/messageTypes.d.ts +5 -4
  97. package/lib/messageTypes.d.ts.map +1 -1
  98. package/lib/messageTypes.js.map +1 -1
  99. package/lib/metadata.d.ts +1 -1
  100. package/lib/metadata.d.ts.map +1 -1
  101. package/lib/metadata.js.map +1 -1
  102. package/lib/opLifecycle/batchManager.d.ts +4 -0
  103. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  104. package/lib/opLifecycle/batchManager.js +7 -0
  105. package/lib/opLifecycle/batchManager.js.map +1 -1
  106. package/lib/opLifecycle/definitions.d.ts +6 -5
  107. package/lib/opLifecycle/definitions.d.ts.map +1 -1
  108. package/lib/opLifecycle/definitions.js.map +1 -1
  109. package/lib/opLifecycle/index.d.ts +1 -1
  110. package/lib/opLifecycle/index.d.ts.map +1 -1
  111. package/lib/opLifecycle/index.js.map +1 -1
  112. package/lib/opLifecycle/opGroupingManager.d.ts +9 -0
  113. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  114. package/lib/opLifecycle/opGroupingManager.js +6 -4
  115. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  116. package/lib/opLifecycle/opSerialization.d.ts +2 -1
  117. package/lib/opLifecycle/opSerialization.d.ts.map +1 -1
  118. package/lib/opLifecycle/opSerialization.js.map +1 -1
  119. package/lib/opLifecycle/outbox.d.ts +1 -0
  120. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  121. package/lib/opLifecycle/outbox.js +6 -1
  122. package/lib/opLifecycle/outbox.js.map +1 -1
  123. package/lib/packageVersion.d.ts +1 -1
  124. package/lib/packageVersion.d.ts.map +1 -1
  125. package/lib/packageVersion.js +1 -1
  126. package/lib/packageVersion.js.map +1 -1
  127. package/lib/pendingStateManager.d.ts +22 -5
  128. package/lib/pendingStateManager.d.ts.map +1 -1
  129. package/lib/pendingStateManager.js +34 -11
  130. package/lib/pendingStateManager.js.map +1 -1
  131. package/lib/runCounter.d.ts.map +1 -1
  132. package/lib/runCounter.js +1 -1
  133. package/lib/runCounter.js.map +1 -1
  134. package/lib/summary/documentSchema.d.ts +42 -18
  135. package/lib/summary/documentSchema.d.ts.map +1 -1
  136. package/lib/summary/documentSchema.js +62 -52
  137. package/lib/summary/documentSchema.js.map +1 -1
  138. package/lib/summary/index.d.ts +1 -1
  139. package/lib/summary/index.d.ts.map +1 -1
  140. package/lib/summary/index.js.map +1 -1
  141. package/package.json +19 -19
  142. package/src/channelCollection.ts +4 -4
  143. package/src/compatUtils.ts +145 -10
  144. package/src/containerRuntime.ts +472 -225
  145. package/src/dataStore.ts +7 -0
  146. package/src/gc/garbageCollection.ts +2 -0
  147. package/src/gc/gcDefinitions.ts +1 -1
  148. package/src/index.ts +2 -1
  149. package/src/messageTypes.ts +12 -5
  150. package/src/metadata.ts +1 -1
  151. package/src/opLifecycle/batchManager.ts +8 -0
  152. package/src/opLifecycle/definitions.ts +7 -3
  153. package/src/opLifecycle/index.ts +1 -0
  154. package/src/opLifecycle/opGroupingManager.ts +17 -4
  155. package/src/opLifecycle/opSerialization.ts +6 -1
  156. package/src/opLifecycle/outbox.ts +8 -1
  157. package/src/packageVersion.ts +1 -1
  158. package/src/pendingStateManager.ts +64 -20
  159. package/src/runCounter.ts +4 -1
  160. package/src/summary/documentSchema.ts +111 -86
  161. package/src/summary/index.ts +2 -1
@@ -7,7 +7,7 @@ import type {
7
7
  ILayerCompatDetails,
8
8
  IProvideLayerCompatDetails,
9
9
  } from "@fluid-internal/client-utils";
10
- import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
10
+ import { createEmitter, Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
11
11
  import type {
12
12
  IAudience,
13
13
  ISelf,
@@ -25,8 +25,17 @@ import type {
25
25
  } from "@fluidframework/container-definitions/internal";
26
26
  import { isIDeltaManagerFull } from "@fluidframework/container-definitions/internal";
27
27
  import type {
28
+ ContainerExtensionFactory,
29
+ ContainerExtensionId,
30
+ ExtensionHost,
31
+ ExtensionHostEvents,
32
+ ExtensionRuntimeProperties,
28
33
  IContainerRuntime,
29
34
  IContainerRuntimeEvents,
35
+ IContainerRuntimeInternal,
36
+ // eslint-disable-next-line import/no-deprecated
37
+ IContainerRuntimeWithResolveHandle_Deprecated,
38
+ OutboundExtensionMessage,
30
39
  } from "@fluidframework/container-runtime-definitions/internal";
31
40
  import type {
32
41
  FluidObject,
@@ -34,6 +43,7 @@ import type {
34
43
  IRequest,
35
44
  IResponse,
36
45
  ITelemetryBaseLogger,
46
+ Listenable,
37
47
  } from "@fluidframework/core-interfaces";
38
48
  import type {
39
49
  IErrorBase,
@@ -41,13 +51,18 @@ import type {
41
51
  IFluidHandleInternal,
42
52
  IProvideFluidHandleContext,
43
53
  ISignalEnvelope,
54
+ JsonDeserialized,
55
+ TypedMessage,
44
56
  } from "@fluidframework/core-interfaces/internal";
45
57
  import {
46
58
  assert,
47
59
  Deferred,
60
+ Lazy,
48
61
  LazyPromise,
49
62
  PromiseCache,
50
63
  delay,
64
+ fail,
65
+ unreachableCase,
51
66
  } from "@fluidframework/core-utils/internal";
52
67
  import type {
53
68
  IClientDetails,
@@ -168,6 +183,7 @@ import {
168
183
  isValidMinVersionForCollab,
169
184
  type RuntimeOptionsAffectingDocSchema,
170
185
  type MinimumVersionForCollab,
186
+ validateRuntimeOptions,
171
187
  } from "./compatUtils.js";
172
188
  import type { ICompressionRuntimeOptions } from "./compressionDefinitions.js";
173
189
  import { CompressionAlgorithms, disabledCompressionConfig } from "./compressionDefinitions.js";
@@ -194,7 +210,7 @@ import {
194
210
  import { InboundBatchAggregator } from "./inboundBatchAggregator.js";
195
211
  import {
196
212
  ContainerMessageType,
197
- type ContainerRuntimeDocumentSchemaMessage,
213
+ type OutboundContainerRuntimeDocumentSchemaMessage,
198
214
  ContainerRuntimeGCMessage,
199
215
  type ContainerRuntimeIdAllocationMessage,
200
216
  type InboundSequencedContainerRuntimeMessage,
@@ -232,7 +248,7 @@ import {
232
248
  import { SignalTelemetryManager } from "./signalTelemetryProcessing.js";
233
249
  // These types are imported as types here because they are present in summaryDelayLoadedModule, which is loaded dynamically when required.
234
250
  import type {
235
- IDocumentSchemaChangeMessage,
251
+ IDocumentSchemaChangeMessageIncoming,
236
252
  IDocumentSchemaCurrent,
237
253
  Summarizer,
238
254
  IDocumentSchemaFeatures,
@@ -284,6 +300,16 @@ import {
284
300
  } from "./summary/index.js";
285
301
  import { Throttler, formExponentialFn } from "./throttler.js";
286
302
 
303
+ /**
304
+ * A {@link ContainerExtension}'s factory function as stored in extension map.
305
+ */
306
+ // eslint-disable-next-line @typescript-eslint/no-explicit-any -- `any` required to allow typed factory to be assignable per ContainerExtension.processSignal
307
+ type ExtensionEntry = ContainerExtensionFactory<unknown, any, unknown[]> extends new (
308
+ ...args: any[]
309
+ ) => infer T
310
+ ? T
311
+ : never;
312
+
287
313
  /**
288
314
  * Creates an error object to be thrown / passed to Container's close fn in case of an unknown message type.
289
315
  * The parameters are typed to support compile-time enforcement of handling all known types/behaviors
@@ -674,6 +700,8 @@ export let getSingleUseLegacyLogCallback = (logger: ITelemetryLoggerExt, type: s
674
700
  };
675
701
  };
676
702
 
703
+ type UnsequencedSignalEnvelope = Omit<ISignalEnvelope, "clientBroadcastSignalSequenceNumber">;
704
+
677
705
  /**
678
706
  * This object holds the parameters necessary for the {@link loadContainerRuntime} function.
679
707
  * @legacy
@@ -748,6 +776,27 @@ export async function loadContainerRuntime(
748
776
 
749
777
  const defaultMaxConsecutiveReconnects = 7;
750
778
 
779
+ /**
780
+ * These are the ONLY message types that are allowed to be submitted while in staging mode
781
+ * (Does not apply to pre-StagingMode batches that are resubmitted, those are not considered to be staged)
782
+ */
783
+ function canStageMessageOfType(
784
+ type: LocalContainerRuntimeMessage["type"],
785
+ ): type is
786
+ | ContainerMessageType.FluidDataStoreOp
787
+ | ContainerMessageType.GC
788
+ | ContainerMessageType.DocumentSchemaChange {
789
+ return (
790
+ // These are user changes coming up from the runtime's DataStores
791
+ type === ContainerMessageType.FluidDataStoreOp ||
792
+ // GC ops are used to detect issues in the reference graph so all clients can repair their GC state.
793
+ // These can be submitted at any time, including while in Staging Mode.
794
+ type === ContainerMessageType.GC ||
795
+ // These are typically sent shortly after boot and will not be common in Staging Mode, but it's possible.
796
+ type === ContainerMessageType.DocumentSchemaChange
797
+ );
798
+ }
799
+
751
800
  /**
752
801
  * Represents the runtime of the container. Contains helper functions/state of the container.
753
802
  * It will define the store level mappings.
@@ -757,9 +806,11 @@ const defaultMaxConsecutiveReconnects = 7;
757
806
  export class ContainerRuntime
758
807
  extends TypedEventEmitter<IContainerRuntimeEvents>
759
808
  implements
760
- IContainerRuntime,
809
+ IContainerRuntimeInternal,
761
810
  // eslint-disable-next-line import/no-deprecated
762
811
  IContainerRuntimeBaseExperimental,
812
+ // eslint-disable-next-line import/no-deprecated
813
+ IContainerRuntimeWithResolveHandle_Deprecated,
763
814
  IRuntime,
764
815
  IGarbageCollectionRuntime,
765
816
  ISummarizerRuntime,
@@ -838,6 +889,10 @@ export class ContainerRuntime
838
889
  `Invalid minVersionForCollab: ${minVersionForCollab}. It must be an existing FF version (i.e. 2.22.1).`,
839
890
  );
840
891
  }
892
+ // We also validate that there is not a mismatch between `minVersionForCollab` and runtime options that
893
+ // were manually set.
894
+ validateRuntimeOptions(minVersionForCollab, runtimeOptions);
895
+
841
896
  const defaultsAffectingDocSchema = getMinVersionForCollabDefaults(minVersionForCollab);
842
897
 
843
898
  // The following are the default values for the options that do not affect the DocumentSchema.
@@ -1136,6 +1191,8 @@ export class ContainerRuntime
1136
1191
 
1137
1192
  public readonly clientDetails: IClientDetails;
1138
1193
 
1194
+ private readonly isSummarizerClient: boolean;
1195
+
1139
1196
  public get storage(): IDocumentStorageService {
1140
1197
  return this._storage;
1141
1198
  }
@@ -1148,10 +1205,10 @@ export class ContainerRuntime
1148
1205
  summaryOp: ISummaryContent,
1149
1206
  referenceSequenceNumber?: number,
1150
1207
  ) => number;
1151
- /**
1152
- * Do not call directly - use submitAddressesSignal
1153
- */
1154
- private readonly submitSignalFn: (content: ISignalEnvelope, targetClientId?: string) => void;
1208
+ private readonly submitSignalFn: (
1209
+ content: UnsequencedSignalEnvelope,
1210
+ targetClientId?: string,
1211
+ ) => void;
1155
1212
  public readonly disposeFn: (error?: ICriticalContainerError) => void;
1156
1213
  public readonly closeFn: (error?: ICriticalContainerError) => void;
1157
1214
 
@@ -1266,9 +1323,9 @@ export class ContainerRuntime
1266
1323
  private readonly batchRunner = new BatchRunCounter();
1267
1324
  private readonly _flushMode: FlushMode;
1268
1325
  private readonly offlineEnabled: boolean;
1269
- private flushTaskExists = false;
1326
+ private flushScheduled = false;
1270
1327
 
1271
- private _connected: boolean;
1328
+ private canSendOps: boolean;
1272
1329
 
1273
1330
  private consecutiveReconnects = 0;
1274
1331
 
@@ -1282,7 +1339,7 @@ export class ContainerRuntime
1282
1339
 
1283
1340
  /**
1284
1341
  * Invokes the given callback and expects that no ops are submitted
1285
- * until execution finishes. If an op is submitted, an error will be raised.
1342
+ * until execution finishes. If an op is submitted, it will be marked as reentrant.
1286
1343
  *
1287
1344
  * @param callback - the callback to be invoked
1288
1345
  */
@@ -1290,8 +1347,12 @@ export class ContainerRuntime
1290
1347
  return this.dataModelChangeRunner.run(callback);
1291
1348
  }
1292
1349
 
1350
+ /**
1351
+ * Indicates whether the container is in a state where it is able to send
1352
+ * ops (connected to op stream and not in readonly mode).
1353
+ */
1293
1354
  public get connected(): boolean {
1294
- return this._connected;
1355
+ return this.canSendOps;
1295
1356
  }
1296
1357
 
1297
1358
  /**
@@ -1306,7 +1367,7 @@ export class ContainerRuntime
1306
1367
  return this._disposed;
1307
1368
  }
1308
1369
 
1309
- private dirtyContainer: boolean;
1370
+ private lastEmittedDirty: boolean;
1310
1371
  private emitDirtyDocumentEvent = true;
1311
1372
  private readonly useDeltaManagerOpsProxy: boolean;
1312
1373
  private readonly closeSummarizerDelayMs: number;
@@ -1405,6 +1466,8 @@ export class ContainerRuntime
1405
1466
  */
1406
1467
  private readonly skipSafetyFlushDuringProcessStack: boolean;
1407
1468
 
1469
+ private readonly extensions = new Map<ContainerExtensionId, ExtensionEntry>();
1470
+
1408
1471
  /***/
1409
1472
  protected constructor(
1410
1473
  context: IContainerContext,
@@ -1476,6 +1539,11 @@ export class ContainerRuntime
1476
1539
  this.mc = createChildMonitoringContext({
1477
1540
  logger: this.baseLogger,
1478
1541
  namespace: "ContainerRuntime",
1542
+ properties: {
1543
+ all: {
1544
+ inStagingMode: this.inStagingMode,
1545
+ },
1546
+ },
1479
1547
  });
1480
1548
 
1481
1549
  // If we support multiple algorithms in the future, then we would need to manage it here carefully.
@@ -1498,13 +1566,41 @@ export class ContainerRuntime
1498
1566
  this.submitSummaryFn =
1499
1567
  submitSummaryFn ??
1500
1568
  ((summaryOp, refseq) => submitFn(MessageType.Summarize, summaryOp, false));
1501
- this.submitSignalFn = submitSignalFn;
1569
+
1570
+ const sequenceAndSubmitSignal = (
1571
+ envelope: UnsequencedSignalEnvelope,
1572
+ targetClientId?: string,
1573
+ ): void => {
1574
+ if (targetClientId === undefined) {
1575
+ this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope);
1576
+ }
1577
+ submitSignalFn(envelope, targetClientId);
1578
+ };
1579
+ this.submitSignalFn = (envelope: UnsequencedSignalEnvelope, targetClientId?: string) => {
1580
+ if (envelope.address?.startsWith("/")) {
1581
+ throw new Error("General path based addressing is not implemented");
1582
+ }
1583
+ sequenceAndSubmitSignal(envelope, targetClientId);
1584
+ };
1585
+ this.submitExtensionSignal = <TMessage extends TypedMessage>(
1586
+ id: string,
1587
+ addressChain: string[],
1588
+ message: OutboundExtensionMessage<TMessage>,
1589
+ ): void => {
1590
+ this.verifyNotClosed();
1591
+ const envelope = createNewSignalEnvelope(
1592
+ `/ext/${id}/${addressChain.join("/")}`,
1593
+ message.type,
1594
+ message.content,
1595
+ );
1596
+ sequenceAndSubmitSignal(envelope, message.targetClientId);
1597
+ };
1502
1598
 
1503
1599
  // TODO: After IContainerContext.options is removed, we'll just create a new blank object {} here.
1504
1600
  // Values are generally expected to be set from the runtime side.
1505
1601
  this.options = options ?? {};
1506
1602
  this.clientDetails = clientDetails;
1507
- const isSummarizerClient = this.clientDetails.type === summarizerClientType;
1603
+ this.isSummarizerClient = this.clientDetails.type === summarizerClientType;
1508
1604
  this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
1509
1605
  // eslint-disable-next-line unicorn/consistent-destructuring
1510
1606
  this._getClientId = () => context.clientId;
@@ -1532,8 +1628,8 @@ export class ContainerRuntime
1532
1628
  this.mc.logger.sendTelemetryEvent({
1533
1629
  eventName: "Attached",
1534
1630
  details: {
1535
- dirtyContainer: this.dirtyContainer,
1536
- hasPendingMessages: this.hasPendingMessages(),
1631
+ lastEmittedDirty: this.lastEmittedDirty,
1632
+ currentDirtyState: this.computeCurrentDirtyState(),
1537
1633
  },
1538
1634
  });
1539
1635
  });
@@ -1545,7 +1641,7 @@ export class ContainerRuntime
1545
1641
  );
1546
1642
 
1547
1643
  // In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
1548
- this.closeFn = isSummarizerClient ? this.disposeFn : closeFn;
1644
+ this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
1549
1645
 
1550
1646
  let loadSummaryNumber: number;
1551
1647
  // Get the container creation metadata. For new container, we initialize these. For existing containers,
@@ -1571,7 +1667,7 @@ export class ContainerRuntime
1571
1667
 
1572
1668
  // Note that we only need to pull the *initial* connected state from the context.
1573
1669
  // Later updates come through calls to setConnectionState.
1574
- this._connected = connected;
1670
+ this.canSendOps = connected;
1575
1671
 
1576
1672
  this.mc.logger.sendTelemetryEvent({
1577
1673
  eventName: "GCFeatureMatrix",
@@ -1706,7 +1802,7 @@ export class ContainerRuntime
1706
1802
  existing,
1707
1803
  metadata,
1708
1804
  createContainerMetadata: this.createContainerMetadata,
1709
- isSummarizerClient,
1805
+ isSummarizerClient: this.isSummarizerClient,
1710
1806
  getNodePackagePath: async (nodePath: string) => this.getGCNodePackagePath(nodePath),
1711
1807
  getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
1712
1808
  readAndParseBlob: async <T>(id: string) => readAndParse<T>(this.storage, id),
@@ -1757,9 +1853,6 @@ export class ContainerRuntime
1757
1853
  // verifyNotClosed is called in FluidDataStoreContext, which is *the* expected caller.
1758
1854
  const envelope1 = content as IEnvelope;
1759
1855
  const envelope2 = createNewSignalEnvelope(envelope1.address, type, envelope1.contents);
1760
- if (targetClientId === undefined) {
1761
- this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope2);
1762
- }
1763
1856
  this.submitSignalFn(envelope2, targetClientId);
1764
1857
  };
1765
1858
 
@@ -1836,7 +1929,7 @@ export class ContainerRuntime
1836
1929
  this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableFlushBeforeProcess") === true;
1837
1930
 
1838
1931
  this.outbox = new Outbox({
1839
- shouldSend: () => this.canSendOps(),
1932
+ shouldSend: () => this.shouldSendOps(),
1840
1933
  pendingStateManager: this.pendingStateManager,
1841
1934
  submitBatchFn,
1842
1935
  legacySendBatchFn,
@@ -1897,9 +1990,9 @@ export class ContainerRuntime
1897
1990
  this.closeSummarizerDelayMs =
1898
1991
  closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
1899
1992
 
1900
- this.dirtyContainer =
1901
- this.attachState !== AttachState.Attached || this.hasPendingMessages();
1902
- context.updateDirtyContainerState(this.dirtyContainer);
1993
+ // We haven't emitted dirty/saved yet, but this is the baseline so we know to emit when it changes
1994
+ this.lastEmittedDirty = this.computeCurrentDirtyState();
1995
+ context.updateDirtyContainerState(this.lastEmittedDirty);
1903
1996
 
1904
1997
  if (!this.skipSafetyFlushDuringProcessStack) {
1905
1998
  // Reference Sequence Number may have just changed, and it must be consistent across a batch,
@@ -2017,7 +2110,18 @@ export class ContainerRuntime
2017
2110
  this.sessionSchema.idCompressorMode === "on" ||
2018
2111
  (this.sessionSchema.idCompressorMode === "delayed" && this.connected)
2019
2112
  ) {
2020
- this._idCompressor = this.createIdCompressorFn();
2113
+ PerformanceEvent.timedExec(
2114
+ this.mc.logger,
2115
+ { eventName: "CreateIdCompressorOnBoot" },
2116
+ (event) => {
2117
+ this._idCompressor = this.createIdCompressorFn();
2118
+ event.end({
2119
+ details: {
2120
+ idCompressorMode: this.sessionSchema.idCompressorMode,
2121
+ },
2122
+ });
2123
+ },
2124
+ );
2021
2125
  // This is called from loadRuntime(), long before we process any ops, so there should be no ops accumulated yet.
2022
2126
  assert(this.pendingIdCompressorOps.length === 0, 0x8ec /* no pending ops */);
2023
2127
  }
@@ -2074,8 +2178,7 @@ export class ContainerRuntime
2074
2178
  maxOpsSinceLastSummary,
2075
2179
  );
2076
2180
 
2077
- const isSummarizerClient = this.clientDetails.type === summarizerClientType;
2078
- if (isSummarizerClient) {
2181
+ if (this.isSummarizerClient) {
2079
2182
  // We want to dynamically import any thing inside summaryDelayLoadedModule module only when we are the summarizer client,
2080
2183
  // so that all non summarizer clients don't have to load the code inside this module.
2081
2184
  const module = await import(
@@ -2524,21 +2627,16 @@ export class ContainerRuntime
2524
2627
 
2525
2628
  private replayPendingStates(): void {
2526
2629
  // We need to be able to send ops to replay states
2527
- if (!this.canSendOps()) {
2630
+ if (!this.shouldSendOps()) {
2528
2631
  return;
2529
2632
  }
2530
2633
 
2531
- // We need to temporary clear the dirty flags and disable
2532
- // dirty state change events to detect whether replaying ops
2533
- // has any effect.
2534
-
2535
- // Save the old state, reset to false, disable event emit
2536
- const oldState = this.dirtyContainer;
2537
- this.dirtyContainer = false;
2538
-
2634
+ // Replaying is an internal operation and we don't want to generate noise while doing it.
2635
+ // So temporarily disable dirty state change events, and save the old state.
2636
+ // When we're done, we'll emit the event if the state changed.
2637
+ const oldState = this.lastEmittedDirty;
2539
2638
  assert(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
2540
2639
  this.emitDirtyDocumentEvent = false;
2541
- let newState: boolean;
2542
2640
 
2543
2641
  try {
2544
2642
  // Any ID Allocation ops that failed to submit after the pending state was queued need to have
@@ -2546,18 +2644,18 @@ export class ContainerRuntime
2546
2644
  // Since we don't submit ID Allocation ops when staged, any outstanding ranges would be from
2547
2645
  // before staging mode so we can simply say staged: false.
2548
2646
  this.submitIdAllocationOpIfNeeded({ resubmitOutstandingRanges: true, staged: false });
2647
+ this.scheduleFlush();
2549
2648
 
2550
2649
  // replay the ops
2551
2650
  this.pendingStateManager.replayPendingStates();
2552
2651
  } finally {
2553
- // Save the new start and restore the old state, re-enable event emit
2554
- newState = this.dirtyContainer;
2555
- this.dirtyContainer = oldState;
2652
+ // Restore the old state, re-enable event emit
2653
+ this.lastEmittedDirty = oldState;
2556
2654
  this.emitDirtyDocumentEvent = true;
2557
2655
  }
2558
2656
 
2559
- // Officially transition from the old state to the new state.
2560
- this.updateDocumentDirtyState(newState);
2657
+ // This will emit an event if the state changed relative to before replay
2658
+ this.updateDocumentDirtyState();
2561
2659
  }
2562
2660
 
2563
2661
  /**
@@ -2625,13 +2723,27 @@ export class ContainerRuntime
2625
2723
  this._idCompressor === undefined &&
2626
2724
  this.sessionSchema.idCompressorMode !== undefined
2627
2725
  ) {
2628
- this._idCompressor = this.createIdCompressorFn();
2629
- // Finalize any ranges we received while the compressor was turned off.
2630
- const ops = this.pendingIdCompressorOps;
2631
- this.pendingIdCompressorOps = [];
2632
- for (const range of ops) {
2633
- this._idCompressor.finalizeCreationRange(range);
2634
- }
2726
+ PerformanceEvent.timedExec(
2727
+ this.mc.logger,
2728
+ { eventName: "CreateIdCompressorOnDelayedLoad" },
2729
+ (event) => {
2730
+ this._idCompressor = this.createIdCompressorFn();
2731
+ // Finalize any ranges we received while the compressor was turned off.
2732
+ const ops = this.pendingIdCompressorOps;
2733
+ this.pendingIdCompressorOps = [];
2734
+ const trace = Trace.start();
2735
+ for (const range of ops) {
2736
+ this._idCompressor.finalizeCreationRange(range);
2737
+ }
2738
+ event.end({
2739
+ details: {
2740
+ finalizeCreationRangeDuration: trace.trace().duration,
2741
+ idCompressorMode: this.sessionSchema.idCompressorMode,
2742
+ pendingIdCompressorOps: ops.length,
2743
+ },
2744
+ });
2745
+ },
2746
+ );
2635
2747
  assert(this.pendingIdCompressorOps.length === 0, 0x976 /* No new ops added */);
2636
2748
  }
2637
2749
  }
@@ -2639,7 +2751,7 @@ export class ContainerRuntime
2639
2751
  private readonly notifyReadOnlyState = (readonly: boolean): void =>
2640
2752
  this.channelCollection.notifyReadOnlyState(readonly);
2641
2753
 
2642
- public setConnectionState(connected: boolean, clientId?: string): void {
2754
+ public setConnectionState(canSendOps: boolean, clientId?: string): void {
2643
2755
  // Validate we have consistent state
2644
2756
  const currentClientId = this._audience.getSelf()?.clientId;
2645
2757
  assert(clientId === currentClientId, 0x977 /* input clientId does not match Audience */);
@@ -2648,10 +2760,10 @@ export class ContainerRuntime
2648
2760
  0x978 /* this.clientId does not match Audience */,
2649
2761
  );
2650
2762
 
2651
- if (connected && this.sessionSchema.idCompressorMode === "delayed") {
2763
+ if (canSendOps && this.sessionSchema.idCompressorMode === "delayed") {
2652
2764
  this.loadIdCompressor();
2653
2765
  }
2654
- if (connected === false && this.delayConnectClientId !== undefined) {
2766
+ if (canSendOps === false && this.delayConnectClientId !== undefined) {
2655
2767
  this.delayConnectClientId = undefined;
2656
2768
  this.mc.logger.sendTelemetryEvent({
2657
2769
  eventName: "UnsuccessfulConnectedTransition",
@@ -2660,14 +2772,10 @@ export class ContainerRuntime
2660
2772
  return;
2661
2773
  }
2662
2774
 
2663
- if (!connected) {
2664
- this.documentsSchemaController.onDisconnect();
2665
- }
2666
-
2667
2775
  // If there are stashed blobs in the pending state, we need to delay
2668
2776
  // propagation of the "connected" event until we have uploaded them to
2669
2777
  // ensure we don't submit ops referencing a blob that has not been uploaded
2670
- const connecting = connected && !this._connected;
2778
+ const connecting = canSendOps && !this.canSendOps;
2671
2779
  if (connecting && this.blobManager.hasPendingStashedUploads()) {
2672
2780
  assert(
2673
2781
  !this.delayConnectClientId,
@@ -2678,10 +2786,15 @@ export class ContainerRuntime
2678
2786
  return;
2679
2787
  }
2680
2788
 
2681
- this.setConnectionStateCore(connected, clientId);
2789
+ this.setConnectionStateCore(canSendOps, clientId);
2682
2790
  }
2683
2791
 
2684
- private setConnectionStateCore(connected: boolean, clientId?: string): void {
2792
+ /**
2793
+ * Raises and propagates connected events.
2794
+ * @param canSendOps - Indicates whether the container can send ops or not (connected and not readonly).
2795
+ * @remarks The connection state from container context used here when raising connected events.
2796
+ */
2797
+ private setConnectionStateCore(canSendOps: boolean, clientId?: string): void {
2685
2798
  assert(
2686
2799
  !this.delayConnectClientId,
2687
2800
  0x394 /* connect event delay must be cleared before propagating connect event */,
@@ -2689,24 +2802,24 @@ export class ContainerRuntime
2689
2802
  this.verifyNotClosed();
2690
2803
 
2691
2804
  // There might be no change of state due to Container calling this API after loading runtime.
2692
- const changeOfState = this._connected !== connected;
2693
- const reconnection = changeOfState && !connected;
2805
+ const canSendOpsChanged = this.canSendOps !== canSendOps;
2806
+ const reconnection = canSendOpsChanged && !canSendOps;
2694
2807
 
2695
2808
  // We need to flush the ops currently collected by Outbox to preserve original order.
2696
2809
  // This flush NEEDS to happen before we set the ContainerRuntime to "connected".
2697
2810
  // We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
2698
- if (changeOfState && connected) {
2811
+ if (canSendOpsChanged && canSendOps) {
2699
2812
  this.flush();
2700
2813
  }
2701
2814
 
2702
- this._connected = connected;
2815
+ this.canSendOps = canSendOps;
2703
2816
 
2704
- if (connected) {
2817
+ if (canSendOps) {
2705
2818
  assert(
2706
2819
  this.attachState === AttachState.Attached,
2707
2820
  0x3cd /* Connection is possible only if container exists in storage */,
2708
2821
  );
2709
- if (changeOfState) {
2822
+ if (canSendOpsChanged) {
2710
2823
  this.signalTelemetryManager.resetTracking();
2711
2824
  }
2712
2825
  }
@@ -2732,14 +2845,14 @@ export class ContainerRuntime
2732
2845
  }
2733
2846
  }
2734
2847
 
2735
- if (changeOfState) {
2848
+ if (canSendOpsChanged) {
2736
2849
  this.replayPendingStates();
2737
2850
  }
2738
2851
 
2739
- this.channelCollection.setConnectionState(connected, clientId);
2740
- this.garbageCollector.setConnectionState(connected, clientId);
2852
+ this.channelCollection.setConnectionState(canSendOps, clientId);
2853
+ this.garbageCollector.setConnectionState(canSendOps, clientId);
2741
2854
 
2742
- raiseConnectedEvent(this.mc.logger, this, connected, clientId);
2855
+ raiseConnectedEvent(this.mc.logger, this, this.connected /* canSendOps */, clientId);
2743
2856
  }
2744
2857
 
2745
2858
  public async notifyOpReplay(message: ISequencedDocumentMessage): Promise<void> {
@@ -2928,6 +3041,9 @@ export class ContainerRuntime
2928
3041
  runtimeBatch: boolean,
2929
3042
  groupedBatch: boolean,
2930
3043
  ): void {
3044
+ // This message could have been the last pending local (dirtyable) message, in which case we need to update dirty state to "saved"
3045
+ this.updateDocumentDirtyState();
3046
+
2931
3047
  if (locationInBatch.batchStart) {
2932
3048
  const firstMessage = messagesWithMetadata[0]?.message;
2933
3049
  assert(firstMessage !== undefined, 0xa31 /* Batch must have at least one message */);
@@ -3043,12 +3159,6 @@ export class ContainerRuntime
3043
3159
 
3044
3160
  this._processedClientSequenceNumber = message.clientSequenceNumber;
3045
3161
 
3046
- // If there are no more pending messages after processing a local message,
3047
- // the document is no longer dirty.
3048
- if (!this.hasPendingMessages()) {
3049
- this.updateDocumentDirtyState(false);
3050
- }
3051
-
3052
3162
  // The DeltaManager used to do this, but doesn't anymore as of Loader v2.4
3053
3163
  // Anyone listening to our "op" event would expect the contents to be parsed per this same logic
3054
3164
  if (
@@ -3079,12 +3189,6 @@ export class ContainerRuntime
3079
3189
  local: boolean,
3080
3190
  savedOp?: boolean,
3081
3191
  ): void {
3082
- // If there are no more pending messages after processing a local message,
3083
- // the document is no longer dirty.
3084
- if (!this.hasPendingMessages()) {
3085
- this.updateDocumentDirtyState(false);
3086
- }
3087
-
3088
3192
  // Get the contents without the localOpMetadata because not all message types know about localOpMetadata.
3089
3193
  const contents = messagesContent.map((c) => c.contents);
3090
3194
 
@@ -3124,7 +3228,7 @@ export class ContainerRuntime
3124
3228
  }
3125
3229
  case ContainerMessageType.DocumentSchemaChange: {
3126
3230
  this.documentsSchemaController.processDocumentSchemaMessages(
3127
- contents as IDocumentSchemaChangeMessage[],
3231
+ contents as IDocumentSchemaChangeMessageIncoming[],
3128
3232
  local,
3129
3233
  message.sequenceNumber,
3130
3234
  );
@@ -3171,9 +3275,15 @@ export class ContainerRuntime
3171
3275
  }
3172
3276
  }
3173
3277
 
3174
- public processSignal(message: ISignalMessage, local: boolean): void {
3175
- const envelope = message.content as ISignalEnvelope;
3176
- const transformed: IInboundSignalMessage = {
3278
+ public processSignal(
3279
+ message: ISignalMessage<{
3280
+ type: string;
3281
+ content: ISignalEnvelope<{ type: string; content: JsonDeserialized<unknown> }>;
3282
+ }>,
3283
+ local: boolean,
3284
+ ): void {
3285
+ const envelope = message.content;
3286
+ const transformed = {
3177
3287
  clientId: message.clientId,
3178
3288
  content: envelope.contents.content,
3179
3289
  type: envelope.contents.type,
@@ -3189,22 +3299,53 @@ export class ContainerRuntime
3189
3299
  );
3190
3300
  }
3191
3301
 
3192
- if (envelope.address === undefined) {
3302
+ const fullAddress = envelope.address;
3303
+ if (fullAddress === undefined) {
3193
3304
  // No address indicates a container signal message.
3194
3305
  this.emit("signal", transformed, local);
3195
3306
  return;
3196
3307
  }
3197
3308
 
3198
- // Due to a mismatch between different layers in terms of
3199
- // what is the interface of passing signals, we need to adjust
3200
- // the signal envelope before sending it to the datastores to be processed
3201
- const envelope2: IEnvelope = {
3202
- address: envelope.address,
3203
- contents: transformed.content,
3204
- };
3205
- transformed.content = envelope2;
3309
+ this.routeNonContainerSignal(fullAddress, transformed, local);
3310
+ }
3311
+
3312
+ private routeNonContainerSignal(
3313
+ address: string,
3314
+ signalMessage: IInboundSignalMessage<{ type: string; content: JsonDeserialized<unknown> }>,
3315
+ local: boolean,
3316
+ ): void {
3317
+ // channelCollection signals are identified by no starting `/` in address.
3318
+ if (!address.startsWith("/")) {
3319
+ // Due to a mismatch between different layers in terms of
3320
+ // what is the interface of passing signals, we need to adjust
3321
+ // the signal envelope before sending it to the datastores to be processed
3322
+ const envelope = {
3323
+ address,
3324
+ contents: signalMessage.content,
3325
+ };
3326
+ signalMessage.content = envelope;
3327
+
3328
+ this.channelCollection.processSignal(signalMessage, local);
3329
+ return;
3330
+ }
3331
+
3332
+ const addresses = address.split("/");
3333
+ if (addresses.length > 2 && addresses[1] === "ext") {
3334
+ const id = addresses[2] as ContainerExtensionId;
3335
+ const entry = this.extensions.get(id);
3336
+ if (entry !== undefined) {
3337
+ entry.extension.processSignal?.(addresses.slice(3), signalMessage, local);
3338
+ return;
3339
+ }
3340
+ }
3206
3341
 
3207
- this.channelCollection.processSignal(transformed, local);
3342
+ assert(!local, 0xba0 /* No recipient found for local signal */);
3343
+ this.mc.logger.sendTelemetryEvent({
3344
+ eventName: "SignalAddressNotFound",
3345
+ ...tagCodeArtifacts({
3346
+ address,
3347
+ }),
3348
+ });
3208
3349
  }
3209
3350
 
3210
3351
  /**
@@ -3215,6 +3356,8 @@ export class ContainerRuntime
3215
3356
  * @param resubmitInfo - If defined, indicates this is a resubmission of a batch with the given Batch info needed for resubmit.
3216
3357
  */
3217
3358
  private flush(resubmitInfo?: BatchResubmitInfo): void {
3359
+ this.flushScheduled = false;
3360
+
3218
3361
  try {
3219
3362
  assert(
3220
3363
  !this.batchRunner.running,
@@ -3239,7 +3382,6 @@ export class ContainerRuntime
3239
3382
  */
3240
3383
  public orderSequentially<T>(callback: () => T): T {
3241
3384
  let checkpoint: IBatchCheckpoint | undefined;
3242
- const checkpointDirtyState = this.dirtyContainer;
3243
3385
  // eslint-disable-next-line import/no-deprecated
3244
3386
  let stageControls: StageControlsExperimental | undefined;
3245
3387
  if (this.mc.config.getBoolean("Fluid.ContainerRuntime.EnableRollback")) {
@@ -3259,12 +3401,10 @@ export class ContainerRuntime
3259
3401
  // This will throw and close the container if rollback fails
3260
3402
  try {
3261
3403
  checkpoint.rollback((message: LocalBatchMessage) =>
3262
- this.rollback(message.runtimeOp, message.localOpMetadata),
3404
+ // These changes are staged since we entered staging mode above
3405
+ this.rollbackStagedChanges(message.runtimeOp, message.localOpMetadata),
3263
3406
  );
3264
- // reset the dirty state after rollback to what it was before to keep it consistent
3265
- if (this.dirtyContainer !== checkpointDirtyState) {
3266
- this.updateDocumentDirtyState(checkpointDirtyState);
3267
- }
3407
+ this.updateDocumentDirtyState();
3268
3408
  stageControls?.discardChanges();
3269
3409
  stageControls = undefined;
3270
3410
  } catch (error_) {
@@ -3328,7 +3468,10 @@ export class ContainerRuntime
3328
3468
  // eslint-disable-next-line import/no-deprecated
3329
3469
  public enterStagingMode = (): StageControlsExperimental => {
3330
3470
  if (this.stageControls !== undefined) {
3331
- throw new Error("already in staging mode");
3471
+ throw new UsageError("already in staging mode");
3472
+ }
3473
+ if (this.attachState === AttachState.Detached) {
3474
+ throw new UsageError("cannot enter staging mode while detached");
3332
3475
  }
3333
3476
 
3334
3477
  // Make sure all BatchManagers are empty before entering staging mode,
@@ -3358,11 +3501,9 @@ export class ContainerRuntime
3358
3501
  runtimeOp !== undefined,
3359
3502
  0xb82 /* Staged batches expected to have runtimeOp defined */,
3360
3503
  );
3361
- this.rollback(runtimeOp, localOpMetadata);
3504
+ this.rollbackStagedChanges(runtimeOp, localOpMetadata);
3362
3505
  });
3363
- if (this.attachState === AttachState.Attached) {
3364
- this.updateDocumentDirtyState(this.pendingMessagesCount !== 0);
3365
- }
3506
+ this.updateDocumentDirtyState();
3366
3507
  }),
3367
3508
  commitChanges: (optionsParam) => {
3368
3509
  const options = { ...defaultStagingCommitOptions, ...optionsParam };
@@ -3444,7 +3585,7 @@ export class ContainerRuntime
3444
3585
  );
3445
3586
  }
3446
3587
 
3447
- private canSendOps(): boolean {
3588
+ private shouldSendOps(): boolean {
3448
3589
  // Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
3449
3590
  // container runtime's ability to send ops depend on the actual readonly state of the delta manager.
3450
3591
  return (
@@ -3452,13 +3593,6 @@ export class ContainerRuntime
3452
3593
  );
3453
3594
  }
3454
3595
 
3455
- /**
3456
- * Typically ops are batched and later flushed together, but in some cases we want to flush immediately.
3457
- */
3458
- private currentlyBatching(): boolean {
3459
- return this.flushMode !== FlushMode.Immediate || this.batchRunner.running;
3460
- }
3461
-
3462
3596
  private readonly _quorum: IQuorumClients;
3463
3597
  public getQuorum(): IQuorumClients {
3464
3598
  return this._quorum;
@@ -3474,40 +3608,20 @@ export class ContainerRuntime
3474
3608
  * either were not sent out to delta stream or were not yet acknowledged.
3475
3609
  */
3476
3610
  public get isDirty(): boolean {
3477
- return this.dirtyContainer;
3611
+ // Rather than recomputing the dirty state in this moment,
3612
+ // just regurgitate the last emitted dirty state.
3613
+ return this.lastEmittedDirty;
3478
3614
  }
3479
3615
 
3480
- private isContainerMessageDirtyable({
3481
- type,
3482
- contents,
3483
- }: LocalContainerRuntimeMessage): boolean {
3484
- // Certain container runtime messages should not mark the container dirty such as the old built-in
3485
- // AgentScheduler and Garbage collector messages.
3486
- switch (type) {
3487
- case ContainerMessageType.Attach: {
3488
- const attachMessage = contents as InboundAttachMessage;
3489
- if (attachMessage.id === agentSchedulerId) {
3490
- return false;
3491
- }
3492
- break;
3493
- }
3494
- case ContainerMessageType.FluidDataStoreOp: {
3495
- const envelope = contents;
3496
- if (envelope.address === agentSchedulerId) {
3497
- return false;
3498
- }
3499
- break;
3500
- }
3501
- case ContainerMessageType.IdAllocation:
3502
- case ContainerMessageType.DocumentSchemaChange:
3503
- case ContainerMessageType.GC: {
3504
- return false;
3505
- }
3506
- default: {
3507
- break;
3508
- }
3509
- }
3510
- return true;
3616
+ /**
3617
+ * Returns true if the container is dirty: not attached, or no pending user messages (could be some "non-dirtyable" ones though)
3618
+ */
3619
+ private computeCurrentDirtyState(): boolean {
3620
+ return (
3621
+ this.attachState !== AttachState.Attached ||
3622
+ this.pendingStateManager.hasPendingUserChanges() ||
3623
+ this.outbox.containsUserChanges()
3624
+ );
3511
3625
  }
3512
3626
 
3513
3627
  /**
@@ -3525,9 +3639,6 @@ export class ContainerRuntime
3525
3639
  public submitSignal(type: string, content: unknown, targetClientId?: string): void {
3526
3640
  this.verifyNotClosed();
3527
3641
  const envelope = createNewSignalEnvelope(undefined /* address */, type, content);
3528
- if (targetClientId === undefined) {
3529
- this.signalTelemetryManager.applyTrackingToBroadcastSignalEnvelope(envelope);
3530
- }
3531
3642
  this.submitSignalFn(envelope, targetClientId);
3532
3643
  }
3533
3644
 
@@ -3545,9 +3656,7 @@ export class ContainerRuntime
3545
3656
  this.emit("attached");
3546
3657
  }
3547
3658
 
3548
- if (attachState === AttachState.Attached && !this.hasPendingMessages()) {
3549
- this.updateDocumentDirtyState(false);
3550
- }
3659
+ this.updateDocumentDirtyState();
3551
3660
  this.channelCollection.setAttachState(attachState);
3552
3661
  }
3553
3662
 
@@ -4333,22 +4442,22 @@ export class ContainerRuntime
4333
4442
  return this.pendingMessagesCount !== 0;
4334
4443
  }
4335
4444
 
4336
- private updateDocumentDirtyState(dirty: boolean): void {
4337
- if (this.attachState === AttachState.Attached) {
4338
- // Other way is not true = see this.isContainerMessageDirtyable()
4339
- assert(
4340
- !dirty || this.hasPendingMessages(),
4341
- 0x3d3 /* if doc is dirty, there has to be pending ops */,
4342
- );
4343
- } else {
4344
- assert(dirty, 0x3d2 /* Non-attached container is dirty */);
4345
- }
4445
+ /**
4446
+ * Emit "dirty" or "saved" event based on the current dirty state of the document.
4447
+ * This must be called every time the states underlying the dirty state change.
4448
+ *
4449
+ * @privateRemarks - It's helpful to think of this as an event handler registered
4450
+ * for hypothetical "changed" events for PendingStateManager, Outbox, and Container Attach machinery.
4451
+ * But those events don't exist so we manually call this wherever we know those changes happen.
4452
+ */
4453
+ private updateDocumentDirtyState(): void {
4454
+ const dirty: boolean = this.computeCurrentDirtyState();
4346
4455
 
4347
- if (this.dirtyContainer === dirty) {
4456
+ if (this.lastEmittedDirty === dirty) {
4348
4457
  return;
4349
4458
  }
4350
4459
 
4351
- this.dirtyContainer = dirty;
4460
+ this.lastEmittedDirty = dirty;
4352
4461
  if (this.emitDirtyDocumentEvent) {
4353
4462
  this.emit(dirty ? "dirty" : "saved");
4354
4463
  }
@@ -4441,6 +4550,11 @@ export class ContainerRuntime
4441
4550
  // If we're resubmitting a batch, keep the same "staged" value as before. Otherwise, use the current "global" state.
4442
4551
  const staged = this.batchRunner.resubmitInfo?.staged ?? this.inStagingMode;
4443
4552
 
4553
+ assert(
4554
+ !staged || canStageMessageOfType(type),
4555
+ 0xbba /* Unexpected message type submitted in Staging Mode */,
4556
+ );
4557
+
4444
4558
  // Before submitting any non-staged change, submit the ID Allocation op to cover any compressed IDs included in the op.
4445
4559
  if (!staged) {
4446
4560
  this.submitIdAllocationOpIfNeeded({ staged: false });
@@ -4449,7 +4563,7 @@ export class ContainerRuntime
4449
4563
  // Allow document schema controller to send a message if it needs to propose change in document schema.
4450
4564
  // If it needs to send a message, it will call provided callback with payload of such message and rely
4451
4565
  // on this callback to do actual sending.
4452
- const schemaChangeMessage = this.documentsSchemaController.maybeSendSchemaMessage();
4566
+ const schemaChangeMessage = this.documentsSchemaController.maybeGenerateSchemaMessage();
4453
4567
  if (schemaChangeMessage) {
4454
4568
  this.mc.logger.sendTelemetryEvent({
4455
4569
  eventName: "SchemaChangeProposal",
@@ -4459,7 +4573,7 @@ export class ContainerRuntime
4459
4573
  sessionRuntimeSchema: JSON.stringify(this.sessionSchema),
4460
4574
  oldRuntimeSchema: JSON.stringify(this.metadata?.documentSchema?.runtime),
4461
4575
  });
4462
- const msg: ContainerRuntimeDocumentSchemaMessage = {
4576
+ const msg: OutboundContainerRuntimeDocumentSchemaMessage = {
4463
4577
  type: ContainerMessageType.DocumentSchemaChange,
4464
4578
  contents: schemaChangeMessage,
4465
4579
  };
@@ -4486,13 +4600,7 @@ export class ContainerRuntime
4486
4600
  this.outbox.submit(message);
4487
4601
  }
4488
4602
 
4489
- // Note: Technically, the system "always" batches - if this case is true we'll just have a single-message batch.
4490
- const flushImmediatelyOnSubmit = !this.currentlyBatching();
4491
- if (flushImmediatelyOnSubmit) {
4492
- this.flush();
4493
- } else {
4494
- this.scheduleFlush();
4495
- }
4603
+ this.scheduleFlush();
4496
4604
  } catch (error) {
4497
4605
  const dpe = DataProcessingError.wrapIfUnrecognized(error, "ContainerRuntime.submit", {
4498
4606
  referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
@@ -4501,31 +4609,28 @@ export class ContainerRuntime
4501
4609
  throw dpe;
4502
4610
  }
4503
4611
 
4504
- if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
4505
- this.updateDocumentDirtyState(true);
4506
- }
4612
+ this.updateDocumentDirtyState();
4507
4613
  }
4508
4614
 
4509
4615
  private scheduleFlush(): void {
4510
- if (this.flushTaskExists) {
4616
+ if (this.flushScheduled) {
4511
4617
  return;
4512
4618
  }
4513
-
4514
- this.flushTaskExists = true;
4515
-
4516
- // TODO: hoist this out of the function scope to save unnecessary allocations
4517
- // eslint-disable-next-line unicorn/consistent-function-scoping -- Separate `flush` method already exists in outer scope
4518
- const flush = (): void => {
4519
- this.flushTaskExists = false;
4520
- this.flush();
4521
- };
4619
+ this.flushScheduled = true;
4522
4620
 
4523
4621
  switch (this.flushMode) {
4622
+ case FlushMode.Immediate: {
4623
+ // When in Immediate flush mode, flush immediately unless we are intentionally batching multiple ops (e.g. via orderSequentially)
4624
+ if (!this.batchRunner.running) {
4625
+ this.flush();
4626
+ }
4627
+ break;
4628
+ }
4524
4629
  case FlushMode.TurnBased: {
4525
4630
  // When in TurnBased flush mode the runtime will buffer operations in the current turn and send them as a single
4526
4631
  // batch at the end of the turn
4527
- // eslint-disable-next-line @typescript-eslint/no-floating-promises
4528
- Promise.resolve().then(flush);
4632
+ // eslint-disable-next-line @typescript-eslint/no-floating-promises -- Container will close if flush throws
4633
+ Promise.resolve().then(() => this.flush());
4529
4634
  break;
4530
4635
  }
4531
4636
 
@@ -4534,16 +4639,12 @@ export class ContainerRuntime
4534
4639
  // When in Async flush mode, the runtime will accumulate all operations across JS turns and send them as a single
4535
4640
  // batch when all micro-tasks are complete.
4536
4641
  // Compared to TurnBased, this flush mode will capture more ops into the same batch.
4537
- setTimeout(flush, 0);
4642
+ setTimeout(() => this.flush(), 0);
4538
4643
  break;
4539
4644
  }
4540
4645
 
4541
4646
  default: {
4542
- assert(
4543
- this.batchRunner.running,
4544
- 0x587 /* Unreachable unless manually accumulating a batch */,
4545
- );
4546
- break;
4647
+ fail(0x587 /* Unreachable unless manually accumulating a batch */);
4547
4648
  }
4548
4649
  }
4549
4650
  }
@@ -4576,14 +4677,25 @@ export class ContainerRuntime
4576
4677
 
4577
4678
  /**
4578
4679
  * Resubmits each message in the batch, and then flushes the outbox.
4680
+ * This typically happens when we reconnect and there are pending messages.
4681
+ *
4682
+ * @remarks
4683
+ * Attempting to resubmit a batch that has been successfully sequenced will not happen due to
4684
+ * checks in the ConnectionStateHandler (Loader layer)
4579
4685
  *
4580
- * @remarks - If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
4686
+ * The only exception to this would be if the Container "forks" due to misuse of the "Offline Load" feature.
4687
+ * If the "Offline Load" feature is enabled, the batchId is included in the resubmitted messages,
4581
4688
  * for correlation to detect container forking.
4582
4689
  */
4583
4690
  private reSubmitBatch(
4584
4691
  batch: PendingMessageResubmitData[],
4585
4692
  { batchId, staged, squash }: PendingBatchResubmitMetadata,
4586
4693
  ): void {
4694
+ assert(
4695
+ this._summarizer === undefined,
4696
+ 0x8f2 /* Summarizer never reconnects so should never resubmit */,
4697
+ );
4698
+
4587
4699
  const resubmitInfo = {
4588
4700
  // Only include Batch ID if "Offline Load" feature is enabled
4589
4701
  // It's only needed to identify batches across container forks arising from misuse of offline load.
@@ -4591,36 +4703,61 @@ export class ContainerRuntime
4591
4703
  staged,
4592
4704
  };
4593
4705
 
4706
+ const resubmitFn = squash
4707
+ ? this.reSubmitWithSquashing.bind(this)
4708
+ : this.reSubmit.bind(this);
4709
+
4594
4710
  this.batchRunner.run(() => {
4595
4711
  for (const message of batch) {
4596
- this.reSubmit(message, squash);
4712
+ resubmitFn(message);
4597
4713
  }
4598
4714
  }, resubmitInfo);
4599
4715
 
4600
4716
  this.flush(resubmitInfo);
4601
4717
  }
4602
4718
 
4603
- private reSubmit(message: PendingMessageResubmitData, squash: boolean): void {
4604
- this.reSubmitCore(message.runtimeOp, message.localOpMetadata, message.opMetadata, squash);
4605
- }
4606
-
4607
4719
  /**
4608
- * Finds the right store and asks it to resubmit the message. This typically happens when we
4609
- * reconnect and there are pending messages.
4610
- * ! Note: successfully resubmitting an op that has been successfully sequenced is not possible due to checks in the ConnectionStateHandler (Loader layer)
4611
- * @param message - The original LocalContainerRuntimeMessage.
4612
- * @param localOpMetadata - The local metadata associated with the original message.
4720
+ * Resubmit the given message as part of a squash rebase upon exiting Staging Mode.
4721
+ * How exactly to resubmit the message is up to the subsystem that submitted the op to begin with.
4613
4722
  */
4614
- private reSubmitCore(
4615
- message: LocalContainerRuntimeMessage,
4616
- localOpMetadata: unknown,
4617
- opMetadata: Record<string, unknown> | undefined,
4618
- squash: boolean,
4619
- ): void {
4723
+ private reSubmitWithSquashing(resubmitData: PendingMessageResubmitData): void {
4724
+ const message = resubmitData.runtimeOp;
4620
4725
  assert(
4621
- this._summarizer === undefined,
4622
- 0x8f2 /* Summarizer never reconnects so should never resubmit */,
4726
+ canStageMessageOfType(message.type),
4727
+ 0xbbb /* Expected message type to be compatible with staging */,
4623
4728
  );
4729
+ switch (message.type) {
4730
+ case ContainerMessageType.FluidDataStoreOp: {
4731
+ this.channelCollection.reSubmit(
4732
+ message.type,
4733
+ message.contents,
4734
+ resubmitData.localOpMetadata,
4735
+ /* squash: */ true,
4736
+ );
4737
+ break;
4738
+ }
4739
+ // NOTE: Squash doesn't apply to GC or DocumentSchemaChange ops, fallback to typical resubmit logic.
4740
+ case ContainerMessageType.GC:
4741
+ case ContainerMessageType.DocumentSchemaChange: {
4742
+ this.reSubmit(resubmitData);
4743
+ break;
4744
+ }
4745
+ default: {
4746
+ unreachableCase(message.type);
4747
+ }
4748
+ }
4749
+ }
4750
+
4751
+ /**
4752
+ * Resubmit the given message which was previously submitted to the ContainerRuntime but not successfully
4753
+ * transmitted to the ordering service (e.g. due to a disconnect, or being in Staging Mode)
4754
+ * How to resubmit is up to the subsystem that submitted the op to begin with
4755
+ */
4756
+ private reSubmit({
4757
+ runtimeOp: message,
4758
+ localOpMetadata,
4759
+ opMetadata,
4760
+ }: PendingMessageResubmitData): void {
4624
4761
  switch (message.type) {
4625
4762
  case ContainerMessageType.FluidDataStoreOp:
4626
4763
  case ContainerMessageType.Attach:
@@ -4631,7 +4768,7 @@ export class ContainerRuntime
4631
4768
  message.type,
4632
4769
  message.contents,
4633
4770
  localOpMetadata,
4634
- squash,
4771
+ /* squash: */ false,
4635
4772
  );
4636
4773
  break;
4637
4774
  }
@@ -4658,9 +4795,9 @@ export class ContainerRuntime
4658
4795
  break;
4659
4796
  }
4660
4797
  case ContainerMessageType.DocumentSchemaChange: {
4661
- // There is no need to resend this message. Document schema controller will properly resend it again (if needed)
4662
- // on a first occasion (any ops sent after reconnect). There is a good chance, though, that it will not want to
4663
- // send any ops, as some other client already changed schema.
4798
+ // We shouldn't directly resubmit due to Compare-And-Swap semantics.
4799
+ // If needed it will be generated from scratch before other ops are submitted.
4800
+ this.documentsSchemaController.pendingOpNotAcked();
4664
4801
  break;
4665
4802
  }
4666
4803
  default: {
@@ -4671,8 +4808,15 @@ export class ContainerRuntime
4671
4808
  }
4672
4809
  }
4673
4810
 
4674
- private rollback(runtimeOp: LocalContainerRuntimeMessage, localOpMetadata: unknown): void {
4675
- const { type, contents } = runtimeOp;
4811
+ /**
4812
+ * Rollback the given op which was only staged but not yet submitted.
4813
+ */
4814
+ private rollbackStagedChanges(
4815
+ { type, contents }: LocalContainerRuntimeMessage,
4816
+ localOpMetadata: unknown,
4817
+ ): void {
4818
+ assert(canStageMessageOfType(type), 0xbbc /* Unexpected message type to be rolled back */);
4819
+
4676
4820
  switch (type) {
4677
4821
  case ContainerMessageType.FluidDataStoreOp: {
4678
4822
  // For operations, call rollbackDataStoreOp which will find the right store
@@ -4680,8 +4824,24 @@ export class ContainerRuntime
4680
4824
  this.channelCollection.rollback(type, contents, localOpMetadata);
4681
4825
  break;
4682
4826
  }
4827
+ case ContainerMessageType.GC: {
4828
+ // Just drop it, but log an error, this is not expected and not ideal, but not critical failure either.
4829
+ // Currently the only expected type here is TombstoneLoaded, which will have been preceded by one of these events as well:
4830
+ // GC_Tombstone_DataStore_Requested, GC_Tombstone_SubDataStore_Requested, GC_Tombstone_Blob_Requested
4831
+ this.mc.logger.sendErrorEvent({
4832
+ eventName: "GC_OpDiscarded",
4833
+ details: { subType: contents.type },
4834
+ });
4835
+ break;
4836
+ }
4837
+ case ContainerMessageType.DocumentSchemaChange: {
4838
+ // Notify the document schema controller that the pending op was not acked.
4839
+ // This will allow it to propose the schema change again if needed.
4840
+ this.documentsSchemaController.pendingOpNotAcked();
4841
+ break;
4842
+ }
4683
4843
  default: {
4684
- throw new Error(`Can't rollback ${type}`);
4844
+ unreachableCase(type);
4685
4845
  }
4686
4846
  }
4687
4847
  }
@@ -4921,6 +5081,60 @@ export class ContainerRuntime
4921
5081
  }
4922
5082
  }
4923
5083
 
5084
+ // While internal, ContainerRuntime has not been converted to use the new events support.
5085
+ // Recreate the required events (new pattern) with injected, wrapper new emitter.
5086
+ // It is lazily create to avoid listeners (old events) that ultimately go nowhere.
5087
+ private readonly lazyEventsForExtensions = new Lazy<Listenable<ExtensionHostEvents>>(() => {
5088
+ const eventEmitter = createEmitter<ExtensionHostEvents>();
5089
+ this.on("connected", (clientId) => eventEmitter.emit("connected", clientId));
5090
+ this.on("disconnected", () => eventEmitter.emit("disconnected"));
5091
+ return eventEmitter;
5092
+ });
5093
+
5094
+ private readonly submitExtensionSignal: <TMessage extends TypedMessage>(
5095
+ id: string,
5096
+ addressChain: string[],
5097
+ message: OutboundExtensionMessage<TMessage>,
5098
+ ) => void;
5099
+
5100
+ public acquireExtension<
5101
+ T,
5102
+ TRuntimeProperties extends ExtensionRuntimeProperties,
5103
+ TUseContext extends unknown[],
5104
+ >(
5105
+ id: ContainerExtensionId,
5106
+ factory: ContainerExtensionFactory<T, TRuntimeProperties, TUseContext>,
5107
+ ...useContext: TUseContext
5108
+ ): T {
5109
+ let entry = this.extensions.get(id);
5110
+ if (entry === undefined) {
5111
+ const runtime = {
5112
+ isConnected: () => this.connected,
5113
+ getClientId: () => this.clientId,
5114
+ events: this.lazyEventsForExtensions.value,
5115
+ logger: this.baseLogger,
5116
+ submitAddressedSignal: (
5117
+ addressChain: string[],
5118
+ message: OutboundExtensionMessage<TRuntimeProperties["SignalMessages"]>,
5119
+ ) => {
5120
+ this.submitExtensionSignal(id, addressChain, message);
5121
+ },
5122
+ getQuorum: this.getQuorum.bind(this),
5123
+ getAudience: this.getAudience.bind(this),
5124
+ supportedFeatures: this.ILayerCompatDetails.supportedFeatures,
5125
+ } satisfies ExtensionHost<TRuntimeProperties>;
5126
+ entry = new factory(runtime, ...useContext);
5127
+ this.extensions.set(id, entry);
5128
+ } else {
5129
+ assert(
5130
+ entry instanceof factory,
5131
+ 0xba1 /* Extension entry is not of the expected type */,
5132
+ );
5133
+ entry.extension.onNewUse(...useContext);
5134
+ }
5135
+ return entry.interface as T;
5136
+ }
5137
+
4924
5138
  private get groupedBatchingEnabled(): boolean {
4925
5139
  return this.sessionSchema.opGroupingEnabled === true;
4926
5140
  }
@@ -4930,11 +5144,44 @@ export function createNewSignalEnvelope(
4930
5144
  address: string | undefined,
4931
5145
  type: string,
4932
5146
  content: unknown,
4933
- ): Omit<ISignalEnvelope, "broadcastSignalSequenceNumber"> {
4934
- const newEnvelope: Omit<ISignalEnvelope, "broadcastSignalSequenceNumber"> = {
5147
+ ): UnsequencedSignalEnvelope {
5148
+ const newEnvelope: UnsequencedSignalEnvelope = {
4935
5149
  address,
4936
5150
  contents: { type, content },
4937
5151
  };
4938
5152
 
4939
5153
  return newEnvelope;
4940
5154
  }
5155
+
5156
+ export function isContainerMessageDirtyable({
5157
+ type,
5158
+ contents,
5159
+ }: LocalContainerRuntimeMessage): boolean {
5160
+ // Certain container runtime messages should not mark the container dirty such as the old built-in
5161
+ // AgentScheduler and Garbage collector messages.
5162
+ switch (type) {
5163
+ case ContainerMessageType.Attach: {
5164
+ const attachMessage = contents as InboundAttachMessage;
5165
+ if (attachMessage.id === agentSchedulerId) {
5166
+ return false;
5167
+ }
5168
+ break;
5169
+ }
5170
+ case ContainerMessageType.FluidDataStoreOp: {
5171
+ const envelope = contents;
5172
+ if (envelope.address === agentSchedulerId) {
5173
+ return false;
5174
+ }
5175
+ break;
5176
+ }
5177
+ case ContainerMessageType.IdAllocation:
5178
+ case ContainerMessageType.DocumentSchemaChange:
5179
+ case ContainerMessageType.GC: {
5180
+ return false;
5181
+ }
5182
+ default: {
5183
+ break;
5184
+ }
5185
+ }
5186
+ return true;
5187
+ }