@fluidframework/container-runtime 2.0.0-dev.1.4.5.105745 → 2.0.0-dev.2.2.0.111723

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (168) hide show
  1. package/.eslintrc.js +1 -1
  2. package/dist/batchManager.d.ts +11 -6
  3. package/dist/batchManager.d.ts.map +1 -1
  4. package/dist/batchManager.js +23 -13
  5. package/dist/batchManager.js.map +1 -1
  6. package/dist/containerRuntime.d.ts +74 -20
  7. package/dist/containerRuntime.d.ts.map +1 -1
  8. package/dist/containerRuntime.js +190 -137
  9. package/dist/containerRuntime.js.map +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +6 -0
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +14 -21
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +71 -57
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStoreContexts.js +1 -1
  18. package/dist/dataStoreContexts.js.map +1 -1
  19. package/dist/dataStores.d.ts +11 -10
  20. package/dist/dataStores.d.ts.map +1 -1
  21. package/dist/dataStores.js +50 -20
  22. package/dist/dataStores.js.map +1 -1
  23. package/dist/garbageCollection.d.ts +36 -19
  24. package/dist/garbageCollection.d.ts.map +1 -1
  25. package/dist/garbageCollection.js +207 -121
  26. package/dist/garbageCollection.js.map +1 -1
  27. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
  28. package/dist/gcSweepReadyUsageDetection.js +3 -12
  29. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  30. package/dist/index.d.ts +4 -6
  31. package/dist/index.d.ts.map +1 -1
  32. package/dist/index.js +3 -5
  33. package/dist/index.js.map +1 -1
  34. package/dist/opCompressor.d.ts +18 -0
  35. package/dist/opCompressor.d.ts.map +1 -0
  36. package/dist/opCompressor.js +50 -0
  37. package/dist/opCompressor.js.map +1 -0
  38. package/dist/opDecompressor.d.ts +20 -0
  39. package/dist/opDecompressor.d.ts.map +1 -0
  40. package/dist/opDecompressor.js +72 -0
  41. package/dist/opDecompressor.js.map +1 -0
  42. package/dist/packageVersion.d.ts +1 -1
  43. package/dist/packageVersion.js +1 -1
  44. package/dist/packageVersion.js.map +1 -1
  45. package/dist/pendingStateManager.d.ts +6 -26
  46. package/dist/pendingStateManager.d.ts.map +1 -1
  47. package/dist/pendingStateManager.js +42 -62
  48. package/dist/pendingStateManager.js.map +1 -1
  49. package/dist/runningSummarizer.d.ts +3 -2
  50. package/dist/runningSummarizer.d.ts.map +1 -1
  51. package/dist/runningSummarizer.js +10 -3
  52. package/dist/runningSummarizer.js.map +1 -1
  53. package/dist/scheduleManager.js.map +1 -1
  54. package/dist/summarizer.js +7 -2
  55. package/dist/summarizer.js.map +1 -1
  56. package/dist/summarizerClientElection.js +1 -1
  57. package/dist/summarizerClientElection.js.map +1 -1
  58. package/dist/summarizerHeuristics.d.ts.map +1 -1
  59. package/dist/summarizerHeuristics.js +0 -3
  60. package/dist/summarizerHeuristics.js.map +1 -1
  61. package/dist/summarizerTypes.d.ts +19 -2
  62. package/dist/summarizerTypes.d.ts.map +1 -1
  63. package/dist/summarizerTypes.js.map +1 -1
  64. package/dist/summaryFormat.d.ts +4 -2
  65. package/dist/summaryFormat.d.ts.map +1 -1
  66. package/dist/summaryFormat.js.map +1 -1
  67. package/dist/summaryGenerator.d.ts.map +1 -1
  68. package/dist/summaryGenerator.js +3 -2
  69. package/dist/summaryGenerator.js.map +1 -1
  70. package/dist/summaryManager.d.ts.map +1 -1
  71. package/dist/summaryManager.js +10 -6
  72. package/dist/summaryManager.js.map +1 -1
  73. package/garbageCollection.md +27 -22
  74. package/lib/batchManager.d.ts +11 -6
  75. package/lib/batchManager.d.ts.map +1 -1
  76. package/lib/batchManager.js +23 -13
  77. package/lib/batchManager.js.map +1 -1
  78. package/lib/containerRuntime.d.ts +74 -20
  79. package/lib/containerRuntime.d.ts.map +1 -1
  80. package/lib/containerRuntime.js +189 -136
  81. package/lib/containerRuntime.js.map +1 -1
  82. package/lib/dataStore.d.ts.map +1 -1
  83. package/lib/dataStore.js +6 -0
  84. package/lib/dataStore.js.map +1 -1
  85. package/lib/dataStoreContext.d.ts +14 -21
  86. package/lib/dataStoreContext.d.ts.map +1 -1
  87. package/lib/dataStoreContext.js +75 -61
  88. package/lib/dataStoreContext.js.map +1 -1
  89. package/lib/dataStoreContexts.js +1 -1
  90. package/lib/dataStoreContexts.js.map +1 -1
  91. package/lib/dataStores.d.ts +11 -10
  92. package/lib/dataStores.d.ts.map +1 -1
  93. package/lib/dataStores.js +53 -23
  94. package/lib/dataStores.js.map +1 -1
  95. package/lib/garbageCollection.d.ts +36 -19
  96. package/lib/garbageCollection.d.ts.map +1 -1
  97. package/lib/garbageCollection.js +208 -122
  98. package/lib/garbageCollection.js.map +1 -1
  99. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
  100. package/lib/gcSweepReadyUsageDetection.js +3 -12
  101. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  102. package/lib/index.d.ts +4 -6
  103. package/lib/index.d.ts.map +1 -1
  104. package/lib/index.js +2 -4
  105. package/lib/index.js.map +1 -1
  106. package/lib/opCompressor.d.ts +18 -0
  107. package/lib/opCompressor.d.ts.map +1 -0
  108. package/lib/opCompressor.js +46 -0
  109. package/lib/opCompressor.js.map +1 -0
  110. package/lib/opDecompressor.d.ts +20 -0
  111. package/lib/opDecompressor.d.ts.map +1 -0
  112. package/lib/opDecompressor.js +68 -0
  113. package/lib/opDecompressor.js.map +1 -0
  114. package/lib/packageVersion.d.ts +1 -1
  115. package/lib/packageVersion.js +1 -1
  116. package/lib/packageVersion.js.map +1 -1
  117. package/lib/pendingStateManager.d.ts +6 -26
  118. package/lib/pendingStateManager.d.ts.map +1 -1
  119. package/lib/pendingStateManager.js +42 -62
  120. package/lib/pendingStateManager.js.map +1 -1
  121. package/lib/runningSummarizer.d.ts +3 -2
  122. package/lib/runningSummarizer.d.ts.map +1 -1
  123. package/lib/runningSummarizer.js +10 -3
  124. package/lib/runningSummarizer.js.map +1 -1
  125. package/lib/scheduleManager.js.map +1 -1
  126. package/lib/summarizer.js +7 -2
  127. package/lib/summarizer.js.map +1 -1
  128. package/lib/summarizerClientElection.js +1 -1
  129. package/lib/summarizerClientElection.js.map +1 -1
  130. package/lib/summarizerHeuristics.d.ts.map +1 -1
  131. package/lib/summarizerHeuristics.js +0 -3
  132. package/lib/summarizerHeuristics.js.map +1 -1
  133. package/lib/summarizerTypes.d.ts +19 -2
  134. package/lib/summarizerTypes.d.ts.map +1 -1
  135. package/lib/summarizerTypes.js.map +1 -1
  136. package/lib/summaryFormat.d.ts +4 -2
  137. package/lib/summaryFormat.d.ts.map +1 -1
  138. package/lib/summaryFormat.js.map +1 -1
  139. package/lib/summaryGenerator.d.ts.map +1 -1
  140. package/lib/summaryGenerator.js +3 -2
  141. package/lib/summaryGenerator.js.map +1 -1
  142. package/lib/summaryManager.d.ts.map +1 -1
  143. package/lib/summaryManager.js +10 -6
  144. package/lib/summaryManager.js.map +1 -1
  145. package/package.json +37 -63
  146. package/prettier.config.cjs +8 -0
  147. package/src/batchManager.ts +32 -15
  148. package/src/containerRuntime.ts +260 -156
  149. package/src/dataStore.ts +13 -1
  150. package/src/dataStoreContext.ts +100 -76
  151. package/src/dataStoreContexts.ts +1 -1
  152. package/src/dataStores.ts +61 -23
  153. package/src/garbageCollection.ts +257 -126
  154. package/src/gcSweepReadyUsageDetection.ts +2 -10
  155. package/src/index.ts +4 -4
  156. package/src/opCompressor.ts +59 -0
  157. package/src/opDecompressor.ts +82 -0
  158. package/src/packageVersion.ts +1 -1
  159. package/src/pendingStateManager.ts +57 -96
  160. package/src/runningSummarizer.ts +11 -3
  161. package/src/scheduleManager.ts +1 -0
  162. package/src/summarizer.ts +6 -6
  163. package/src/summarizerClientElection.ts +1 -1
  164. package/src/summarizerHeuristics.ts +0 -3
  165. package/src/summarizerTypes.ts +20 -7
  166. package/src/summaryFormat.ts +4 -2
  167. package/src/summaryGenerator.ts +3 -2
  168. package/src/summaryManager.ts +18 -7
@@ -74,16 +74,8 @@ export class SweepReadyUsageDetectionHandler {
74
74
  localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
75
75
  ) {
76
76
  const noopStorage = { getItem: () => null, setItem: () => {} };
77
- if (localStorageOverride !== undefined) {
78
- this.localStorage = localStorageOverride;
79
- } else {
80
- try {
81
- // localStorage is not defined in Node environment so this throws
82
- this.localStorage = localStorage ?? noopStorage;
83
- } catch (error) {
84
- this.localStorage = noopStorage;
85
- }
86
- }
77
+ // localStorage is not defined in Node environment, so fall back to noopStorage if needed.
78
+ this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
87
79
 
88
80
  if (this.localStorage === noopStorage) {
89
81
  // This means the Skip Closure Period logic will not work.
package/src/index.ts CHANGED
@@ -23,23 +23,22 @@ export {
23
23
  RuntimeHeaders,
24
24
  ISummaryConfiguration,
25
25
  DefaultSummaryConfiguration,
26
+ ICompressionRuntimeOptions,
27
+ CompressionAlgorithms,
26
28
  } from "./containerRuntime";
27
- export { DeltaScheduler } from "./deltaScheduler";
28
29
  export { FluidDataStoreRegistry } from "./dataStoreRegistry";
29
30
  export {
30
31
  gcBlobPrefix,
32
+ gcTombstoneBlobKey,
31
33
  gcTreeKey,
32
- IGarbageCollectionRuntime,
33
34
  IGCStats,
34
35
  } from "./garbageCollection";
35
36
  export {
36
37
  IPendingFlush,
37
- IPendingFlushMode,
38
38
  IPendingLocalState,
39
39
  IPendingMessage,
40
40
  IPendingState,
41
41
  } from "./pendingStateManager";
42
- export { ScheduleManager } from "./scheduleManager";
43
42
  export { Summarizer } from "./summarizer";
44
43
  export {
45
44
  EnqueueSummarizeResult,
@@ -54,6 +53,7 @@ export {
54
53
  INackSummaryResult,
55
54
  IOnDemandSummarizeOptions,
56
55
  IProvideSummarizer,
56
+ IRefreshSummaryAckOptions,
57
57
  ISubmitSummaryOpResult,
58
58
  ISubmitSummaryOptions,
59
59
  ISummarizeOptions,
@@ -0,0 +1,59 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
+ import { IsoBuffer } from "@fluidframework/common-utils";
8
+ import { ChildLogger } from "@fluidframework/telemetry-utils";
9
+ import { compress } from "lz4js";
10
+ import { BatchMessage } from "./batchManager";
11
+ import { CompressionAlgorithms, ContainerRuntimeMessage } from "./containerRuntime";
12
+
13
+ /**
14
+ * Compresses batches of ops. It generates a single compressed op that contains
15
+ * the contents of each op in the batch. It then submits empty ops for each original
16
+ * op to reserve sequence numbers.
17
+ */
18
+ export class OpCompressor {
19
+ private readonly logger;
20
+ private compressedBatchCount = 0;
21
+
22
+ constructor(logger: ITelemetryLogger) {
23
+ this.logger = ChildLogger.create(logger, "OpCompressor");
24
+ }
25
+
26
+ public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {
27
+ const batchToSend: BatchMessage[] = [];
28
+ this.compressedBatchCount++;
29
+ const batchedContents: ContainerRuntimeMessage[] = [];
30
+ for (const message of batch) {
31
+ batchedContents.push(message.deserializedContent);
32
+ }
33
+
34
+ const compressionStart = Date.now();
35
+ const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));
36
+ const compressedContents = compress(contentsAsBuffer);
37
+ const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
38
+ const duration = Date.now() - compressionStart;
39
+
40
+ if (originalLength > 200000 || this.compressedBatchCount % 100) {
41
+ this.logger.sendPerformanceEvent({
42
+ eventName: "CompressedBatch",
43
+ duration,
44
+ sizeBeforeCompression: originalLength,
45
+ sizeAfterCompression: compressedContent.length,
46
+ });
47
+ }
48
+
49
+ batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),
50
+ metadata: { ...batch[0].metadata, compressed: true },
51
+ compression: CompressionAlgorithms.lz4 });
52
+
53
+ for (const message of batch.slice(1)) {
54
+ batchToSend.push({ ...message, contents: undefined });
55
+ }
56
+
57
+ return batchToSend;
58
+ }
59
+ }
@@ -0,0 +1,82 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { decompress } from "lz4js";
7
+ import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
8
+ import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
9
+ import { CompressionAlgorithms } from ".";
10
+
11
+ /**
12
+ * State machine that "unrolls" contents of compressed batches of ops after decompressing them.
13
+ * This class relies on some implicit contracts defined below:
14
+ * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
15
+ * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
16
+ * 3. The final message of a batch will have batch metadata set to false
17
+ * 4. An individually compressed op will have undefined batch metadata and compression set to true
18
+ */
19
+ export class OpDecompressor {
20
+ private activeBatch = false;
21
+ private rootMessageContents: any | undefined;
22
+ private processedCount = 0;
23
+
24
+ public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
25
+ // We're checking for compression = true or top level compression property so
26
+ // that we can enable compression without waiting on all ordering services
27
+ // to pick up protocol change. Eventually only the top level property should
28
+ // be used.
29
+ if (message.metadata?.batch === true
30
+ && (message.metadata?.compressed || message.compression !== undefined)) {
31
+ // Beginning of a compressed batch
32
+ assert(this.activeBatch === false, "shouldn't have multiple active batches");
33
+ if (message.compression) {
34
+ // lz4 is the only supported compression algorithm for now
35
+ assert(message.compression === CompressionAlgorithms.lz4,
36
+ "lz4 is currently the only supported compression algorithm");
37
+ }
38
+
39
+ this.activeBatch = true;
40
+
41
+ const contents = IsoBuffer.from(message.contents.packedContents, "base64");
42
+ const decompressedMessage = decompress(contents);
43
+ const intoString = Uint8ArrayToString(decompressedMessage);
44
+ const asObj = JSON.parse(intoString);
45
+ this.rootMessageContents = asObj;
46
+
47
+ return { ...message, contents: this.rootMessageContents[this.processedCount++] };
48
+ }
49
+
50
+ if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
51
+ // Continuation of compressed batch
52
+ return { ...message, contents: this.rootMessageContents[this.processedCount++] };
53
+ }
54
+
55
+ if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
56
+ // End of compressed batch
57
+ const returnMessage = { ...message,
58
+ contents: this.rootMessageContents[this.processedCount++] };
59
+
60
+ this.activeBatch = false;
61
+ this.rootMessageContents = undefined;
62
+ this.processedCount = 0;
63
+
64
+ return returnMessage;
65
+ }
66
+
67
+ if (message.metadata?.batch === undefined &&
68
+ (message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
69
+ // Single compressed message
70
+ assert(this.activeBatch === false, "shouldn't receive compressed message in middle of a batch");
71
+
72
+ const contents = IsoBuffer.from(message.contents.packedContents, "base64");
73
+ const decompressedMessage = decompress(contents);
74
+ const intoString = new TextDecoder().decode(decompressedMessage);
75
+ const asObj = JSON.parse(intoString);
76
+
77
+ return { ...message, contents: asObj[0] };
78
+ }
79
+
80
+ return message;
81
+ }
82
+ }
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.0.0-dev.1.4.5.105745";
9
+ export const pkgVersion = "2.0.0-dev.2.2.0.111723";
@@ -10,7 +10,6 @@ import { DataProcessingError } from "@fluidframework/container-utils";
10
10
  import {
11
11
  ISequencedDocumentMessage,
12
12
  } from "@fluidframework/protocol-definitions";
13
- import { FlushMode } from "@fluidframework/runtime-definitions";
14
13
  import Deque from "double-ended-queue";
15
14
  import { ContainerMessageType } from "./containerRuntime";
16
15
  import { pkgVersion } from "./packageVersion";
@@ -29,15 +28,6 @@ export interface IPendingMessage {
29
28
  opMetadata: Record<string, unknown> | undefined;
30
29
  }
31
30
 
32
- /**
33
- * This represents a FlushMode update and is added to the pending queue when `setFlushMode` is called on the
34
- * ContainerRuntime and the FlushMode changes.
35
- */
36
- export interface IPendingFlushMode {
37
- type: "flushMode";
38
- flushMode: FlushMode;
39
- }
40
-
41
31
  /**
42
32
  * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
43
33
  * to flush pending messages.
@@ -46,7 +36,7 @@ export interface IPendingFlush {
46
36
  type: "flush";
47
37
  }
48
38
 
49
- export type IPendingState = IPendingMessage | IPendingFlushMode | IPendingFlush;
39
+ export type IPendingState = IPendingMessage | IPendingFlush;
50
40
 
51
41
  export interface IPendingLocalState {
52
42
  /**
@@ -58,8 +48,6 @@ export interface IPendingLocalState {
58
48
  export interface IRuntimeStateHandler{
59
49
  connected(): boolean;
60
50
  clientId(): string | undefined;
61
- flushMode(): FlushMode;
62
- setFlushMode(mode: FlushMode): void;
63
51
  close(error?: ICriticalContainerError): void;
64
52
  applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
65
53
  flush(): void;
@@ -68,13 +56,18 @@ export interface IRuntimeStateHandler{
68
56
  content: any,
69
57
  localOpMetadata: unknown,
70
58
  opMetadata: Record<string, unknown> | undefined): void;
59
+ rollback(
60
+ type: ContainerMessageType,
61
+ content: any,
62
+ localOpMetadata: unknown): void;
63
+ orderSequentially(callback: () => void): void;
71
64
  }
72
65
 
73
66
  /**
74
67
  * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
75
68
  * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
76
69
  * batches along with the messages.
77
- * When the Container reconnects, it replays the pending states, which includes setting the FlushMode, manual flushing
70
+ * When the Container reconnects, it replays the pending states, which includes manual flushing
78
71
  * of messages and triggering resubmission of unacked ops.
79
72
  *
80
73
  * It verifies that all the ops are acked, are received in the right order and batch information is correct.
@@ -100,13 +93,6 @@ export class PendingStateManager implements IDisposable {
100
93
  // the correct batch metadata.
101
94
  private pendingBatchBeginMessage: ISequencedDocumentMessage | undefined;
102
95
 
103
- /**
104
- * This tracks the flush mode for the next message in the pending state queue. When replaying messages, we need to
105
- * first set the flush mode to this value and then send ops. It is important to do this info because the flush
106
- * mode could have been updated.
107
- */
108
- private flushModeForNextMessage: FlushMode;
109
-
110
96
  private clientId: string | undefined;
111
97
 
112
98
  /**
@@ -131,13 +117,9 @@ export class PendingStateManager implements IDisposable {
131
117
 
132
118
  constructor(
133
119
  private readonly stateHandler: IRuntimeStateHandler,
134
- initialFlushMode: FlushMode,
135
120
  initialLocalState: IPendingLocalState | undefined,
136
121
  ) {
137
122
  this.initialStates = new Deque<IPendingState>(initialLocalState?.pendingStates ?? []);
138
-
139
- this.flushModeForNextMessage = initialFlushMode;
140
- this.onFlushModeUpdated(initialFlushMode);
141
123
  }
142
124
 
143
125
  public get disposed() { return this.disposeOnce.evaluated; }
@@ -174,24 +156,10 @@ export class PendingStateManager implements IDisposable {
174
156
  this._pendingMessagesCount++;
175
157
  }
176
158
 
177
- /**
178
- * Called when the FlushMode is updated. Adds the FlushMode to the pending state queue.
179
- * @param flushMode - The flushMode that was updated.
180
- */
181
- public onFlushModeUpdated(flushMode: FlushMode) {
182
- this.pendingStates.push({ type: "flushMode", flushMode });
183
- }
184
-
185
159
  /**
186
160
  * Called when flush() is called on the ContainerRuntime to manually flush messages.
187
161
  */
188
162
  public onFlush() {
189
- // If the FlushMode is Immediate, we don't need to track an explicit flush call because every message is
190
- // automatically flushed. So, flush is a no-op.
191
- if (this.stateHandler.flushMode() === FlushMode.Immediate) {
192
- return;
193
- }
194
-
195
163
  // If the previous state is not a message, flush is a no-op.
196
164
  const previousState = this.pendingStates.peekBack();
197
165
  if (previousState?.type !== "message") {
@@ -279,11 +247,6 @@ export class PendingStateManager implements IDisposable {
279
247
  * @param message - The message that is being processed.
280
248
  */
281
249
  private maybeProcessBatchBegin(message: ISequencedDocumentMessage) {
282
- // Tracks the last FlushMode that was set before this message was sent.
283
- let pendingFlushMode: FlushMode | undefined;
284
- // Tracks whether a flush was called before this message was sent.
285
- let pendingFlush: boolean = false;
286
-
287
250
  /**
288
251
  * We are checking if the next message is the start of a batch. It can happen in the following scenarios:
289
252
  *
@@ -295,34 +258,12 @@ export class PendingStateManager implements IDisposable {
295
258
  * Keep reading pending states from the queue until we encounter a message. It's possible that the FlushMode was
296
259
  * updated a bunch of times without sending any messages.
297
260
  */
298
- let nextPendingState = this.peekNextPendingState();
299
- while (nextPendingState.type !== "message") {
300
- if (nextPendingState.type === "flushMode") {
301
- pendingFlushMode = nextPendingState.flushMode;
302
- }
303
- if (nextPendingState.type === "flush") {
304
- pendingFlush = true;
305
- }
261
+ while (this.peekNextPendingState().type !== "message") {
306
262
  this.pendingStates.shift();
307
- nextPendingState = this.peekNextPendingState();
308
263
  }
309
264
 
310
- if (pendingFlushMode !== undefined) {
311
- this.flushModeForNextMessage = pendingFlushMode;
312
- }
313
-
314
- // If the FlushMode was set to Immediate before this message was sent, this message won't be a batch message
315
- // because in Immediate mode, every message is flushed individually.
316
- if (pendingFlushMode === FlushMode.Immediate) {
317
- return;
318
- }
319
-
320
- /**
321
- * This message is the first in a batch if before it was sent either the FlushMode was set to TurnBased or there
322
- * was an explicit flush call. Note that a flush call is tracked only in TurnBased mode and it indicates the end
323
- * of one batch and beginning of another.
324
- */
325
- if (pendingFlushMode === FlushMode.TurnBased || pendingFlush) {
265
+ // This message is the first in a batch if the "batch" property on the metadata is set to true
266
+ if (message.metadata?.batch) {
326
267
  // We should not already be processing a batch and there should be no pending batch begin message.
327
268
  assert(!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined,
328
269
  0x16b /* "The pending batch state indicates we are already processing a batch" */);
@@ -347,16 +288,6 @@ export class PendingStateManager implements IDisposable {
347
288
  return;
348
289
  }
349
290
 
350
- /**
351
- * We are in the middle of processing a batch. The batch ends when we see an explicit flush. We should never see
352
- * a FlushMode before flush. This is true because we track batches only when FlushMode is TurnBased and in this
353
- * mode, a batch ends either by calling flush or by changing the mode to Immediate which also triggers a flush.
354
- */
355
- assert(
356
- nextPendingState.type !== "flushMode",
357
- 0x2bd /* "We should not see a pending FlushMode until we see a flush when processing a batch" */,
358
- );
359
-
360
291
  // There should be a pending batch begin message.
361
292
  assert(this.pendingBatchBeginMessage !== undefined, 0x16d /* "There is no pending batch begin message" */);
362
293
 
@@ -406,7 +337,7 @@ export class PendingStateManager implements IDisposable {
406
337
 
407
338
  /**
408
339
  * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
409
- * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.
340
+ * states in its queue. This includes triggering resubmission of unacked ops.
410
341
  */
411
342
  public replayPendingStates() {
412
343
  assert(this.stateHandler.connected(), 0x172 /* "The connection state is not consistent with the runtime" */);
@@ -426,12 +357,7 @@ export class PendingStateManager implements IDisposable {
426
357
  // Reset the pending message count because all these messages will be removed from the queue.
427
358
  this._pendingMessagesCount = 0;
428
359
 
429
- // Save the current FlushMode so that we can revert it back after replaying the states.
430
- const savedFlushMode = this.stateHandler.flushMode();
431
-
432
- // Set the flush mode for the next message. This step is important because the flush mode may have been changed
433
- // after the next pending message was sent.
434
- this.stateHandler.setFlushMode(this.flushModeForNextMessage);
360
+ const messageBatchQueue = new Deque<IPendingMessage>();
435
361
 
436
362
  // Process exactly `pendingStatesCount` items in the queue as it represents the number of states that were
437
363
  // pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
@@ -441,16 +367,43 @@ export class PendingStateManager implements IDisposable {
441
367
  const pendingState = this.pendingStates.shift()!;
442
368
  switch (pendingState.type) {
443
369
  case "message":
444
- this.stateHandler.reSubmit(
445
- pendingState.messageType,
446
- pendingState.content,
447
- pendingState.localOpMetadata,
448
- pendingState.opMetadata);
449
- break;
450
- case "flushMode":
451
- this.stateHandler.setFlushMode(pendingState.flushMode);
370
+ assert(pendingState.opMetadata?.batch !== false || messageBatchQueue.length > 0,
371
+ 0x41b /* We cannot process batches in chunks */);
372
+ /**
373
+ * We want to ensure grouped messages get processed in a batch.
374
+ * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
375
+ * either receive the whole batch ack or nothing at all.
376
+ */
377
+ if (messageBatchQueue.length > 0 || pendingState.opMetadata?.batch) {
378
+ messageBatchQueue.enqueue(pendingState);
379
+ } else {
380
+ this.stateHandler.reSubmit(
381
+ pendingState.messageType,
382
+ pendingState.content,
383
+ pendingState.localOpMetadata,
384
+ pendingState.opMetadata);
385
+ }
452
386
  break;
453
387
  case "flush":
388
+ /**
389
+ * A "flush" call can indicate the end of a batch.
390
+ * We can't rely on the "batch" property in the message metadata as it gets
391
+ * updated elsewhere and it is not the same object instance that gets updated.
392
+ */
393
+ if (messageBatchQueue.length > 0) {
394
+ this.stateHandler.orderSequentially(() => {
395
+ while (messageBatchQueue.length > 0) {
396
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
397
+ const message = messageBatchQueue.dequeue()!;
398
+ this.stateHandler.reSubmit(
399
+ message.messageType,
400
+ message.content,
401
+ message.localOpMetadata,
402
+ message.opMetadata);
403
+ }
404
+ });
405
+ }
406
+ assert(messageBatchQueue.length === 0, 0x41c /* cannot flush in the middle of a batch */);
454
407
  this.stateHandler.flush();
455
408
  break;
456
409
  default:
@@ -459,7 +412,15 @@ export class PendingStateManager implements IDisposable {
459
412
  pendingStatesCount--;
460
413
  }
461
414
 
462
- // Revert the FlushMode.
463
- this.stateHandler.setFlushMode(savedFlushMode);
415
+ // There are some cases where ops are stashed but not flushed. We need to ensure they are resubmitted
416
+ while (messageBatchQueue.length > 0) {
417
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
418
+ const message = messageBatchQueue.dequeue()!;
419
+ this.stateHandler.reSubmit(
420
+ message.messageType,
421
+ message.content,
422
+ message.localOpMetadata,
423
+ message.opMetadata);
424
+ }
464
425
  }
465
426
  }
@@ -263,9 +263,9 @@ export class RunningSummarizer implements IDisposable {
263
263
 
264
264
  /**
265
265
  * Can the given op trigger a summary?
266
- * # Currently only prevents summaries for Summarize and SummaryAck ops
266
+ * # Currently always prevents summaries for Summarize and SummaryAck/Nack ops
267
267
  * @param op - op to check
268
- * @returns true if this type of op can trigger a summary
268
+ * @returns true if this op can trigger a summary
269
269
  */
270
270
  private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
271
271
  switch (op.type) {
@@ -274,10 +274,18 @@ export class RunningSummarizer implements IDisposable {
274
274
  case MessageType.SummaryNack:
275
275
  return false;
276
276
  default:
277
- return true;
277
+ return isRuntimeMessage(op) || this.nonRuntimeOpCanTriggerSummary();
278
278
  }
279
279
  }
280
280
 
281
+ private nonRuntimeOpCanTriggerSummary(): boolean {
282
+ // eslint-disable-next-line max-len
283
+ const opsSinceLastAck = this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
284
+ return this.configuration.state === "enabled"
285
+ && (this.configuration.nonRuntimeHeuristicThreshold === undefined
286
+ || this.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck);
287
+ }
288
+
281
289
  public async waitStop(allowLastSummary: boolean): Promise<void> {
282
290
  if (this.stopping) {
283
291
  return;
@@ -221,6 +221,7 @@ class ScheduleManagerCore {
221
221
  length: endBatch - startBatch,
222
222
  });
223
223
  }
224
+
224
225
  this.deltaManager.inbound.resume();
225
226
  }
226
227
 
package/src/summarizer.ts CHANGED
@@ -334,7 +334,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
334
334
  const coordinatorCreateP = this.runCoordinatorCreateFn(this.runtime);
335
335
 
336
336
  coordinatorCreateP.then((runCoordinator) => {
337
- // Successully created the cancellation token. Start the summarizer.
337
+ // Successfully created the cancellation token. Start the summarizer.
338
338
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
339
339
  const startP = this.start(this.runtime.clientId!, runCoordinator);
340
340
  startP.then(async (runningSummarizer) => {
@@ -382,11 +382,11 @@ export class Summarizer extends EventEmitter implements ISummarizer {
382
382
  // executing the refreshLatestSummaryAck.
383
383
  // https://dev.azure.com/fluidframework/internal/_workitems/edit/779
384
384
  await this.runningSummarizer.lockedRefreshSummaryAckAction(async () =>
385
- this.internalsProvider.refreshLatestSummaryAck(
386
- summaryOpHandle,
387
- summaryAckHandle,
388
- refSequenceNumber,
389
- summaryLogger,
385
+ this.internalsProvider.refreshLatestSummaryAck({
386
+ proposalHandle: summaryOpHandle,
387
+ ackHandle: summaryAckHandle,
388
+ summaryRefSeq: refSequenceNumber,
389
+ summaryLogger },
390
390
  ).catch(async (error) => {
391
391
  // If the error is 404, so maybe the fetched version no longer exists on server. We just
392
392
  // ignore this error in that case, as that means we will have another summaryAck for the
@@ -77,7 +77,7 @@ export class SummarizerClientElection
77
77
  // Log and elect a new summarizer client.
78
78
  const opsSinceLastReport = sequenceNumber - this.lastReportedSeq;
79
79
  if (opsSinceLastReport > this.maxOpsSinceLastSummary) {
80
- this.logger.sendErrorEvent({
80
+ this.logger.sendTelemetryEvent({
81
81
  eventName: "ElectedClientNotSummarizing",
82
82
  electedClientId,
83
83
  lastSummaryAckSeqForClient: this.lastSummaryAckSeqForClient,
@@ -119,9 +119,6 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
119
119
  }
120
120
 
121
121
  public get idleTime(): number {
122
- if (this.configuration.idleTime !== undefined) {
123
- return this.configuration.idleTime;
124
- }
125
122
  const maxIdleTime = this.configuration.maxIdleTime;
126
123
  const minIdleTime = this.configuration.minIdleTime;
127
124
  const weightedNumOfOps = getWeightedNumberOfOps(
@@ -61,12 +61,7 @@ export interface ISummarizerInternalsProvider {
61
61
  submitSummary(options: ISubmitSummaryOptions): Promise<SubmitSummaryResult>;
62
62
 
63
63
  /** Callback whenever a new SummaryAck is received, to update internal tracking state */
64
- refreshLatestSummaryAck(
65
- proposalHandle: string,
66
- ackHandle: string,
67
- summaryRefSeq: number,
68
- summaryLogger: ITelemetryLogger,
69
- ): Promise<void>;
64
+ refreshLatestSummaryAck(options: IRefreshSummaryAckOptions): Promise<void>;
70
65
  }
71
66
 
72
67
  /**
@@ -114,6 +109,20 @@ export interface ISummarizeOptions {
114
109
  readonly refreshLatestAck?: boolean;
115
110
  }
116
111
 
112
+ /**
113
+ * Data required to update internal tracking state after receiving a Summary Ack.
114
+ */
115
+ export interface IRefreshSummaryAckOptions {
116
+ /** Handle from the ack's summary op. */
117
+ readonly proposalHandle: string | undefined;
118
+ /** Handle from the summary ack just received */
119
+ readonly ackHandle: string;
120
+ /** Reference sequence number from the ack's summary op */
121
+ readonly summaryRefSeq: number;
122
+ /** Telemetry logger to which telemetry events will be forwarded. */
123
+ readonly summaryLogger: ITelemetryLogger;
124
+ }
125
+
117
126
  export interface ISubmitSummaryOptions extends ISummarizeOptions {
118
127
  /** Logger to use for correlated summary events */
119
128
  readonly summaryLogger: ITelemetryLogger;
@@ -292,7 +301,11 @@ export type SummarizerStopReason =
292
301
  * client to no longer be elected as responsible for summaries. Then it
293
302
  * tries to stop its spawned summarizer client.
294
303
  */
295
- | "parentShouldNotSummarize"
304
+ | "notElectedParent"
305
+ /**
306
+ * We are not already running the summarizer and we are not the current elected client id.
307
+ */
308
+ | "notElectedClient"
296
309
  /** Summarizer client was disconnected */
297
310
  | "summarizerClientDisconnected"
298
311
  /* running summarizer threw an exception */
@@ -99,14 +99,16 @@ export interface IGCMetadata {
99
99
  * - A value greater than 0 means GC is enabled.
100
100
  */
101
101
  readonly gcFeature?: GCVersion;
102
- /** If this is present, the session for this container will expire after this time and the container will close */
103
- readonly sessionExpiryTimeoutMs?: number;
104
102
  /**
105
103
  * Tells whether the GC sweep phase is enabled for this container.
106
104
  * - True means sweep phase is enabled.
107
105
  * - False means sweep phase is disabled. If GC is disabled as per gcFeature, sweep is also disabled.
108
106
  */
109
107
  readonly sweepEnabled?: boolean;
108
+ /** If this is present, the session for this container will expire after this time and the container will close */
109
+ readonly sessionExpiryTimeoutMs?: number;
110
+ /** How long to wait after an object is unreferenced before deleting it via GC Sweep */
111
+ readonly sweepTimeoutMs?: number;
110
112
  }
111
113
 
112
114
  /** The properties of an ISequencedDocumentMessage to be stored in the metadata blob in summary. */
@@ -231,13 +231,14 @@ export class SummaryGenerator {
231
231
  const category = cancellationToken.cancelled || error?.errorType === DriverErrorType.offlineError ?
232
232
  "generic" : "error";
233
233
 
234
+ const message = getFailMessage(errorCode);
234
235
  summarizeEvent.cancel({
235
236
  ...properties,
236
237
  reason: errorCode,
237
238
  category,
238
239
  retryAfterSeconds,
239
- }, error);
240
- resultsBuilder.fail(getFailMessage(errorCode), error, nackSummaryResult, retryAfterSeconds);
240
+ }, error ?? message); // disconnect & summaryAckTimeout do not have proper error.
241
+ resultsBuilder.fail(message, error, nackSummaryResult, retryAfterSeconds);
241
242
  };
242
243
 
243
244
  // Wait to generate and send summary