@fluidframework/container-runtime 2.0.0-dev.1.4.6.106135 → 2.0.0-dev.2.3.0.115467

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (239) hide show
  1. package/.eslintrc.js +1 -1
  2. package/dist/blobManager.d.ts +20 -5
  3. package/dist/blobManager.d.ts.map +1 -1
  4. package/dist/blobManager.js +57 -15
  5. package/dist/blobManager.js.map +1 -1
  6. package/dist/containerRuntime.d.ts +88 -51
  7. package/dist/containerRuntime.d.ts.map +1 -1
  8. package/dist/containerRuntime.js +205 -300
  9. package/dist/containerRuntime.js.map +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +6 -0
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +14 -21
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +71 -57
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStoreContexts.js +1 -1
  18. package/dist/dataStoreContexts.js.map +1 -1
  19. package/dist/dataStores.d.ts +11 -10
  20. package/dist/dataStores.d.ts.map +1 -1
  21. package/dist/dataStores.js +51 -20
  22. package/dist/dataStores.js.map +1 -1
  23. package/dist/garbageCollection.d.ts +40 -32
  24. package/dist/garbageCollection.d.ts.map +1 -1
  25. package/dist/garbageCollection.js +227 -161
  26. package/dist/garbageCollection.js.map +1 -1
  27. package/dist/garbageCollectionConstants.d.ts +19 -0
  28. package/dist/garbageCollectionConstants.d.ts.map +1 -0
  29. package/dist/garbageCollectionConstants.js +34 -0
  30. package/dist/garbageCollectionConstants.js.map +1 -0
  31. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
  32. package/dist/gcSweepReadyUsageDetection.js +5 -14
  33. package/dist/gcSweepReadyUsageDetection.js.map +1 -1
  34. package/dist/index.d.ts +6 -6
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/index.js +8 -9
  37. package/dist/index.js.map +1 -1
  38. package/dist/opLifecycle/batchManager.d.ts +30 -0
  39. package/dist/opLifecycle/batchManager.d.ts.map +1 -0
  40. package/dist/{batchManager.js → opLifecycle/batchManager.js} +25 -15
  41. package/dist/opLifecycle/batchManager.js.map +1 -0
  42. package/dist/opLifecycle/definitions.d.ts +40 -0
  43. package/dist/opLifecycle/definitions.d.ts.map +1 -0
  44. package/dist/opLifecycle/definitions.js +7 -0
  45. package/dist/opLifecycle/definitions.js.map +1 -0
  46. package/dist/opLifecycle/index.d.ts +12 -0
  47. package/dist/opLifecycle/index.d.ts.map +1 -0
  48. package/dist/opLifecycle/index.js +21 -0
  49. package/dist/opLifecycle/index.js.map +1 -0
  50. package/dist/opLifecycle/opCompressor.d.ts +18 -0
  51. package/dist/opLifecycle/opCompressor.d.ts.map +1 -0
  52. package/dist/opLifecycle/opCompressor.js +53 -0
  53. package/dist/opLifecycle/opCompressor.js.map +1 -0
  54. package/dist/opLifecycle/opDecompressor.d.ts +20 -0
  55. package/dist/opLifecycle/opDecompressor.d.ts.map +1 -0
  56. package/dist/opLifecycle/opDecompressor.js +72 -0
  57. package/dist/opLifecycle/opDecompressor.js.map +1 -0
  58. package/dist/opLifecycle/opSplitter.d.ts +17 -0
  59. package/dist/opLifecycle/opSplitter.d.ts.map +1 -0
  60. package/dist/opLifecycle/opSplitter.js +61 -0
  61. package/dist/opLifecycle/opSplitter.js.map +1 -0
  62. package/dist/opLifecycle/outbox.d.ts +47 -0
  63. package/dist/opLifecycle/outbox.d.ts.map +1 -0
  64. package/dist/opLifecycle/outbox.js +153 -0
  65. package/dist/opLifecycle/outbox.js.map +1 -0
  66. package/dist/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  67. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  68. package/dist/opLifecycle/remoteMessageProcessor.js +81 -0
  69. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -0
  70. package/dist/packageVersion.d.ts +1 -1
  71. package/dist/packageVersion.js +1 -1
  72. package/dist/packageVersion.js.map +1 -1
  73. package/dist/pendingStateManager.d.ts +6 -26
  74. package/dist/pendingStateManager.d.ts.map +1 -1
  75. package/dist/pendingStateManager.js +42 -62
  76. package/dist/pendingStateManager.js.map +1 -1
  77. package/dist/runningSummarizer.d.ts +3 -2
  78. package/dist/runningSummarizer.d.ts.map +1 -1
  79. package/dist/runningSummarizer.js +10 -3
  80. package/dist/runningSummarizer.js.map +1 -1
  81. package/dist/scheduleManager.js.map +1 -1
  82. package/dist/summarizer.js +7 -2
  83. package/dist/summarizer.js.map +1 -1
  84. package/dist/summarizerClientElection.js +1 -1
  85. package/dist/summarizerClientElection.js.map +1 -1
  86. package/dist/summarizerHeuristics.d.ts.map +1 -1
  87. package/dist/summarizerHeuristics.js +0 -3
  88. package/dist/summarizerHeuristics.js.map +1 -1
  89. package/dist/summarizerTypes.d.ts +19 -2
  90. package/dist/summarizerTypes.d.ts.map +1 -1
  91. package/dist/summarizerTypes.js.map +1 -1
  92. package/dist/summaryFormat.d.ts +4 -2
  93. package/dist/summaryFormat.d.ts.map +1 -1
  94. package/dist/summaryFormat.js +2 -2
  95. package/dist/summaryFormat.js.map +1 -1
  96. package/dist/summaryGenerator.d.ts.map +1 -1
  97. package/dist/summaryGenerator.js +3 -2
  98. package/dist/summaryGenerator.js.map +1 -1
  99. package/dist/summaryManager.d.ts.map +1 -1
  100. package/dist/summaryManager.js +10 -6
  101. package/dist/summaryManager.js.map +1 -1
  102. package/garbageCollection.md +27 -22
  103. package/lib/blobManager.d.ts +20 -5
  104. package/lib/blobManager.d.ts.map +1 -1
  105. package/lib/blobManager.js +59 -17
  106. package/lib/blobManager.js.map +1 -1
  107. package/lib/containerRuntime.d.ts +88 -51
  108. package/lib/containerRuntime.d.ts.map +1 -1
  109. package/lib/containerRuntime.js +203 -297
  110. package/lib/containerRuntime.js.map +1 -1
  111. package/lib/dataStore.d.ts.map +1 -1
  112. package/lib/dataStore.js +6 -0
  113. package/lib/dataStore.js.map +1 -1
  114. package/lib/dataStoreContext.d.ts +14 -21
  115. package/lib/dataStoreContext.d.ts.map +1 -1
  116. package/lib/dataStoreContext.js +75 -61
  117. package/lib/dataStoreContext.js.map +1 -1
  118. package/lib/dataStoreContexts.js +1 -1
  119. package/lib/dataStoreContexts.js.map +1 -1
  120. package/lib/dataStores.d.ts +11 -10
  121. package/lib/dataStores.d.ts.map +1 -1
  122. package/lib/dataStores.js +53 -22
  123. package/lib/dataStores.js.map +1 -1
  124. package/lib/garbageCollection.d.ts +40 -32
  125. package/lib/garbageCollection.d.ts.map +1 -1
  126. package/lib/garbageCollection.js +220 -154
  127. package/lib/garbageCollection.js.map +1 -1
  128. package/lib/garbageCollectionConstants.d.ts +19 -0
  129. package/lib/garbageCollectionConstants.d.ts.map +1 -0
  130. package/lib/garbageCollectionConstants.js +31 -0
  131. package/lib/garbageCollectionConstants.js.map +1 -0
  132. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
  133. package/lib/gcSweepReadyUsageDetection.js +4 -13
  134. package/lib/gcSweepReadyUsageDetection.js.map +1 -1
  135. package/lib/index.d.ts +6 -6
  136. package/lib/index.d.ts.map +1 -1
  137. package/lib/index.js +3 -4
  138. package/lib/index.js.map +1 -1
  139. package/lib/opLifecycle/batchManager.d.ts +30 -0
  140. package/lib/opLifecycle/batchManager.d.ts.map +1 -0
  141. package/lib/{batchManager.js → opLifecycle/batchManager.js} +25 -15
  142. package/lib/opLifecycle/batchManager.js.map +1 -0
  143. package/lib/opLifecycle/definitions.d.ts +40 -0
  144. package/lib/opLifecycle/definitions.d.ts.map +1 -0
  145. package/lib/opLifecycle/definitions.js +6 -0
  146. package/lib/opLifecycle/definitions.js.map +1 -0
  147. package/lib/opLifecycle/index.d.ts +12 -0
  148. package/lib/opLifecycle/index.d.ts.map +1 -0
  149. package/lib/opLifecycle/index.js +11 -0
  150. package/lib/opLifecycle/index.js.map +1 -0
  151. package/lib/opLifecycle/opCompressor.d.ts +18 -0
  152. package/lib/opLifecycle/opCompressor.d.ts.map +1 -0
  153. package/lib/opLifecycle/opCompressor.js +49 -0
  154. package/lib/opLifecycle/opCompressor.js.map +1 -0
  155. package/lib/opLifecycle/opDecompressor.d.ts +20 -0
  156. package/lib/opLifecycle/opDecompressor.d.ts.map +1 -0
  157. package/lib/opLifecycle/opDecompressor.js +68 -0
  158. package/lib/opLifecycle/opDecompressor.js.map +1 -0
  159. package/lib/opLifecycle/opSplitter.d.ts +17 -0
  160. package/lib/opLifecycle/opSplitter.d.ts.map +1 -0
  161. package/lib/opLifecycle/opSplitter.js +57 -0
  162. package/lib/opLifecycle/opSplitter.js.map +1 -0
  163. package/lib/opLifecycle/outbox.d.ts +47 -0
  164. package/lib/opLifecycle/outbox.d.ts.map +1 -0
  165. package/lib/opLifecycle/outbox.js +149 -0
  166. package/lib/opLifecycle/outbox.js.map +1 -0
  167. package/lib/opLifecycle/remoteMessageProcessor.d.ts +26 -0
  168. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -0
  169. package/lib/opLifecycle/remoteMessageProcessor.js +76 -0
  170. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -0
  171. package/lib/packageVersion.d.ts +1 -1
  172. package/lib/packageVersion.js +1 -1
  173. package/lib/packageVersion.js.map +1 -1
  174. package/lib/pendingStateManager.d.ts +6 -26
  175. package/lib/pendingStateManager.d.ts.map +1 -1
  176. package/lib/pendingStateManager.js +42 -62
  177. package/lib/pendingStateManager.js.map +1 -1
  178. package/lib/runningSummarizer.d.ts +3 -2
  179. package/lib/runningSummarizer.d.ts.map +1 -1
  180. package/lib/runningSummarizer.js +10 -3
  181. package/lib/runningSummarizer.js.map +1 -1
  182. package/lib/scheduleManager.js.map +1 -1
  183. package/lib/summarizer.js +7 -2
  184. package/lib/summarizer.js.map +1 -1
  185. package/lib/summarizerClientElection.js +1 -1
  186. package/lib/summarizerClientElection.js.map +1 -1
  187. package/lib/summarizerHeuristics.d.ts.map +1 -1
  188. package/lib/summarizerHeuristics.js +0 -3
  189. package/lib/summarizerHeuristics.js.map +1 -1
  190. package/lib/summarizerTypes.d.ts +19 -2
  191. package/lib/summarizerTypes.d.ts.map +1 -1
  192. package/lib/summarizerTypes.js.map +1 -1
  193. package/lib/summaryFormat.d.ts +4 -2
  194. package/lib/summaryFormat.d.ts.map +1 -1
  195. package/lib/summaryFormat.js +1 -1
  196. package/lib/summaryFormat.js.map +1 -1
  197. package/lib/summaryGenerator.d.ts.map +1 -1
  198. package/lib/summaryGenerator.js +3 -2
  199. package/lib/summaryGenerator.js.map +1 -1
  200. package/lib/summaryManager.d.ts.map +1 -1
  201. package/lib/summaryManager.js +10 -6
  202. package/lib/summaryManager.js.map +1 -1
  203. package/package.json +32 -71
  204. package/prettier.config.cjs +8 -0
  205. package/src/blobManager.ts +74 -19
  206. package/src/containerRuntime.ts +286 -369
  207. package/src/dataStore.ts +13 -1
  208. package/src/dataStoreContext.ts +100 -76
  209. package/src/dataStoreContexts.ts +1 -1
  210. package/src/dataStores.ts +61 -22
  211. package/src/garbageCollection.ts +282 -163
  212. package/src/garbageCollectionConstants.ts +35 -0
  213. package/src/gcSweepReadyUsageDetection.ts +3 -11
  214. package/src/index.ts +9 -8
  215. package/src/{batchManager.ts → opLifecycle/batchManager.ts} +42 -28
  216. package/src/opLifecycle/definitions.ts +44 -0
  217. package/src/opLifecycle/index.ts +17 -0
  218. package/src/opLifecycle/opCompressor.ts +64 -0
  219. package/src/opLifecycle/opDecompressor.ts +84 -0
  220. package/src/opLifecycle/opSplitter.ts +78 -0
  221. package/src/opLifecycle/outbox.ts +204 -0
  222. package/src/opLifecycle/remoteMessageProcessor.ts +90 -0
  223. package/src/packageVersion.ts +1 -1
  224. package/src/pendingStateManager.ts +57 -96
  225. package/src/runningSummarizer.ts +11 -3
  226. package/src/scheduleManager.ts +1 -0
  227. package/src/summarizer.ts +6 -6
  228. package/src/summarizerClientElection.ts +1 -1
  229. package/src/summarizerHeuristics.ts +0 -3
  230. package/src/summarizerTypes.ts +20 -7
  231. package/src/summaryFormat.ts +5 -3
  232. package/src/summaryGenerator.ts +3 -2
  233. package/src/summaryManager.ts +18 -7
  234. package/dist/batchManager.d.ts +0 -37
  235. package/dist/batchManager.d.ts.map +0 -1
  236. package/dist/batchManager.js.map +0 -1
  237. package/lib/batchManager.d.ts +0 -37
  238. package/lib/batchManager.d.ts.map +0 -1
  239. package/lib/batchManager.js.map +0 -1
@@ -0,0 +1,204 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { assert } from "@fluidframework/common-utils";
7
+ import { IContainerContext } from "@fluidframework/container-definitions";
8
+ import { GenericError } from "@fluidframework/container-utils";
9
+ import { MessageType } from "@fluidframework/protocol-definitions";
10
+ import { ICompressionRuntimeOptions } from "../containerRuntime";
11
+ import { PendingStateManager } from "../pendingStateManager";
12
+ import { BatchManager } from "./batchManager";
13
+ import { BatchMessage, IBatch } from "./definitions";
14
+ import { OpCompressor } from "./opCompressor";
15
+
16
+ export interface IOutboxConfig {
17
+ readonly compressionOptions: ICompressionRuntimeOptions;
18
+ // The maximum size of a batch that we can send over the wire.
19
+ readonly maxBatchSizeInBytes: number;
20
+ };
21
+
22
+ export interface IOutboxParameters {
23
+ readonly shouldSend: () => boolean,
24
+ readonly pendingStateManager: PendingStateManager,
25
+ readonly containerContext: IContainerContext,
26
+ readonly config: IOutboxConfig,
27
+ readonly compressor: OpCompressor;
28
+ }
29
+
30
+ export class Outbox {
31
+ private readonly attachFlowBatch: BatchManager;
32
+ private readonly mainBatch: BatchManager;
33
+ private readonly defaultAttachFlowSoftLimitInBytes = 64 * 1024;
34
+
35
+ constructor(private readonly params: IOutboxParameters) {
36
+ const isCompressionEnabled = this.params.config.compressionOptions.minimumBatchSizeInBytes !== Number.POSITIVE_INFINITY;
37
+ // We need to allow infinite size batches if we enable compression
38
+ const hardLimit = isCompressionEnabled ? Infinity : this.params.config.maxBatchSizeInBytes;
39
+ const softLimit = isCompressionEnabled ? Infinity : this.defaultAttachFlowSoftLimitInBytes;
40
+
41
+ this.attachFlowBatch = new BatchManager({
42
+ hardLimit,
43
+ softLimit,
44
+ });
45
+ this.mainBatch = new BatchManager({
46
+ hardLimit
47
+ });
48
+ }
49
+
50
+ public get isEmpty(): boolean {
51
+ return this.attachFlowBatch.length === 0 && this.mainBatch.length === 0;
52
+ }
53
+
54
+ public submit(message: BatchMessage) {
55
+ if (!this.mainBatch.push(message)) {
56
+ throw new GenericError(
57
+ "BatchTooLarge",
58
+ /* error */ undefined,
59
+ {
60
+ opSize: (message.contents?.length) ?? 0,
61
+ count: this.mainBatch.length,
62
+ limit: this.mainBatch.options.hardLimit,
63
+ });
64
+ }
65
+ }
66
+
67
+ public submitAttach(message: BatchMessage) {
68
+ if (!this.attachFlowBatch.push(message)) {
69
+ // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
70
+ // when queue is not empty.
71
+ // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
72
+ this.flushInternal(this.attachFlowBatch.popBatch());
73
+ if (!this.attachFlowBatch.push(message)) {
74
+ throw new GenericError(
75
+ "BatchTooLarge",
76
+ /* error */ undefined,
77
+ {
78
+ opSize: (message.contents?.length) ?? 0,
79
+ count: this.attachFlowBatch.length,
80
+ limit: this.attachFlowBatch.options.hardLimit,
81
+ });
82
+ }
83
+ }
84
+
85
+ // If compression is enabled, we will always successfully receive
86
+ // attach ops and compress then send them at the next JS turn, regardless
87
+ // of the overall size of the accumulated ops in the batch.
88
+ // However, it is more efficient to flush these ops faster, preferably
89
+ // after they reach a size which would benefit from compression.
90
+ if (this.attachFlowBatch.contentSizeInBytes >= this.params.config.compressionOptions.minimumBatchSizeInBytes) {
91
+ this.flushInternal(this.attachFlowBatch.popBatch());
92
+ }
93
+ }
94
+
95
+ public flush() {
96
+ this.flushInternal(this.attachFlowBatch.popBatch());
97
+ this.flushInternal(this.mainBatch.popBatch());
98
+ }
99
+
100
+ private flushInternal(rawBatch: IBatch) {
101
+ const processedBatch = this.compressBatch(rawBatch);
102
+ const clientSequenceNumber = this.sendBatch(processedBatch);
103
+
104
+ this.persistBatch(clientSequenceNumber, rawBatch.content);
105
+ }
106
+
107
+ private compressBatch(batch: IBatch): IBatch {
108
+ if (batch.content.length === 0
109
+ || this.params.config.compressionOptions === undefined
110
+ || this.params.config.compressionOptions.minimumBatchSizeInBytes > batch.contentSizeInBytes) {
111
+ // Nothing to do if the batch is empty or if compression is disabled or if we don't need to compress
112
+ return batch;
113
+ }
114
+
115
+ const compressedBatch = this.params.compressor.compressBatch(batch);
116
+ if (compressedBatch.contentSizeInBytes > this.params.config.maxBatchSizeInBytes) {
117
+ throw new GenericError(
118
+ "BatchTooLarge",
119
+ /* error */ undefined,
120
+ {
121
+ opSize: batch.contentSizeInBytes,
122
+ count: batch.content.length,
123
+ limit: this.params.config.maxBatchSizeInBytes,
124
+ compressed: true,
125
+ });
126
+ }
127
+
128
+ // If we don't reach the maximum supported size of a batch, it safe to be sent as is
129
+ return compressedBatch;
130
+ }
131
+
132
+ /**
133
+ * Sends the batch object to the container context to be sent over the wire.
134
+ *
135
+ * @param batch - batch to be sent
136
+ * @returns the client sequence number of the last batched op which was sent and
137
+ * -1 if there are no ops or the container cannot send ops.
138
+ */
139
+ private sendBatch(batch: IBatch): number {
140
+ let clientSequenceNumber: number = -1;
141
+ const length = batch.content.length;
142
+
143
+ // Did we disconnect in the middle of turn-based batch?
144
+ // If so, do nothing, as pending state manager will resubmit it correctly on reconnect.
145
+ if (length === 0 || !this.params.shouldSend()) {
146
+ return clientSequenceNumber;
147
+ }
148
+
149
+ if (this.params.containerContext.submitBatchFn === undefined) {
150
+ // Legacy path - supporting old loader versions. Can be removed only when LTS moves above
151
+ // version that has support for batches (submitBatchFn)
152
+ for (const message of batch.content) {
153
+ // Legacy path doesn't support compressed payloads and will submit uncompressed payload anyways
154
+ if (message.metadata?.compressed) {
155
+ delete message.metadata.compressed;
156
+ }
157
+
158
+ clientSequenceNumber = this.params.containerContext.submitFn(
159
+ MessageType.Operation,
160
+ message.deserializedContent,
161
+ true, // batch
162
+ message.metadata);
163
+ }
164
+
165
+ this.params.containerContext.deltaManager.flush();
166
+ } else {
167
+ // returns clientSequenceNumber of last message in a batch
168
+ clientSequenceNumber = this.params.containerContext.submitBatchFn(
169
+ batch.content.map((message) => ({ contents: message.contents, metadata: message.metadata })));
170
+ }
171
+
172
+ // Convert from clientSequenceNumber of last message in the batch to clientSequenceNumber of first message.
173
+ clientSequenceNumber -= length - 1;
174
+ assert(clientSequenceNumber >= 0, 0x3d0 /* clientSequenceNumber can't be negative */);
175
+ return clientSequenceNumber;
176
+ }
177
+
178
+ private persistBatch(initialClientSequenceNumber: number, batch: BatchMessage[]) {
179
+ let clientSequenceNumber = initialClientSequenceNumber;
180
+ // Let the PendingStateManager know that a message was submitted.
181
+ // In future, need to shift toward keeping batch as a whole!
182
+ for (const message of batch) {
183
+ this.params.pendingStateManager.onSubmitMessage(
184
+ message.deserializedContent.type,
185
+ clientSequenceNumber,
186
+ message.referenceSequenceNumber,
187
+ message.deserializedContent.contents,
188
+ message.localOpMetadata,
189
+ message.metadata,
190
+ );
191
+
192
+ clientSequenceNumber++;
193
+ }
194
+
195
+ this.params.pendingStateManager.onFlush();
196
+ }
197
+
198
+ public checkpoint() {
199
+ return {
200
+ mainBatch: this.mainBatch.checkpoint(),
201
+ attachFlowBatch: this.attachFlowBatch.checkpoint(),
202
+ };
203
+ }
204
+ }
@@ -0,0 +1,90 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { ISequencedDocumentMessage, MessageType } from "@fluidframework/protocol-definitions";
7
+ import { ContainerMessageType, ContainerRuntimeMessage } from "../containerRuntime";
8
+ import { OpDecompressor } from "./opDecompressor";
9
+ import { OpSplitter } from "./opSplitter";
10
+
11
+ export class RemoteMessageProcessor {
12
+ constructor(
13
+ private readonly opSplitter: OpSplitter,
14
+ private readonly opDecompressor: OpDecompressor,
15
+ ) { }
16
+
17
+ public get partialMessages(): ReadonlyMap<string, string[]> {
18
+ return this.opSplitter.chunks;
19
+ }
20
+
21
+ public clearPartialMessagesFor(clientId: string) {
22
+ this.opSplitter.clearPartialChunks(clientId);
23
+ }
24
+
25
+ public process(remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage {
26
+ let message = copy(remoteMessage);
27
+
28
+ message = this.opDecompressor.processMessage(message);
29
+ unpackRuntimeMessage(message);
30
+ message = this.opSplitter.processRemoteMessage(message);
31
+
32
+ return message;
33
+ }
34
+ }
35
+
36
+ const copy = (remoteMessage: ISequencedDocumentMessage): ISequencedDocumentMessage => {
37
+ // Do shallow copy of message, as the processing flow will modify it.
38
+ // There might be multiple container instances receiving same message
39
+ // We do not need to make deep copy, as each layer will just replace message.content itself,
40
+ // but would not modify contents details
41
+ const message = { ...remoteMessage };
42
+
43
+ // back-compat: ADO #1385: eventually should become unconditional, but only for runtime messages!
44
+ // System message may have no contents, or in some cases (mostly for back-compat) they may have actual objects.
45
+ // Old ops may contain empty string (I assume noops).
46
+ if (typeof message.contents === "string" && message.contents !== "") {
47
+ message.contents = JSON.parse(message.contents);
48
+ }
49
+
50
+ return message;
51
+ };
52
+
53
+ /**
54
+ * For a given message, it moves the nested contents and type on level up.
55
+ *
56
+ */
57
+ const unpack = (message: ISequencedDocumentMessage) => {
58
+ const innerContents = message.contents as ContainerRuntimeMessage;
59
+ message.type = innerContents.type;
60
+ message.contents = innerContents.contents;
61
+ };
62
+
63
+ /**
64
+ * Unpacks runtime messages.
65
+ *
66
+ * @remarks This API makes no promises regarding backward-compatibility. This is internal API.
67
+ * @param message - message (as it observed in storage / service)
68
+ * @returns unpacked runtime message
69
+ *
70
+ * @internal
71
+ */
72
+ export function unpackRuntimeMessage(message: ISequencedDocumentMessage): boolean {
73
+ if (message.type !== MessageType.Operation) {
74
+ // Legacy format, but it's already "unpacked",
75
+ // i.e. message.type is actually ContainerMessageType.
76
+ // Or it's non-runtime message.
77
+ // Nothing to do in such case.
78
+ return false;
79
+ }
80
+
81
+ // legacy op format?
82
+ if (message.contents.address !== undefined && message.contents.type === undefined) {
83
+ message.type = ContainerMessageType.FluidDataStoreOp;
84
+ } else {
85
+ // new format
86
+ unpack(message);
87
+ }
88
+
89
+ return true;
90
+ }
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.0.0-dev.1.4.6.106135";
9
+ export const pkgVersion = "2.0.0-dev.2.3.0.115467";
@@ -10,7 +10,6 @@ import { DataProcessingError } from "@fluidframework/container-utils";
10
10
  import {
11
11
  ISequencedDocumentMessage,
12
12
  } from "@fluidframework/protocol-definitions";
13
- import { FlushMode } from "@fluidframework/runtime-definitions";
14
13
  import Deque from "double-ended-queue";
15
14
  import { ContainerMessageType } from "./containerRuntime";
16
15
  import { pkgVersion } from "./packageVersion";
@@ -29,15 +28,6 @@ export interface IPendingMessage {
29
28
  opMetadata: Record<string, unknown> | undefined;
30
29
  }
31
30
 
32
- /**
33
- * This represents a FlushMode update and is added to the pending queue when `setFlushMode` is called on the
34
- * ContainerRuntime and the FlushMode changes.
35
- */
36
- export interface IPendingFlushMode {
37
- type: "flushMode";
38
- flushMode: FlushMode;
39
- }
40
-
41
31
  /**
42
32
  * This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
43
33
  * to flush pending messages.
@@ -46,7 +36,7 @@ export interface IPendingFlush {
46
36
  type: "flush";
47
37
  }
48
38
 
49
- export type IPendingState = IPendingMessage | IPendingFlushMode | IPendingFlush;
39
+ export type IPendingState = IPendingMessage | IPendingFlush;
50
40
 
51
41
  export interface IPendingLocalState {
52
42
  /**
@@ -58,8 +48,6 @@ export interface IPendingLocalState {
58
48
  export interface IRuntimeStateHandler{
59
49
  connected(): boolean;
60
50
  clientId(): string | undefined;
61
- flushMode(): FlushMode;
62
- setFlushMode(mode: FlushMode): void;
63
51
  close(error?: ICriticalContainerError): void;
64
52
  applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
65
53
  flush(): void;
@@ -68,13 +56,18 @@ export interface IRuntimeStateHandler{
68
56
  content: any,
69
57
  localOpMetadata: unknown,
70
58
  opMetadata: Record<string, unknown> | undefined): void;
59
+ rollback(
60
+ type: ContainerMessageType,
61
+ content: any,
62
+ localOpMetadata: unknown): void;
63
+ orderSequentially(callback: () => void): void;
71
64
  }
72
65
 
73
66
  /**
74
67
  * PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
75
68
  * acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
76
69
  * batches along with the messages.
77
- * When the Container reconnects, it replays the pending states, which includes setting the FlushMode, manual flushing
70
+ * When the Container reconnects, it replays the pending states, which includes manual flushing
78
71
  * of messages and triggering resubmission of unacked ops.
79
72
  *
80
73
  * It verifies that all the ops are acked, are received in the right order and batch information is correct.
@@ -100,13 +93,6 @@ export class PendingStateManager implements IDisposable {
100
93
  // the correct batch metadata.
101
94
  private pendingBatchBeginMessage: ISequencedDocumentMessage | undefined;
102
95
 
103
- /**
104
- * This tracks the flush mode for the next message in the pending state queue. When replaying messages, we need to
105
- * first set the flush mode to this value and then send ops. It is important to do this info because the flush
106
- * mode could have been updated.
107
- */
108
- private flushModeForNextMessage: FlushMode;
109
-
110
96
  private clientId: string | undefined;
111
97
 
112
98
  /**
@@ -131,13 +117,9 @@ export class PendingStateManager implements IDisposable {
131
117
 
132
118
  constructor(
133
119
  private readonly stateHandler: IRuntimeStateHandler,
134
- initialFlushMode: FlushMode,
135
120
  initialLocalState: IPendingLocalState | undefined,
136
121
  ) {
137
122
  this.initialStates = new Deque<IPendingState>(initialLocalState?.pendingStates ?? []);
138
-
139
- this.flushModeForNextMessage = initialFlushMode;
140
- this.onFlushModeUpdated(initialFlushMode);
141
123
  }
142
124
 
143
125
  public get disposed() { return this.disposeOnce.evaluated; }
@@ -174,24 +156,10 @@ export class PendingStateManager implements IDisposable {
174
156
  this._pendingMessagesCount++;
175
157
  }
176
158
 
177
- /**
178
- * Called when the FlushMode is updated. Adds the FlushMode to the pending state queue.
179
- * @param flushMode - The flushMode that was updated.
180
- */
181
- public onFlushModeUpdated(flushMode: FlushMode) {
182
- this.pendingStates.push({ type: "flushMode", flushMode });
183
- }
184
-
185
159
  /**
186
160
  * Called when flush() is called on the ContainerRuntime to manually flush messages.
187
161
  */
188
162
  public onFlush() {
189
- // If the FlushMode is Immediate, we don't need to track an explicit flush call because every message is
190
- // automatically flushed. So, flush is a no-op.
191
- if (this.stateHandler.flushMode() === FlushMode.Immediate) {
192
- return;
193
- }
194
-
195
163
  // If the previous state is not a message, flush is a no-op.
196
164
  const previousState = this.pendingStates.peekBack();
197
165
  if (previousState?.type !== "message") {
@@ -279,11 +247,6 @@ export class PendingStateManager implements IDisposable {
279
247
  * @param message - The message that is being processed.
280
248
  */
281
249
  private maybeProcessBatchBegin(message: ISequencedDocumentMessage) {
282
- // Tracks the last FlushMode that was set before this message was sent.
283
- let pendingFlushMode: FlushMode | undefined;
284
- // Tracks whether a flush was called before this message was sent.
285
- let pendingFlush: boolean = false;
286
-
287
250
  /**
288
251
  * We are checking if the next message is the start of a batch. It can happen in the following scenarios:
289
252
  *
@@ -295,34 +258,12 @@ export class PendingStateManager implements IDisposable {
295
258
  * Keep reading pending states from the queue until we encounter a message. It's possible that the FlushMode was
296
259
  * updated a bunch of times without sending any messages.
297
260
  */
298
- let nextPendingState = this.peekNextPendingState();
299
- while (nextPendingState.type !== "message") {
300
- if (nextPendingState.type === "flushMode") {
301
- pendingFlushMode = nextPendingState.flushMode;
302
- }
303
- if (nextPendingState.type === "flush") {
304
- pendingFlush = true;
305
- }
261
+ while (this.peekNextPendingState().type !== "message") {
306
262
  this.pendingStates.shift();
307
- nextPendingState = this.peekNextPendingState();
308
263
  }
309
264
 
310
- if (pendingFlushMode !== undefined) {
311
- this.flushModeForNextMessage = pendingFlushMode;
312
- }
313
-
314
- // If the FlushMode was set to Immediate before this message was sent, this message won't be a batch message
315
- // because in Immediate mode, every message is flushed individually.
316
- if (pendingFlushMode === FlushMode.Immediate) {
317
- return;
318
- }
319
-
320
- /**
321
- * This message is the first in a batch if before it was sent either the FlushMode was set to TurnBased or there
322
- * was an explicit flush call. Note that a flush call is tracked only in TurnBased mode and it indicates the end
323
- * of one batch and beginning of another.
324
- */
325
- if (pendingFlushMode === FlushMode.TurnBased || pendingFlush) {
265
+ // This message is the first in a batch if the "batch" property on the metadata is set to true
266
+ if (message.metadata?.batch) {
326
267
  // We should not already be processing a batch and there should be no pending batch begin message.
327
268
  assert(!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined,
328
269
  0x16b /* "The pending batch state indicates we are already processing a batch" */);
@@ -347,16 +288,6 @@ export class PendingStateManager implements IDisposable {
347
288
  return;
348
289
  }
349
290
 
350
- /**
351
- * We are in the middle of processing a batch. The batch ends when we see an explicit flush. We should never see
352
- * a FlushMode before flush. This is true because we track batches only when FlushMode is TurnBased and in this
353
- * mode, a batch ends either by calling flush or by changing the mode to Immediate which also triggers a flush.
354
- */
355
- assert(
356
- nextPendingState.type !== "flushMode",
357
- 0x2bd /* "We should not see a pending FlushMode until we see a flush when processing a batch" */,
358
- );
359
-
360
291
  // There should be a pending batch begin message.
361
292
  assert(this.pendingBatchBeginMessage !== undefined, 0x16d /* "There is no pending batch begin message" */);
362
293
 
@@ -406,7 +337,7 @@ export class PendingStateManager implements IDisposable {
406
337
 
407
338
  /**
408
339
  * Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
409
- * states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.
340
+ * states in its queue. This includes triggering resubmission of unacked ops.
410
341
  */
411
342
  public replayPendingStates() {
412
343
  assert(this.stateHandler.connected(), 0x172 /* "The connection state is not consistent with the runtime" */);
@@ -426,12 +357,7 @@ export class PendingStateManager implements IDisposable {
426
357
  // Reset the pending message count because all these messages will be removed from the queue.
427
358
  this._pendingMessagesCount = 0;
428
359
 
429
- // Save the current FlushMode so that we can revert it back after replaying the states.
430
- const savedFlushMode = this.stateHandler.flushMode();
431
-
432
- // Set the flush mode for the next message. This step is important because the flush mode may have been changed
433
- // after the next pending message was sent.
434
- this.stateHandler.setFlushMode(this.flushModeForNextMessage);
360
+ const messageBatchQueue = new Deque<IPendingMessage>();
435
361
 
436
362
  // Process exactly `pendingStatesCount` items in the queue as it represents the number of states that were
437
363
  // pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
@@ -441,16 +367,43 @@ export class PendingStateManager implements IDisposable {
441
367
  const pendingState = this.pendingStates.shift()!;
442
368
  switch (pendingState.type) {
443
369
  case "message":
444
- this.stateHandler.reSubmit(
445
- pendingState.messageType,
446
- pendingState.content,
447
- pendingState.localOpMetadata,
448
- pendingState.opMetadata);
449
- break;
450
- case "flushMode":
451
- this.stateHandler.setFlushMode(pendingState.flushMode);
370
+ assert(pendingState.opMetadata?.batch !== false || messageBatchQueue.length > 0,
371
+ 0x41b /* We cannot process batches in chunks */);
372
+ /**
373
+ * We want to ensure grouped messages get processed in a batch.
374
+ * Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
375
+ * either receive the whole batch ack or nothing at all.
376
+ */
377
+ if (messageBatchQueue.length > 0 || pendingState.opMetadata?.batch) {
378
+ messageBatchQueue.enqueue(pendingState);
379
+ } else {
380
+ this.stateHandler.reSubmit(
381
+ pendingState.messageType,
382
+ pendingState.content,
383
+ pendingState.localOpMetadata,
384
+ pendingState.opMetadata);
385
+ }
452
386
  break;
453
387
  case "flush":
388
+ /**
389
+ * A "flush" call can indicate the end of a batch.
390
+ * We can't rely on the "batch" property in the message metadata as it gets
391
+ * updated elsewhere and it is not the same object instance that gets updated.
392
+ */
393
+ if (messageBatchQueue.length > 0) {
394
+ this.stateHandler.orderSequentially(() => {
395
+ while (messageBatchQueue.length > 0) {
396
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
397
+ const message = messageBatchQueue.dequeue()!;
398
+ this.stateHandler.reSubmit(
399
+ message.messageType,
400
+ message.content,
401
+ message.localOpMetadata,
402
+ message.opMetadata);
403
+ }
404
+ });
405
+ }
406
+ assert(messageBatchQueue.length === 0, 0x41c /* cannot flush in the middle of a batch */);
454
407
  this.stateHandler.flush();
455
408
  break;
456
409
  default:
@@ -459,7 +412,15 @@ export class PendingStateManager implements IDisposable {
459
412
  pendingStatesCount--;
460
413
  }
461
414
 
462
- // Revert the FlushMode.
463
- this.stateHandler.setFlushMode(savedFlushMode);
415
+ // There are some cases where ops are stashed but not flushed. We need to ensure they are resubmitted
416
+ while (messageBatchQueue.length > 0) {
417
+ // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
418
+ const message = messageBatchQueue.dequeue()!;
419
+ this.stateHandler.reSubmit(
420
+ message.messageType,
421
+ message.content,
422
+ message.localOpMetadata,
423
+ message.opMetadata);
424
+ }
464
425
  }
465
426
  }
@@ -263,9 +263,9 @@ export class RunningSummarizer implements IDisposable {
263
263
 
264
264
  /**
265
265
  * Can the given op trigger a summary?
266
- * # Currently only prevents summaries for Summarize and SummaryAck ops
266
+ * # Currently always prevents summaries for Summarize and SummaryAck/Nack ops
267
267
  * @param op - op to check
268
- * @returns true if this type of op can trigger a summary
268
+ * @returns true if this op can trigger a summary
269
269
  */
270
270
  private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
271
271
  switch (op.type) {
@@ -274,10 +274,18 @@ export class RunningSummarizer implements IDisposable {
274
274
  case MessageType.SummaryNack:
275
275
  return false;
276
276
  default:
277
- return true;
277
+ return isRuntimeMessage(op) || this.nonRuntimeOpCanTriggerSummary();
278
278
  }
279
279
  }
280
280
 
281
+ private nonRuntimeOpCanTriggerSummary(): boolean {
282
+ // eslint-disable-next-line max-len
283
+ const opsSinceLastAck = this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
284
+ return this.configuration.state === "enabled"
285
+ && (this.configuration.nonRuntimeHeuristicThreshold === undefined
286
+ || this.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck);
287
+ }
288
+
281
289
  public async waitStop(allowLastSummary: boolean): Promise<void> {
282
290
  if (this.stopping) {
283
291
  return;
@@ -221,6 +221,7 @@ class ScheduleManagerCore {
221
221
  length: endBatch - startBatch,
222
222
  });
223
223
  }
224
+
224
225
  this.deltaManager.inbound.resume();
225
226
  }
226
227
 
package/src/summarizer.ts CHANGED
@@ -334,7 +334,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
334
334
  const coordinatorCreateP = this.runCoordinatorCreateFn(this.runtime);
335
335
 
336
336
  coordinatorCreateP.then((runCoordinator) => {
337
- // Successully created the cancellation token. Start the summarizer.
337
+ // Successfully created the cancellation token. Start the summarizer.
338
338
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
339
339
  const startP = this.start(this.runtime.clientId!, runCoordinator);
340
340
  startP.then(async (runningSummarizer) => {
@@ -382,11 +382,11 @@ export class Summarizer extends EventEmitter implements ISummarizer {
382
382
  // executing the refreshLatestSummaryAck.
383
383
  // https://dev.azure.com/fluidframework/internal/_workitems/edit/779
384
384
  await this.runningSummarizer.lockedRefreshSummaryAckAction(async () =>
385
- this.internalsProvider.refreshLatestSummaryAck(
386
- summaryOpHandle,
387
- summaryAckHandle,
388
- refSequenceNumber,
389
- summaryLogger,
385
+ this.internalsProvider.refreshLatestSummaryAck({
386
+ proposalHandle: summaryOpHandle,
387
+ ackHandle: summaryAckHandle,
388
+ summaryRefSeq: refSequenceNumber,
389
+ summaryLogger },
390
390
  ).catch(async (error) => {
391
391
  // If the error is 404, so maybe the fetched version no longer exists on server. We just
392
392
  // ignore this error in that case, as that means we will have another summaryAck for the
@@ -77,7 +77,7 @@ export class SummarizerClientElection
77
77
  // Log and elect a new summarizer client.
78
78
  const opsSinceLastReport = sequenceNumber - this.lastReportedSeq;
79
79
  if (opsSinceLastReport > this.maxOpsSinceLastSummary) {
80
- this.logger.sendErrorEvent({
80
+ this.logger.sendTelemetryEvent({
81
81
  eventName: "ElectedClientNotSummarizing",
82
82
  electedClientId,
83
83
  lastSummaryAckSeqForClient: this.lastSummaryAckSeqForClient,
@@ -119,9 +119,6 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
119
119
  }
120
120
 
121
121
  public get idleTime(): number {
122
- if (this.configuration.idleTime !== undefined) {
123
- return this.configuration.idleTime;
124
- }
125
122
  const maxIdleTime = this.configuration.maxIdleTime;
126
123
  const minIdleTime = this.configuration.minIdleTime;
127
124
  const weightedNumOfOps = getWeightedNumberOfOps(