@fluidframework/container-runtime 2.0.0-dev.4.2.0.153917 → 2.0.0-dev.4.3.0.158678

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/dist/containerRuntime.d.ts +33 -3
  2. package/dist/containerRuntime.d.ts.map +1 -1
  3. package/dist/containerRuntime.js +187 -58
  4. package/dist/containerRuntime.js.map +1 -1
  5. package/dist/dataStoreContext.d.ts +2 -1
  6. package/dist/dataStoreContext.d.ts.map +1 -1
  7. package/dist/dataStoreContext.js +3 -0
  8. package/dist/dataStoreContext.js.map +1 -1
  9. package/dist/dataStores.d.ts +5 -5
  10. package/dist/dataStores.d.ts.map +1 -1
  11. package/dist/dataStores.js +3 -6
  12. package/dist/dataStores.js.map +1 -1
  13. package/dist/gc/garbageCollection.d.ts.map +1 -1
  14. package/dist/gc/garbageCollection.js +5 -5
  15. package/dist/gc/garbageCollection.js.map +1 -1
  16. package/dist/gc/gcConfigs.d.ts.map +1 -1
  17. package/dist/gc/gcConfigs.js +1 -3
  18. package/dist/gc/gcConfigs.js.map +1 -1
  19. package/dist/gc/gcDefinitions.js +1 -1
  20. package/dist/gc/gcDefinitions.js.map +1 -1
  21. package/dist/id-compressor/appendOnlySortedMap.d.ts +146 -0
  22. package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -0
  23. package/dist/id-compressor/appendOnlySortedMap.js +360 -0
  24. package/dist/id-compressor/appendOnlySortedMap.js.map +1 -0
  25. package/dist/id-compressor/idCompressor.d.ts +279 -0
  26. package/dist/id-compressor/idCompressor.d.ts.map +1 -0
  27. package/dist/id-compressor/idCompressor.js +1258 -0
  28. package/dist/id-compressor/idCompressor.js.map +1 -0
  29. package/dist/id-compressor/idRange.d.ts +11 -0
  30. package/dist/id-compressor/idRange.d.ts.map +1 -0
  31. package/dist/id-compressor/idRange.js +29 -0
  32. package/dist/id-compressor/idRange.js.map +1 -0
  33. package/dist/id-compressor/index.d.ts +14 -0
  34. package/dist/id-compressor/index.d.ts.map +1 -0
  35. package/dist/id-compressor/index.js +38 -0
  36. package/dist/id-compressor/index.js.map +1 -0
  37. package/dist/id-compressor/numericUuid.d.ts +59 -0
  38. package/dist/id-compressor/numericUuid.d.ts.map +1 -0
  39. package/dist/id-compressor/numericUuid.js +325 -0
  40. package/dist/id-compressor/numericUuid.js.map +1 -0
  41. package/dist/id-compressor/sessionIdNormalizer.d.ts +138 -0
  42. package/dist/id-compressor/sessionIdNormalizer.d.ts.map +1 -0
  43. package/dist/id-compressor/sessionIdNormalizer.js +488 -0
  44. package/dist/id-compressor/sessionIdNormalizer.js.map +1 -0
  45. package/dist/id-compressor/utils.d.ts +57 -0
  46. package/dist/id-compressor/utils.d.ts.map +1 -0
  47. package/dist/id-compressor/utils.js +90 -0
  48. package/dist/id-compressor/utils.js.map +1 -0
  49. package/dist/id-compressor/uuidUtilities.d.ts +30 -0
  50. package/dist/id-compressor/uuidUtilities.d.ts.map +1 -0
  51. package/dist/id-compressor/uuidUtilities.js +106 -0
  52. package/dist/id-compressor/uuidUtilities.js.map +1 -0
  53. package/dist/index.d.ts +1 -0
  54. package/dist/index.d.ts.map +1 -1
  55. package/dist/index.js +5 -1
  56. package/dist/index.js.map +1 -1
  57. package/dist/opLifecycle/batchManager.d.ts +9 -2
  58. package/dist/opLifecycle/batchManager.d.ts.map +1 -1
  59. package/dist/opLifecycle/batchManager.js +21 -2
  60. package/dist/opLifecycle/batchManager.js.map +1 -1
  61. package/dist/opLifecycle/index.d.ts +1 -1
  62. package/dist/opLifecycle/index.d.ts.map +1 -1
  63. package/dist/opLifecycle/index.js.map +1 -1
  64. package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
  65. package/dist/opLifecycle/opGroupingManager.js +5 -0
  66. package/dist/opLifecycle/opGroupingManager.js.map +1 -1
  67. package/dist/opLifecycle/outbox.d.ts +2 -2
  68. package/dist/opLifecycle/outbox.d.ts.map +1 -1
  69. package/dist/opLifecycle/outbox.js +34 -22
  70. package/dist/opLifecycle/outbox.js.map +1 -1
  71. package/dist/packageVersion.d.ts +1 -1
  72. package/dist/packageVersion.js +1 -1
  73. package/dist/packageVersion.js.map +1 -1
  74. package/dist/pendingStateManager.d.ts +1 -1
  75. package/dist/pendingStateManager.d.ts.map +1 -1
  76. package/dist/pendingStateManager.js +11 -3
  77. package/dist/pendingStateManager.js.map +1 -1
  78. package/dist/summary/index.d.ts +1 -1
  79. package/dist/summary/index.d.ts.map +1 -1
  80. package/dist/summary/index.js +2 -1
  81. package/dist/summary/index.js.map +1 -1
  82. package/dist/summary/orderedClientElection.d.ts +1 -0
  83. package/dist/summary/orderedClientElection.d.ts.map +1 -1
  84. package/dist/summary/orderedClientElection.js +17 -1
  85. package/dist/summary/orderedClientElection.js.map +1 -1
  86. package/dist/summary/runningSummarizer.d.ts +0 -1
  87. package/dist/summary/runningSummarizer.d.ts.map +1 -1
  88. package/dist/summary/runningSummarizer.js +1 -17
  89. package/dist/summary/runningSummarizer.js.map +1 -1
  90. package/dist/summary/summaryFormat.d.ts +3 -0
  91. package/dist/summary/summaryFormat.d.ts.map +1 -1
  92. package/dist/summary/summaryFormat.js +3 -1
  93. package/dist/summary/summaryFormat.js.map +1 -1
  94. package/dist/summary/summaryManager.d.ts.map +1 -1
  95. package/dist/summary/summaryManager.js +2 -0
  96. package/dist/summary/summaryManager.js.map +1 -1
  97. package/lib/containerRuntime.d.ts +33 -3
  98. package/lib/containerRuntime.d.ts.map +1 -1
  99. package/lib/containerRuntime.js +170 -60
  100. package/lib/containerRuntime.js.map +1 -1
  101. package/lib/dataStoreContext.d.ts +2 -1
  102. package/lib/dataStoreContext.d.ts.map +1 -1
  103. package/lib/dataStoreContext.js +3 -0
  104. package/lib/dataStoreContext.js.map +1 -1
  105. package/lib/dataStores.d.ts +5 -5
  106. package/lib/dataStores.d.ts.map +1 -1
  107. package/lib/dataStores.js +3 -6
  108. package/lib/dataStores.js.map +1 -1
  109. package/lib/gc/garbageCollection.d.ts.map +1 -1
  110. package/lib/gc/garbageCollection.js +5 -5
  111. package/lib/gc/garbageCollection.js.map +1 -1
  112. package/lib/gc/gcConfigs.d.ts.map +1 -1
  113. package/lib/gc/gcConfigs.js +1 -3
  114. package/lib/gc/gcConfigs.js.map +1 -1
  115. package/lib/gc/gcDefinitions.js +1 -1
  116. package/lib/gc/gcDefinitions.js.map +1 -1
  117. package/lib/id-compressor/appendOnlySortedMap.d.ts +146 -0
  118. package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -0
  119. package/lib/id-compressor/appendOnlySortedMap.js +355 -0
  120. package/lib/id-compressor/appendOnlySortedMap.js.map +1 -0
  121. package/lib/id-compressor/idCompressor.d.ts +279 -0
  122. package/lib/id-compressor/idCompressor.d.ts.map +1 -0
  123. package/lib/id-compressor/idCompressor.js +1248 -0
  124. package/lib/id-compressor/idCompressor.js.map +1 -0
  125. package/lib/id-compressor/idRange.d.ts +11 -0
  126. package/lib/id-compressor/idRange.d.ts.map +1 -0
  127. package/lib/id-compressor/idRange.js +25 -0
  128. package/lib/id-compressor/idRange.js.map +1 -0
  129. package/lib/id-compressor/index.d.ts +14 -0
  130. package/lib/id-compressor/index.d.ts.map +1 -0
  131. package/lib/id-compressor/index.js +14 -0
  132. package/lib/id-compressor/index.js.map +1 -0
  133. package/lib/id-compressor/numericUuid.d.ts +59 -0
  134. package/lib/id-compressor/numericUuid.d.ts.map +1 -0
  135. package/lib/id-compressor/numericUuid.js +315 -0
  136. package/lib/id-compressor/numericUuid.js.map +1 -0
  137. package/lib/id-compressor/sessionIdNormalizer.d.ts +138 -0
  138. package/lib/id-compressor/sessionIdNormalizer.d.ts.map +1 -0
  139. package/lib/id-compressor/sessionIdNormalizer.js +484 -0
  140. package/lib/id-compressor/sessionIdNormalizer.js.map +1 -0
  141. package/lib/id-compressor/utils.d.ts +57 -0
  142. package/lib/id-compressor/utils.d.ts.map +1 -0
  143. package/lib/id-compressor/utils.js +79 -0
  144. package/lib/id-compressor/utils.js.map +1 -0
  145. package/lib/id-compressor/uuidUtilities.d.ts +30 -0
  146. package/lib/id-compressor/uuidUtilities.d.ts.map +1 -0
  147. package/lib/id-compressor/uuidUtilities.js +98 -0
  148. package/lib/id-compressor/uuidUtilities.js.map +1 -0
  149. package/lib/index.d.ts +1 -0
  150. package/lib/index.d.ts.map +1 -1
  151. package/lib/index.js +1 -0
  152. package/lib/index.js.map +1 -1
  153. package/lib/opLifecycle/batchManager.d.ts +9 -2
  154. package/lib/opLifecycle/batchManager.d.ts.map +1 -1
  155. package/lib/opLifecycle/batchManager.js +19 -1
  156. package/lib/opLifecycle/batchManager.js.map +1 -1
  157. package/lib/opLifecycle/index.d.ts +1 -1
  158. package/lib/opLifecycle/index.d.ts.map +1 -1
  159. package/lib/opLifecycle/index.js.map +1 -1
  160. package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
  161. package/lib/opLifecycle/opGroupingManager.js +5 -0
  162. package/lib/opLifecycle/opGroupingManager.js.map +1 -1
  163. package/lib/opLifecycle/outbox.d.ts +2 -2
  164. package/lib/opLifecycle/outbox.d.ts.map +1 -1
  165. package/lib/opLifecycle/outbox.js +35 -23
  166. package/lib/opLifecycle/outbox.js.map +1 -1
  167. package/lib/packageVersion.d.ts +1 -1
  168. package/lib/packageVersion.js +1 -1
  169. package/lib/packageVersion.js.map +1 -1
  170. package/lib/pendingStateManager.d.ts +1 -1
  171. package/lib/pendingStateManager.d.ts.map +1 -1
  172. package/lib/pendingStateManager.js +11 -3
  173. package/lib/pendingStateManager.js.map +1 -1
  174. package/lib/summary/index.d.ts +1 -1
  175. package/lib/summary/index.d.ts.map +1 -1
  176. package/lib/summary/index.js +1 -1
  177. package/lib/summary/index.js.map +1 -1
  178. package/lib/summary/orderedClientElection.d.ts +1 -0
  179. package/lib/summary/orderedClientElection.d.ts.map +1 -1
  180. package/lib/summary/orderedClientElection.js +17 -1
  181. package/lib/summary/orderedClientElection.js.map +1 -1
  182. package/lib/summary/runningSummarizer.d.ts +0 -1
  183. package/lib/summary/runningSummarizer.d.ts.map +1 -1
  184. package/lib/summary/runningSummarizer.js +1 -17
  185. package/lib/summary/runningSummarizer.js.map +1 -1
  186. package/lib/summary/summaryFormat.d.ts +3 -0
  187. package/lib/summary/summaryFormat.d.ts.map +1 -1
  188. package/lib/summary/summaryFormat.js +2 -0
  189. package/lib/summary/summaryFormat.js.map +1 -1
  190. package/lib/summary/summaryManager.d.ts.map +1 -1
  191. package/lib/summary/summaryManager.js +2 -0
  192. package/lib/summary/summaryManager.js.map +1 -1
  193. package/package.json +27 -18
  194. package/src/containerRuntime.ts +256 -88
  195. package/src/dataStoreContext.ts +6 -0
  196. package/src/dataStores.ts +4 -7
  197. package/src/gc/garbageCollection.ts +7 -6
  198. package/src/gc/gcConfigs.ts +1 -3
  199. package/src/gc/gcDefinitions.ts +1 -1
  200. package/src/id-compressor/README.md +3 -0
  201. package/src/id-compressor/appendOnlySortedMap.ts +427 -0
  202. package/src/id-compressor/idCompressor.ts +1854 -0
  203. package/src/id-compressor/idRange.ts +35 -0
  204. package/src/id-compressor/index.ts +35 -0
  205. package/src/id-compressor/numericUuid.ts +383 -0
  206. package/src/id-compressor/sessionIdNormalizer.ts +609 -0
  207. package/src/id-compressor/utils.ts +114 -0
  208. package/src/id-compressor/uuidUtilities.ts +123 -0
  209. package/src/index.ts +1 -0
  210. package/src/opLifecycle/README.md +13 -0
  211. package/src/opLifecycle/batchManager.ts +35 -2
  212. package/src/opLifecycle/index.ts +1 -1
  213. package/src/opLifecycle/opGroupingManager.ts +5 -1
  214. package/src/opLifecycle/outbox.ts +57 -23
  215. package/src/packageVersion.ts +1 -1
  216. package/src/pendingStateManager.ts +21 -7
  217. package/src/summary/index.ts +1 -0
  218. package/src/summary/orderedClientElection.ts +15 -2
  219. package/src/summary/runningSummarizer.ts +3 -24
  220. package/src/summary/summaryFormat.ts +4 -0
  221. package/src/summary/summaryManager.ts +2 -0
@@ -0,0 +1,114 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ /**
7
+ * Remove `readonly` from all fields.
8
+ */
9
+ export type Mutable<T> = { -readonly [P in keyof T]: T[P] };
10
+
11
+ /** A union type of the first `N` positive integers */
12
+ export type TakeWholeNumbers<N extends number, A extends never[] = []> = N extends A["length"]
13
+ ? never
14
+ : A["length"] | TakeWholeNumbers<N, [never, ...A]>;
15
+
16
+ /** Returns a tuple type with exactly `Length` elements of type `T` */
17
+ export type ArrayOfLength<T, Length extends number, A extends T[] = []> = Length extends A["length"]
18
+ ? A
19
+ : ArrayOfLength<T, Length, [T, ...A]>;
20
+
21
+ /**
22
+ * Fails true iff `array` has at least `length` elements
23
+ */
24
+ export function hasAtLeastLength<T, Len extends TakeWholeNumbers<16>>(
25
+ array: readonly T[],
26
+ length: Len,
27
+ ): array is [...ArrayOfLength<T, Len>, ...T[]] {
28
+ return array.length >= length;
29
+ }
30
+
31
+ /**
32
+ * A numeric comparator used for sorting in ascending order.
33
+ *
34
+ * Handles +/-0 like Map: -0 is equal to +0.
35
+ */
36
+ export function compareFiniteNumbers<T extends number>(a: T, b: T): number {
37
+ return a - b;
38
+ }
39
+
40
+ /**
41
+ * A numeric comparator used for sorting in descending order.
42
+ *
43
+ * Handles +/-0 like Map: -0 is equal to +0.
44
+ */
45
+ export function compareFiniteNumbersReversed<T extends number>(a: T, b: T): number {
46
+ return b - a;
47
+ }
48
+
49
+ /**
50
+ * Compare two maps and return true if their contents are equivalent.
51
+ * @param mapA - The first array to compare
52
+ * @param mapB - The second array to compare
53
+ * @param elementComparator - The function used to check if two `T`s are equivalent.
54
+ * Defaults to `Object.is()` equality (a shallow compare)
55
+ */
56
+ export function compareMaps<K, V>(
57
+ mapA: ReadonlyMap<K, V>,
58
+ mapB: ReadonlyMap<K, V>,
59
+ elementComparator: (a: V, b: V) => boolean = Object.is,
60
+ ): boolean {
61
+ if (mapA.size !== mapB.size) {
62
+ return false;
63
+ }
64
+
65
+ for (const [keyA, valueA] of mapA) {
66
+ const valueB = mapB.get(keyA);
67
+ if (valueB === undefined || !elementComparator(valueA, valueB)) {
68
+ return false;
69
+ }
70
+ }
71
+
72
+ return true;
73
+ }
74
+
75
+ /**
76
+ * Retrieve a value from a map with the given key, or create a new entry if the key is not in the map.
77
+ * @param map - The map to query/update
78
+ * @param key - The key to lookup in the map
79
+ * @param defaultValue - a function which returns a default value. This is called and used to set an initial value for the given key in the map if none exists
80
+ * @returns either the existing value for the given key, or the newly-created value (the result of `defaultValue`)
81
+ */
82
+ export function getOrCreate<K, V>(map: Map<K, V>, key: K, defaultValue: (key: K) => V): V {
83
+ let value = map.get(key);
84
+ if (value === undefined) {
85
+ value = defaultValue(key);
86
+ map.set(key, value);
87
+ }
88
+ return value;
89
+ }
90
+
91
+ /**
92
+ * Compares strings lexically to form a strict partial ordering.
93
+ */
94
+ export function compareStrings<T extends string>(a: T, b: T): number {
95
+ return a > b ? 1 : a === b ? 0 : -1;
96
+ }
97
+
98
+ export function fail(message: string): never {
99
+ throw new Error(message);
100
+ }
101
+
102
+ /**
103
+ * Sets a property in such a way that it is only set on `destination` if the provided value is not undefined.
104
+ * This avoids having explicit undefined values under properties that would cause `Object.hasOwnProperty` to return true.
105
+ */
106
+ export function setPropertyIfDefined<TDst, P extends keyof TDst>(
107
+ value: TDst[P] | undefined,
108
+ destination: TDst,
109
+ property: P,
110
+ ): void {
111
+ if (value !== undefined) {
112
+ destination[property] = value;
113
+ }
114
+ }
@@ -0,0 +1,123 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { assert } from "@fluidframework/common-utils";
7
+ import { StableId, UuidString } from "@fluidframework/runtime-definitions";
8
+ import { v4, NIL } from "uuid";
9
+
10
+ const hexadecimalCharCodes = Array.from("09afAF").map((c) => c.charCodeAt(0)) as [
11
+ zero: number,
12
+ nine: number,
13
+ a: number,
14
+ f: number,
15
+ A: number,
16
+ F: number,
17
+ ];
18
+
19
+ function isHexadecimalCharacter(charCode: number): boolean {
20
+ return (
21
+ (charCode >= hexadecimalCharCodes[0] && charCode <= hexadecimalCharCodes[1]) ||
22
+ (charCode >= hexadecimalCharCodes[2] && charCode <= hexadecimalCharCodes[3]) ||
23
+ (charCode >= hexadecimalCharCodes[4] && charCode <= hexadecimalCharCodes[5])
24
+ );
25
+ }
26
+
27
+ /** The null (lowest/all-zeros) UUID */
28
+ export const nilUuid = assertIsUuidString(NIL);
29
+
30
+ /**
31
+ * Asserts that the given string is a UUID
32
+ */
33
+ export function assertIsUuidString(uuidString: string): UuidString {
34
+ assert(isUuidString(uuidString), 0x4a2 /* Expected an UuidString */);
35
+ return uuidString;
36
+ }
37
+
38
+ /**
39
+ * Returns true iff the given string is a valid UUID-like string of hexadecimal characters
40
+ * 'xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx'
41
+ */
42
+ export function isUuidString(str: string): str is UuidString {
43
+ for (let i = 0; i < str.length; i++) {
44
+ switch (i) {
45
+ case 8:
46
+ case 13:
47
+ case 18:
48
+ case 23:
49
+ if (str.charAt(i) !== "-") {
50
+ return false;
51
+ }
52
+ break;
53
+
54
+ default:
55
+ if (!isHexadecimalCharacter(str.charCodeAt(i))) {
56
+ return false;
57
+ }
58
+ break;
59
+ }
60
+ }
61
+
62
+ return true;
63
+ }
64
+
65
+ /**
66
+ * Generate a random stable ID
67
+ */
68
+ export function generateStableId(): StableId {
69
+ return assertIsStableId(v4());
70
+ }
71
+
72
+ /**
73
+ * Asserts that the given string is a stable ID.
74
+ */
75
+ export function assertIsStableId(stableId: string): StableId {
76
+ assert(isStableId(stableId), 0x4a3 /* Expected a StableId */);
77
+ return stableId;
78
+ }
79
+
80
+ /**
81
+ * Returns true iff the given string is a valid Version 4, variant 2 UUID
82
+ * 'xxxxxxxx-xxxx-4xxx-vxxx-xxxxxxxxxxxx'
83
+ */
84
+ export function isStableId(str: string): str is StableId {
85
+ if (str.length !== 36) {
86
+ return false;
87
+ }
88
+
89
+ for (let i = 0; i < str.length; i++) {
90
+ switch (i) {
91
+ case 8:
92
+ case 13:
93
+ case 18:
94
+ case 23:
95
+ if (str.charAt(i) !== "-") {
96
+ return false;
97
+ }
98
+ break;
99
+
100
+ case 14:
101
+ if (str.charAt(i) !== "4") {
102
+ return false;
103
+ }
104
+ break;
105
+
106
+ case 19: {
107
+ const char = str.charAt(i);
108
+ if (char !== "8" && char !== "9" && char !== "a" && char !== "b") {
109
+ return false;
110
+ }
111
+ break;
112
+ }
113
+
114
+ default:
115
+ if (!isHexadecimalCharacter(str.charCodeAt(i))) {
116
+ return false;
117
+ }
118
+ break;
119
+ }
120
+ }
121
+
122
+ return true;
123
+ }
package/src/index.ts CHANGED
@@ -69,3 +69,4 @@ export {
69
69
  ICancellableSummarizerController,
70
70
  } from "./summary";
71
71
  export { IChunkedOp, unpackRuntimeMessage } from "./opLifecycle";
72
+ export { generateStableId, isStableId, assertIsStableId } from "./id-compressor";
@@ -13,6 +13,7 @@ By default, the runtime is configured with a max batch size of `716800` bytes, w
13
13
  - [Introduction](#introduction)
14
14
  - [Compression](#compression)
15
15
  - [Grouped batching](#grouped-batching)
16
+ - [Risks](#risks)
16
17
  - [Chunking for compression](#chunking-for-compression)
17
18
  - [Disabling in case of emergency](#disabling-in-case-of-emergency)
18
19
  - [Example configs](#example-configs)
@@ -38,6 +39,18 @@ The purpose for enabling grouped batching on top of compression is that regular
38
39
 
39
40
  See [below](#how-grouped-batching-works) for an example.
40
41
 
42
+ ### Risks
43
+
44
+ This option is experimental and should not be enabled yet in production. This option should **ONLY** be enabled after observing that 99.9% of your application sessions contains these changes (runtime version "2.0.0-internal.4.1.0" or later). Containers created with this option may not open in future versions of the framework.
45
+
46
+ This option will change a couple of expectations around message structure and runtime layer expectations. Only enable this option after testing
47
+ and verifying that the following expectation changes won't have any effects:
48
+
49
+ - batch messages observed at the runtime layer will not match messages seen at the loader layer (i.e. grouped form at loader layer, ungrouped form at runtime layer)
50
+ - messages within the same batch will have the same sequence number
51
+ - client sequence numbers on batch messages can only be used to order messages with the same sequenceNumber
52
+ - requires all ops to be processed by runtime layer (version "2.0.0-internal.1.2.0" or later https://github.com/microsoft/FluidFramework/pull/11832)
53
+
41
54
  ## Chunking for compression
42
55
 
43
56
  **Op chunking for compression targets payloads which exceed the max batch size after compression.** So, only payloads which are already compressed. By default, the feature is enabled.
@@ -12,6 +12,11 @@ export interface IBatchManagerOptions {
12
12
  readonly compressionOptions?: ICompressionRuntimeOptions;
13
13
  }
14
14
 
15
+ export interface BatchSequenceNumbers {
16
+ referenceSequenceNumber?: number;
17
+ clientSequenceNumber?: number;
18
+ }
19
+
15
20
  /**
16
21
  * Estimated size of the stringification overhead for an op accumulated
17
22
  * from runtime to loader to the service.
@@ -32,15 +37,24 @@ export class BatchManager {
32
37
  return this.batchContentSize;
33
38
  }
34
39
 
35
- public get referenceSequenceNumber(): number | undefined {
40
+ public get sequenceNumbers(): BatchSequenceNumbers {
41
+ return {
42
+ referenceSequenceNumber: this.referenceSequenceNumber,
43
+ clientSequenceNumber: this.clientSequenceNumber,
44
+ };
45
+ }
46
+
47
+ private get referenceSequenceNumber(): number | undefined {
36
48
  return this.pendingBatch.length === 0
37
49
  ? undefined
38
50
  : this.pendingBatch[this.pendingBatch.length - 1].referenceSequenceNumber;
39
51
  }
40
52
 
53
+ private clientSequenceNumber: number | undefined;
54
+
41
55
  constructor(public readonly options: IBatchManagerOptions) {}
42
56
 
43
- public push(message: BatchMessage): boolean {
57
+ public push(message: BatchMessage, currentClientSequenceNumber?: number): boolean {
44
58
  const contentSize = this.batchContentSize + (message.contents?.length ?? 0);
45
59
  const opCount = this.pendingBatch.length;
46
60
 
@@ -68,6 +82,10 @@ export class BatchManager {
68
82
  return false;
69
83
  }
70
84
 
85
+ if (this.pendingBatch.length === 0) {
86
+ this.clientSequenceNumber = currentClientSequenceNumber;
87
+ }
88
+
71
89
  this.batchContentSize = contentSize;
72
90
  this.pendingBatch.push(message);
73
91
  return true;
@@ -86,6 +104,7 @@ export class BatchManager {
86
104
 
87
105
  this.pendingBatch = [];
88
106
  this.batchContentSize = 0;
107
+ this.clientSequenceNumber = undefined;
89
108
 
90
109
  return addBatchMetadata(batch);
91
110
  }
@@ -136,3 +155,17 @@ const addBatchMetadata = (batch: IBatch): IBatch => {
136
155
  export const estimateSocketSize = (batch: IBatch): number => {
137
156
  return batch.contentSizeInBytes + opOverhead * batch.content.length;
138
157
  };
158
+
159
+ export const sequenceNumbersMatch = (
160
+ seqNums: BatchSequenceNumbers,
161
+ otherSeqNums: BatchSequenceNumbers,
162
+ ): boolean => {
163
+ return (
164
+ (seqNums.referenceSequenceNumber === undefined ||
165
+ otherSeqNums.referenceSequenceNumber === undefined ||
166
+ seqNums.referenceSequenceNumber === otherSeqNums.referenceSequenceNumber) &&
167
+ (seqNums.clientSequenceNumber === undefined ||
168
+ otherSeqNums.clientSequenceNumber === undefined ||
169
+ seqNums.clientSequenceNumber === otherSeqNums.clientSequenceNumber)
170
+ );
171
+ };
@@ -3,7 +3,7 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
- export { BatchManager, estimateSocketSize } from "./batchManager";
6
+ export { BatchManager, estimateSocketSize, BatchSequenceNumbers } from "./batchManager";
7
7
  export {
8
8
  BatchMessage,
9
9
  IBatch,
@@ -5,7 +5,7 @@
5
5
 
6
6
  import { assert } from "@fluidframework/common-utils";
7
7
  import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
8
- import { ContainerRuntimeMessage } from "..";
8
+ import { ContainerMessageType, ContainerRuntimeMessage } from "..";
9
9
  import { IBatch } from "./definitions";
10
10
 
11
11
  interface IGroupedMessage {
@@ -25,6 +25,10 @@ export class OpGroupingManager {
25
25
  }
26
26
 
27
27
  for (const message of batch.content) {
28
+ // Blob attaches cannot be grouped (grouped batching would hide metadata)
29
+ if (message.deserializedContent.type === ContainerMessageType.BlobAttach) {
30
+ return batch;
31
+ }
28
32
  if (message.metadata) {
29
33
  const keys = Object.keys(message.metadata);
30
34
  assert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
@@ -15,7 +15,12 @@ import {
15
15
  } from "@fluidframework/telemetry-utils";
16
16
  import { ICompressionRuntimeOptions } from "../containerRuntime";
17
17
  import { PendingStateManager } from "../pendingStateManager";
18
- import { BatchManager, estimateSocketSize } from "./batchManager";
18
+ import {
19
+ BatchManager,
20
+ BatchSequenceNumbers,
21
+ estimateSocketSize,
22
+ sequenceNumbersMatch,
23
+ } from "./batchManager";
19
24
  import { BatchMessage, IBatch } from "./definitions";
20
25
  import { OpCompressor } from "./opCompressor";
21
26
  import { OpGroupingManager } from "./opGroupingManager";
@@ -37,6 +42,20 @@ export interface IOutboxParameters {
37
42
  readonly splitter: OpSplitter;
38
43
  readonly logger: ITelemetryLogger;
39
44
  readonly groupingManager: OpGroupingManager;
45
+ readonly getCurrentSequenceNumbers: () => BatchSequenceNumbers;
46
+ }
47
+
48
+ function getLongStack(action: () => Error): Error {
49
+ // Increase the stack trace limit temporarily, so as to debug better in case it occurs.
50
+ try {
51
+ const originalStackTraceLimit = (Error as any).stackTraceLimit;
52
+ (Error as any).stackTraceLimit = 50;
53
+ const result = action();
54
+ (Error as any).stackTraceLimit = originalStackTraceLimit;
55
+ return result;
56
+ } catch (error) {
57
+ return action();
58
+ }
40
59
  }
41
60
 
42
61
  export class Outbox {
@@ -76,39 +95,39 @@ export class Outbox {
76
95
  * what was already in the batch managers, this means that batching has been interrupted so
77
96
  * we will flush the accumulated messages to account for that and create a new batch with the new
78
97
  * message as the first message.
79
- *
80
- * @param message - the incoming message
81
98
  */
82
- private maybeFlushPartialBatch(message: BatchMessage) {
83
- const mainBatchReference = this.mainBatch.referenceSequenceNumber;
84
- const attachFlowBatchReference = this.attachFlowBatch.referenceSequenceNumber;
99
+ private maybeFlushPartialBatch() {
100
+ const mainBatchSeqNums = this.mainBatch.sequenceNumbers;
101
+ const attachFlowBatchSeqNums = this.attachFlowBatch.sequenceNumbers;
85
102
  assert(
86
103
  this.params.config.disablePartialFlush ||
87
- mainBatchReference === undefined ||
88
- attachFlowBatchReference === undefined ||
89
- mainBatchReference === attachFlowBatchReference,
104
+ sequenceNumbersMatch(mainBatchSeqNums, attachFlowBatchSeqNums),
90
105
  0x58d /* Reference sequence numbers from both batches must be in sync */,
91
106
  );
92
107
 
108
+ const currentSequenceNumbers = this.params.getCurrentSequenceNumbers();
109
+
93
110
  if (
94
- (mainBatchReference === undefined ||
95
- mainBatchReference === message.referenceSequenceNumber) &&
96
- (attachFlowBatchReference === undefined ||
97
- attachFlowBatchReference === message.referenceSequenceNumber)
111
+ sequenceNumbersMatch(mainBatchSeqNums, currentSequenceNumbers) &&
112
+ sequenceNumbersMatch(attachFlowBatchSeqNums, currentSequenceNumbers)
98
113
  ) {
99
114
  // The reference sequence numbers are stable, there is nothing to do
100
115
  return;
101
116
  }
102
117
 
103
118
  if (++this.mismatchedOpsReported <= this.maxMismatchedOpsToReport) {
104
- this.mc.logger.sendErrorEvent(
119
+ this.mc.logger.sendTelemetryEvent(
105
120
  {
121
+ category: this.params.config.disablePartialFlush ? "error" : "generic",
106
122
  eventName: "ReferenceSequenceNumberMismatch",
107
- mainReferenceSequenceNumber: mainBatchReference,
108
- attachReferenceSequenceNumber: attachFlowBatchReference,
109
- messageReferenceSequenceNumber: message.referenceSequenceNumber,
123
+ mainReferenceSequenceNumber: mainBatchSeqNums.referenceSequenceNumber,
124
+ mainClientSequenceNumber: mainBatchSeqNums.clientSequenceNumber,
125
+ attachReferenceSequenceNumber: attachFlowBatchSeqNums.referenceSequenceNumber,
126
+ attachClientSequenceNumber: attachFlowBatchSeqNums.clientSequenceNumber,
127
+ currentReferenceSequenceNumber: currentSequenceNumbers.referenceSequenceNumber,
128
+ currentClientSequenceNumber: currentSequenceNumbers.clientSequenceNumber,
110
129
  },
111
- new UsageError("Submission of an out of order message"),
130
+ getLongStack(() => new UsageError("Submission of an out of order message")),
112
131
  );
113
132
  }
114
133
 
@@ -118,9 +137,14 @@ export class Outbox {
118
137
  }
119
138
 
120
139
  public submit(message: BatchMessage) {
121
- this.maybeFlushPartialBatch(message);
140
+ this.maybeFlushPartialBatch();
122
141
 
123
- if (!this.mainBatch.push(message)) {
142
+ if (
143
+ !this.mainBatch.push(
144
+ message,
145
+ this.params.getCurrentSequenceNumbers().clientSequenceNumber,
146
+ )
147
+ ) {
124
148
  throw new GenericError("BatchTooLarge", /* error */ undefined, {
125
149
  opSize: message.contents?.length ?? 0,
126
150
  batchSize: this.mainBatch.contentSizeInBytes,
@@ -131,14 +155,24 @@ export class Outbox {
131
155
  }
132
156
 
133
157
  public submitAttach(message: BatchMessage) {
134
- this.maybeFlushPartialBatch(message);
158
+ this.maybeFlushPartialBatch();
135
159
 
136
- if (!this.attachFlowBatch.push(message)) {
160
+ if (
161
+ !this.attachFlowBatch.push(
162
+ message,
163
+ this.params.getCurrentSequenceNumbers().clientSequenceNumber,
164
+ )
165
+ ) {
137
166
  // BatchManager has two limits - soft limit & hard limit. Soft limit is only engaged
138
167
  // when queue is not empty.
139
168
  // Flush queue & retry. Failure on retry would mean - single message is bigger than hard limit
140
169
  this.flushInternal(this.attachFlowBatch.popBatch());
141
- if (!this.attachFlowBatch.push(message)) {
170
+ if (
171
+ !this.attachFlowBatch.push(
172
+ message,
173
+ this.params.getCurrentSequenceNumbers().clientSequenceNumber,
174
+ )
175
+ ) {
142
176
  throw new GenericError("BatchTooLarge", /* error */ undefined, {
143
177
  opSize: message.contents?.length ?? 0,
144
178
  batchSize: this.attachFlowBatch.contentSizeInBytes,
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "2.0.0-dev.4.2.0.153917";
9
+ export const pkgVersion = "2.0.0-dev.4.3.0.158678";
@@ -49,10 +49,7 @@ export interface IRuntimeStateHandler {
49
49
  connected(): boolean;
50
50
  clientId(): string | undefined;
51
51
  close(error?: ICriticalContainerError): void;
52
- applyStashedOp: (
53
- type: ContainerMessageType,
54
- content: ISequencedDocumentMessage,
55
- ) => Promise<unknown>;
52
+ applyStashedOp: (type: ContainerMessageType, content: unknown) => Promise<unknown>;
56
53
  reSubmit(
57
54
  type: ContainerMessageType,
58
55
  content: any,
@@ -110,9 +107,20 @@ export class PendingStateManager implements IDisposable {
110
107
  return {
111
108
  // delete localOpMetadata since it may not be serializable
112
109
  // and will be regenerated by applyStashedOp()
113
- pendingStates: this.pendingMessages
114
- .toArray()
115
- .map((message) => ({ ...message, localOpMetadata: undefined })),
110
+ pendingStates: this.pendingMessages.toArray().map((message) =>
111
+ // IdAllocations need their localOpMetadata stashed in the contents
112
+ // of the op to correctly resume the session when processing stashed ops
113
+ message.messageType === ContainerMessageType.IdAllocation
114
+ ? {
115
+ ...message,
116
+ content: {
117
+ ...message.content,
118
+ stashedState: message.localOpMetadata,
119
+ },
120
+ localOpMetadata: undefined,
121
+ }
122
+ : { ...message, localOpMetadata: undefined },
123
+ ),
116
124
  };
117
125
  }
118
126
  }
@@ -211,6 +219,12 @@ export class PendingStateManager implements IDisposable {
211
219
  );
212
220
  nextMessage.localOpMetadata = localOpMetadata;
213
221
 
222
+ if (nextMessage.messageType === ContainerMessageType.IdAllocation) {
223
+ // Remove the stashed state from the op
224
+ // so that it doesn't go over the wire
225
+ delete nextMessage.content.stashedState;
226
+ }
227
+
214
228
  // then we push onto pendingMessages which will cause PendingStateManager to resubmit when we connect
215
229
  // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
216
230
  this.pendingMessages.push(this.initialMessages.shift()!);
@@ -94,6 +94,7 @@ export {
94
94
  rootHasIsolatedChannels,
95
95
  WriteFluidDataStoreAttributes,
96
96
  wrapSummaryInChannelsTree,
97
+ idCompressorBlobName,
97
98
  } from "./summaryFormat";
98
99
  export { getFailMessage, SummarizeReason } from "./summaryGenerator";
99
100
  export {
@@ -336,7 +336,7 @@ export class OrderedClientElection
336
336
  }
337
337
 
338
338
  constructor(
339
- logger: ITelemetryLogger,
339
+ private readonly logger: ITelemetryLogger,
340
340
  private readonly orderedClientCollection: IOrderedClientCollection,
341
341
  /** Serialized state from summary or current sequence number at time of load if new. */
342
342
  initialState: ISerializedElection | number,
@@ -416,6 +416,13 @@ export class OrderedClientElection
416
416
  change = true;
417
417
  }
418
418
  if (change) {
419
+ this.logger.sendTelemetryEvent({
420
+ eventName: "SummarizerClientElected",
421
+ electedClientId: this._electedClient?.clientId,
422
+ electedParentId: this._electedParent?.clientId,
423
+ electionSequenceNumber: sequenceNumber,
424
+ isSummarizerClient,
425
+ });
419
426
  this.emit("election", client, sequenceNumber, prevClient);
420
427
  }
421
428
  }
@@ -423,6 +430,12 @@ export class OrderedClientElection
423
430
  private tryElectingParent(client: ILinkedClient | undefined, sequenceNumber: number): void {
424
431
  if (this._electedParent !== client) {
425
432
  this._electedParent = client;
433
+ this.logger.sendTelemetryEvent({
434
+ eventName: "SummarizerParentElected",
435
+ electedClientId: this._electedClient?.clientId,
436
+ electedParentId: this._electedParent?.clientId,
437
+ electionSequenceNumber: sequenceNumber,
438
+ });
426
439
  this.emit("election", this._electedClient, sequenceNumber, this._electedClient);
427
440
  }
428
441
  }
@@ -457,7 +470,7 @@ export class OrderedClientElection
457
470
  const newClientIsSummarizer = client.client.details.type === summarizerClientType;
458
471
  const electedClientIsSummarizer =
459
472
  this._electedClient?.client.details.type === summarizerClientType;
460
- // Note that we allow a summarizer client to supercede an interactive client as elected client.
473
+ // Note that we allow a summarizer client to supersede an interactive client as elected client.
461
474
  if (
462
475
  this._electedClient === undefined ||
463
476
  (!electedClientIsSummarizer && newClientIsSummarizer)
@@ -46,7 +46,6 @@ import {
46
46
  const maxSummarizeAckWaitTime = 10 * 60 * 1000; // 10 minutes
47
47
 
48
48
  const defaultNumberSummarizationAttempts = 2; // only up to 2 attempts
49
- const numberOfAttemptsOnRestartAsRecovery = 1; // Only summarize once
50
49
 
51
50
  /**
52
51
  * An instance of RunningSummarizer manages the heuristics for summarizing.
@@ -133,7 +132,6 @@ export class RunningSummarizer implements IDisposable {
133
132
  private heuristicRunner?: ISummarizeHeuristicRunner;
134
133
  private readonly generator: SummaryGenerator;
135
134
  private readonly mc: MonitoringContext;
136
- private readonly shouldAbortOnSummaryFailure: boolean;
137
135
 
138
136
  private enqueuedSummary:
139
137
  | {
@@ -177,10 +175,6 @@ export class RunningSummarizer implements IDisposable {
177
175
  }),
178
176
  );
179
177
 
180
- this.shouldAbortOnSummaryFailure =
181
- this.mc.config.getString("Fluid.ContainerRuntime.Test.SummarizationRecoveryMethod") ===
182
- "restart";
183
-
184
178
  if (configuration.state !== "disableHeuristics") {
185
179
  assert(
186
180
  this.configuration.state === "enabled",
@@ -601,10 +595,9 @@ export class RunningSummarizer implements IDisposable {
601
595
  let summaryAttempts = 0;
602
596
  let summaryAttemptsPerPhase = 0;
603
597
  // Reducing the default number of attempts to defaultNumberofSummarizationAttempts.
604
- let totalAttempts = this.shouldAbortOnSummaryFailure
605
- ? numberOfAttemptsOnRestartAsRecovery
606
- : this.mc.config.getNumber("Fluid.Summarizer.Attempts") ??
607
- defaultNumberSummarizationAttempts;
598
+ let totalAttempts =
599
+ this.mc.config.getNumber("Fluid.Summarizer.Attempts") ??
600
+ defaultNumberSummarizationAttempts;
608
601
 
609
602
  if (totalAttempts > attempts.length) {
610
603
  this.mc.logger.sendTelemetryEvent({
@@ -676,20 +669,6 @@ export class RunningSummarizer implements IDisposable {
676
669
  }
677
670
  }
678
671
 
679
- if (this.shouldAbortOnSummaryFailure) {
680
- this.mc.logger.sendTelemetryEvent(
681
- {
682
- eventName: "ClosingSummarizerOnSummaryStale",
683
- reason,
684
- message: lastResult?.message,
685
- },
686
- lastResult?.error,
687
- );
688
-
689
- this.stopSummarizerCallback("latestSummaryStateStale");
690
- this.runtime.closeFn();
691
- return;
692
- }
693
672
  // If all attempts failed, log error (with last attempt info) and close the summarizer container
694
673
  this.mc.logger.sendErrorEvent(
695
674
  {