@fluidframework/container-runtime 1.2.6 → 2.0.0-dev.1.3.0.96595

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/.mocharc.js +12 -0
  2. package/dist/batchManager.d.ts +37 -0
  3. package/dist/batchManager.d.ts.map +1 -0
  4. package/dist/batchManager.js +73 -0
  5. package/dist/batchManager.js.map +1 -0
  6. package/dist/batchTracker.d.ts +1 -2
  7. package/dist/batchTracker.d.ts.map +1 -1
  8. package/dist/batchTracker.js +2 -3
  9. package/dist/batchTracker.js.map +1 -1
  10. package/dist/blobManager.d.ts +87 -25
  11. package/dist/blobManager.d.ts.map +1 -1
  12. package/dist/blobManager.js +317 -99
  13. package/dist/blobManager.js.map +1 -1
  14. package/dist/containerRuntime.d.ts +109 -124
  15. package/dist/containerRuntime.d.ts.map +1 -1
  16. package/dist/containerRuntime.js +349 -542
  17. package/dist/containerRuntime.js.map +1 -1
  18. package/dist/dataStore.js +29 -24
  19. package/dist/dataStore.js.map +1 -1
  20. package/dist/dataStoreContext.d.ts +20 -14
  21. package/dist/dataStoreContext.d.ts.map +1 -1
  22. package/dist/dataStoreContext.js +49 -58
  23. package/dist/dataStoreContext.js.map +1 -1
  24. package/dist/dataStores.d.ts +12 -5
  25. package/dist/dataStores.d.ts.map +1 -1
  26. package/dist/dataStores.js +21 -20
  27. package/dist/dataStores.js.map +1 -1
  28. package/dist/deltaScheduler.d.ts +6 -4
  29. package/dist/deltaScheduler.d.ts.map +1 -1
  30. package/dist/deltaScheduler.js +6 -4
  31. package/dist/deltaScheduler.js.map +1 -1
  32. package/dist/garbageCollection.d.ts +74 -14
  33. package/dist/garbageCollection.d.ts.map +1 -1
  34. package/dist/garbageCollection.js +249 -170
  35. package/dist/garbageCollection.js.map +1 -1
  36. package/dist/gcSweepReadyUsageDetection.d.ts +53 -0
  37. package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -0
  38. package/dist/gcSweepReadyUsageDetection.js +126 -0
  39. package/dist/gcSweepReadyUsageDetection.js.map +1 -0
  40. package/dist/index.d.ts +2 -1
  41. package/dist/index.d.ts.map +1 -1
  42. package/dist/index.js +3 -2
  43. package/dist/index.js.map +1 -1
  44. package/dist/opProperties.d.ts +7 -0
  45. package/dist/opProperties.d.ts.map +1 -0
  46. package/dist/opProperties.js +20 -0
  47. package/dist/opProperties.js.map +1 -0
  48. package/dist/orderedClientElection.d.ts +28 -10
  49. package/dist/orderedClientElection.d.ts.map +1 -1
  50. package/dist/orderedClientElection.js +14 -4
  51. package/dist/orderedClientElection.js.map +1 -1
  52. package/dist/packageVersion.d.ts +1 -1
  53. package/dist/packageVersion.d.ts.map +1 -1
  54. package/dist/packageVersion.js +1 -1
  55. package/dist/packageVersion.js.map +1 -1
  56. package/dist/pendingStateManager.d.ts +0 -11
  57. package/dist/pendingStateManager.d.ts.map +1 -1
  58. package/dist/pendingStateManager.js +24 -46
  59. package/dist/pendingStateManager.js.map +1 -1
  60. package/dist/runningSummarizer.d.ts +14 -4
  61. package/dist/runningSummarizer.d.ts.map +1 -1
  62. package/dist/runningSummarizer.js +68 -26
  63. package/dist/runningSummarizer.js.map +1 -1
  64. package/dist/scheduleManager.d.ts +31 -0
  65. package/dist/scheduleManager.d.ts.map +1 -0
  66. package/dist/scheduleManager.js +243 -0
  67. package/dist/scheduleManager.js.map +1 -0
  68. package/dist/summarizer.d.ts +0 -2
  69. package/dist/summarizer.d.ts.map +1 -1
  70. package/dist/summarizer.js +1 -12
  71. package/dist/summarizer.js.map +1 -1
  72. package/dist/summarizerHeuristics.d.ts +26 -4
  73. package/dist/summarizerHeuristics.d.ts.map +1 -1
  74. package/dist/summarizerHeuristics.js +95 -18
  75. package/dist/summarizerHeuristics.js.map +1 -1
  76. package/dist/summarizerTypes.d.ts +45 -18
  77. package/dist/summarizerTypes.d.ts.map +1 -1
  78. package/dist/summarizerTypes.js +1 -1
  79. package/dist/summarizerTypes.js.map +1 -1
  80. package/dist/summaryCollection.d.ts +1 -0
  81. package/dist/summaryCollection.d.ts.map +1 -1
  82. package/dist/summaryCollection.js +31 -15
  83. package/dist/summaryCollection.js.map +1 -1
  84. package/dist/summaryFormat.d.ts +0 -5
  85. package/dist/summaryFormat.d.ts.map +1 -1
  86. package/dist/summaryFormat.js.map +1 -1
  87. package/dist/summaryGenerator.d.ts +1 -0
  88. package/dist/summaryGenerator.d.ts.map +1 -1
  89. package/dist/summaryGenerator.js +11 -9
  90. package/dist/summaryGenerator.js.map +1 -1
  91. package/dist/summaryManager.d.ts +2 -2
  92. package/dist/summaryManager.d.ts.map +1 -1
  93. package/dist/summaryManager.js +22 -7
  94. package/dist/summaryManager.js.map +1 -1
  95. package/lib/batchManager.d.ts +37 -0
  96. package/lib/batchManager.d.ts.map +1 -0
  97. package/lib/batchManager.js +69 -0
  98. package/lib/batchManager.js.map +1 -0
  99. package/lib/batchTracker.d.ts +1 -2
  100. package/lib/batchTracker.d.ts.map +1 -1
  101. package/lib/batchTracker.js +2 -3
  102. package/lib/batchTracker.js.map +1 -1
  103. package/lib/blobManager.d.ts +87 -25
  104. package/lib/blobManager.d.ts.map +1 -1
  105. package/lib/blobManager.js +319 -101
  106. package/lib/blobManager.js.map +1 -1
  107. package/lib/containerRuntime.d.ts +109 -124
  108. package/lib/containerRuntime.d.ts.map +1 -1
  109. package/lib/containerRuntime.js +355 -547
  110. package/lib/containerRuntime.js.map +1 -1
  111. package/lib/dataStore.js +29 -24
  112. package/lib/dataStore.js.map +1 -1
  113. package/lib/dataStoreContext.d.ts +20 -14
  114. package/lib/dataStoreContext.d.ts.map +1 -1
  115. package/lib/dataStoreContext.js +46 -55
  116. package/lib/dataStoreContext.js.map +1 -1
  117. package/lib/dataStores.d.ts +12 -5
  118. package/lib/dataStores.d.ts.map +1 -1
  119. package/lib/dataStores.js +21 -20
  120. package/lib/dataStores.js.map +1 -1
  121. package/lib/deltaScheduler.d.ts +6 -4
  122. package/lib/deltaScheduler.d.ts.map +1 -1
  123. package/lib/deltaScheduler.js +6 -4
  124. package/lib/deltaScheduler.js.map +1 -1
  125. package/lib/garbageCollection.d.ts +74 -14
  126. package/lib/garbageCollection.d.ts.map +1 -1
  127. package/lib/garbageCollection.js +238 -160
  128. package/lib/garbageCollection.js.map +1 -1
  129. package/lib/gcSweepReadyUsageDetection.d.ts +53 -0
  130. package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -0
  131. package/lib/gcSweepReadyUsageDetection.js +121 -0
  132. package/lib/gcSweepReadyUsageDetection.js.map +1 -0
  133. package/lib/index.d.ts +2 -1
  134. package/lib/index.d.ts.map +1 -1
  135. package/lib/index.js +2 -1
  136. package/lib/index.js.map +1 -1
  137. package/lib/opProperties.d.ts +7 -0
  138. package/lib/opProperties.d.ts.map +1 -0
  139. package/lib/opProperties.js +16 -0
  140. package/lib/opProperties.js.map +1 -0
  141. package/lib/orderedClientElection.d.ts +28 -10
  142. package/lib/orderedClientElection.d.ts.map +1 -1
  143. package/lib/orderedClientElection.js +14 -4
  144. package/lib/orderedClientElection.js.map +1 -1
  145. package/lib/packageVersion.d.ts +1 -1
  146. package/lib/packageVersion.d.ts.map +1 -1
  147. package/lib/packageVersion.js +1 -1
  148. package/lib/packageVersion.js.map +1 -1
  149. package/lib/pendingStateManager.d.ts +0 -11
  150. package/lib/pendingStateManager.d.ts.map +1 -1
  151. package/lib/pendingStateManager.js +24 -46
  152. package/lib/pendingStateManager.js.map +1 -1
  153. package/lib/runningSummarizer.d.ts +14 -4
  154. package/lib/runningSummarizer.d.ts.map +1 -1
  155. package/lib/runningSummarizer.js +68 -26
  156. package/lib/runningSummarizer.js.map +1 -1
  157. package/lib/scheduleManager.d.ts +31 -0
  158. package/lib/scheduleManager.d.ts.map +1 -0
  159. package/lib/scheduleManager.js +239 -0
  160. package/lib/scheduleManager.js.map +1 -0
  161. package/lib/summarizer.d.ts +0 -2
  162. package/lib/summarizer.d.ts.map +1 -1
  163. package/lib/summarizer.js +1 -12
  164. package/lib/summarizer.js.map +1 -1
  165. package/lib/summarizerHeuristics.d.ts +26 -4
  166. package/lib/summarizerHeuristics.d.ts.map +1 -1
  167. package/lib/summarizerHeuristics.js +95 -18
  168. package/lib/summarizerHeuristics.js.map +1 -1
  169. package/lib/summarizerTypes.d.ts +45 -18
  170. package/lib/summarizerTypes.d.ts.map +1 -1
  171. package/lib/summarizerTypes.js +1 -1
  172. package/lib/summarizerTypes.js.map +1 -1
  173. package/lib/summaryCollection.d.ts +1 -0
  174. package/lib/summaryCollection.d.ts.map +1 -1
  175. package/lib/summaryCollection.js +31 -15
  176. package/lib/summaryCollection.js.map +1 -1
  177. package/lib/summaryFormat.d.ts +0 -5
  178. package/lib/summaryFormat.d.ts.map +1 -1
  179. package/lib/summaryFormat.js.map +1 -1
  180. package/lib/summaryGenerator.d.ts +1 -0
  181. package/lib/summaryGenerator.d.ts.map +1 -1
  182. package/lib/summaryGenerator.js +11 -9
  183. package/lib/summaryGenerator.js.map +1 -1
  184. package/lib/summaryManager.d.ts +2 -2
  185. package/lib/summaryManager.d.ts.map +1 -1
  186. package/lib/summaryManager.js +22 -7
  187. package/lib/summaryManager.js.map +1 -1
  188. package/package.json +65 -24
  189. package/src/batchManager.ts +91 -0
  190. package/src/batchTracker.ts +2 -3
  191. package/src/blobManager.ts +385 -118
  192. package/src/containerRuntime.ts +529 -740
  193. package/src/dataStore.ts +49 -37
  194. package/src/dataStoreContext.ts +44 -56
  195. package/src/dataStores.ts +34 -30
  196. package/src/deltaScheduler.ts +6 -4
  197. package/src/garbageCollection.ts +297 -206
  198. package/src/gcSweepReadyUsageDetection.ts +139 -0
  199. package/src/index.ts +1 -2
  200. package/src/opProperties.ts +19 -0
  201. package/src/orderedClientElection.ts +31 -10
  202. package/src/packageVersion.ts +1 -1
  203. package/src/pendingStateManager.ts +27 -59
  204. package/src/runningSummarizer.ts +75 -22
  205. package/src/scheduleManager.ts +314 -0
  206. package/src/summarizer.ts +1 -18
  207. package/src/summarizerHeuristics.ts +133 -19
  208. package/src/summarizerTypes.ts +53 -18
  209. package/src/summaryCollection.ts +33 -18
  210. package/src/summaryFormat.ts +0 -6
  211. package/src/summaryGenerator.ts +40 -22
  212. package/src/summaryManager.ts +22 -7
  213. package/dist/opTelemetry.d.ts +0 -22
  214. package/dist/opTelemetry.d.ts.map +0 -1
  215. package/dist/opTelemetry.js +0 -59
  216. package/dist/opTelemetry.js.map +0 -1
  217. package/lib/opTelemetry.d.ts +0 -22
  218. package/lib/opTelemetry.d.ts.map +0 -1
  219. package/lib/opTelemetry.js +0 -55
  220. package/lib/opTelemetry.js.map +0 -1
  221. package/src/opTelemetry.ts +0 -71
package/.mocharc.js ADDED
@@ -0,0 +1,12 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ 'use strict';
7
+
8
+ const getFluidTestMochaConfig = require('@fluidframework/mocha-test-setup/mocharc-common');
9
+
10
+ const packageDir = __dirname;
11
+ const config = getFluidTestMochaConfig(packageDir);
12
+ module.exports = config;
@@ -0,0 +1,37 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+ import { IBatchMessage } from "@fluidframework/container-definitions";
6
+ import { ContainerRuntimeMessage } from "./containerRuntime";
7
+ /**
8
+ * Message type used by BatchManager
9
+ */
10
+ export declare type BatchMessage = IBatchMessage & {
11
+ localOpMetadata: unknown;
12
+ deserializedContent: ContainerRuntimeMessage;
13
+ referenceSequenceNumber: number;
14
+ };
15
+ /**
16
+ * Helper class that manages partial batch & rollback.
17
+ */
18
+ export declare class BatchManager {
19
+ readonly softLimit?: number | undefined;
20
+ private pendingBatch;
21
+ private batchContentSize;
22
+ private static readonly hardLimit;
23
+ get length(): number;
24
+ get limit(): number;
25
+ static get limit(): number;
26
+ constructor(softLimit?: number | undefined);
27
+ push(message: BatchMessage): boolean;
28
+ get empty(): boolean;
29
+ popBatch(): BatchMessage[];
30
+ /**
31
+ * Capture the pending state at this point
32
+ */
33
+ checkpoint(): {
34
+ rollback: (process: (message: BatchMessage) => void) => void;
35
+ };
36
+ }
37
+ //# sourceMappingURL=batchManager.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batchManager.d.ts","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,uBAAuB,EAAE,MAAM,oBAAoB,CAAC;AAE7D;;GAEG;AACH,oBAAY,YAAY,GAAG,aAAa,GAAG;IACvC,eAAe,EAAE,OAAO,CAAC;IACzB,mBAAmB,EAAE,uBAAuB,CAAC;IAC7C,uBAAuB,EAAE,MAAM,CAAC;CACnC,CAAC;AAEF;;GAEG;AACH,qBAAa,YAAY;aAcO,SAAS,CAAC;IAbtC,OAAO,CAAC,YAAY,CAAuB;IAC3C,OAAO,CAAC,gBAAgB,CAAK;IAM7B,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAc;IAE/C,IAAW,MAAM,WAAuC;IACxD,IAAW,KAAK,WAAqC;IACrD,WAAkB,KAAK,WAAqC;gBAEhC,SAAS,CAAC,oBAAQ;IAEvC,IAAI,CAAC,OAAO,EAAE,YAAY,GAAG,OAAO;IA2B3C,IAAW,KAAK,YAA6C;IAEtD,QAAQ;IAOf;;OAEG;IACK,UAAU;sCAGoB,YAAY,KAAK,IAAI;;CAY9D"}
@@ -0,0 +1,73 @@
1
+ "use strict";
2
+ /*!
3
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
4
+ * Licensed under the MIT License.
5
+ */
6
+ Object.defineProperty(exports, "__esModule", { value: true });
7
+ exports.BatchManager = void 0;
8
+ /**
9
+ * Helper class that manages partial batch & rollback.
10
+ */
11
+ class BatchManager {
12
+ constructor(softLimit) {
13
+ this.softLimit = softLimit;
14
+ this.pendingBatch = [];
15
+ this.batchContentSize = 0;
16
+ }
17
+ get length() { return this.pendingBatch.length; }
18
+ get limit() { return BatchManager.hardLimit; }
19
+ static get limit() { return BatchManager.hardLimit; }
20
+ push(message) {
21
+ const contentSize = this.batchContentSize + message.contents.length;
22
+ const opCount = this.pendingBatch.length;
23
+ // Attempt to estimate batch size, aka socket message size.
24
+ // Each op has pretty large envelope, estimating to be 200 bytes.
25
+ // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.
26
+ // Not taking it into account, as compression work should help there - compressed payload will be
27
+ // initially stored as base64, and that requires only 2 extra escape characters.
28
+ const socketMessageSize = contentSize + 200 * opCount;
29
+ // If we were provided soft limit, check for exceeding it.
30
+ // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)
31
+ // and start over. That's not an option if we have no ops.
32
+ if (this.softLimit !== undefined && this.length > 0 && socketMessageSize >= this.softLimit) {
33
+ return false;
34
+ }
35
+ if (socketMessageSize >= this.limit) {
36
+ return false;
37
+ }
38
+ this.batchContentSize = contentSize;
39
+ this.pendingBatch.push(message);
40
+ return true;
41
+ }
42
+ get empty() { return this.pendingBatch.length === 0; }
43
+ popBatch() {
44
+ const batch = this.pendingBatch;
45
+ this.pendingBatch = [];
46
+ this.batchContentSize = 0;
47
+ return batch;
48
+ }
49
+ /**
50
+ * Capture the pending state at this point
51
+ */
52
+ checkpoint() {
53
+ const startPoint = this.pendingBatch.length;
54
+ return {
55
+ rollback: (process) => {
56
+ for (let i = this.pendingBatch.length; i > startPoint;) {
57
+ i--;
58
+ const message = this.pendingBatch[i];
59
+ this.batchContentSize -= message.contents.length;
60
+ process(message);
61
+ }
62
+ this.pendingBatch.length = startPoint;
63
+ },
64
+ };
65
+ }
66
+ }
67
+ exports.BatchManager = BatchManager;
68
+ // The actual limit is 1Mb (socket.io and Kafka limits)
69
+ // We can't estimate it fully, as we
70
+ // - do not know what properties relay service will add
71
+ // - we do not stringify final op, thus we do not know how much escaping will be added.
72
+ BatchManager.hardLimit = 950 * 1024;
73
+ //# sourceMappingURL=batchManager.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"batchManager.js","sourceRoot":"","sources":["../src/batchManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAcH;;GAEG;AACH,MAAa,YAAY;IAcrB,YAA4B,SAAkB;QAAlB,cAAS,GAAT,SAAS,CAAS;QAbtC,iBAAY,GAAoB,EAAE,CAAC;QACnC,qBAAgB,GAAG,CAAC,CAAC;IAYoB,CAAC;IAJlD,IAAW,MAAM,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;IACxD,IAAW,KAAK,KAAK,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAC9C,MAAM,KAAK,KAAK,KAAK,OAAO,YAAY,CAAC,SAAS,CAAC,CAAC,CAAC;IAIrD,IAAI,CAAC,OAAqB;QAC7B,MAAM,WAAW,GAAG,IAAI,CAAC,gBAAgB,GAAG,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;QACpE,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAEzC,2DAA2D;QAC3D,iEAAiE;QACjE,sGAAsG;QACtG,iGAAiG;QACjG,gFAAgF;QAChF,MAAM,iBAAiB,GAAG,WAAW,GAAG,GAAG,GAAG,OAAO,CAAC;QAEtD,0DAA0D;QAC1D,wGAAwG;QACxG,0DAA0D;QAC1D,IAAI,IAAI,CAAC,SAAS,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,iBAAiB,IAAI,IAAI,CAAC,SAAS,EAAE;YACxF,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,iBAAiB,IAAI,IAAI,CAAC,KAAK,EAAE;YACjC,OAAO,KAAK,CAAC;SAChB;QAED,IAAI,CAAC,gBAAgB,GAAG,WAAW,CAAC;QACpC,IAAI,CAAC,YAAY,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAChC,OAAO,IAAI,CAAC;IAChB,CAAC;IAED,IAAW,KAAK,KAAK,OAAO,IAAI,CAAC,YAAY,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC;IAEtD,QAAQ;QACX,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,CAAC;QAChC,IAAI,CAAC,YAAY,GAAG,EAAE,CAAC;QACvB,IAAI,CAAC,gBAAgB,GAAG,CAAC,CAAC;QAC1B,OAAO,KAAK,CAAC;IACjB,CAAC;IAED;;OAEG;IACK,UAAU;QACd,MAAM,UAAU,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;QAC5C,OAAO;YACH,QAAQ,EAAE,CAAC,OAAwC,EAAE,EAAE;gBACnD,KAAK,IAAI,CAAC,GAAG,IAAI,CAAC,YAAY,CAAC,MAAM,EAAE,CAAC,GAAG,UAAU,GAAG;oBACpD,CAAC,EAAE,CAAC;oBACJ,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,CAAC;oBACrC,IAAI,CAAC,gBAAgB,IAAI,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;oBACjD,OAAO,CAAC,OAAO,CAAC,CAAC;iBACpB;gBAED,IAAI,CAAC,YAAY,CAAC,MAAM,GAAG,UAAU,CAAC;YAC1C,CAAC;SACJ,CAAC;IACN,CAAC;;AArEL,oCAsEC;AAlEG,uDAAuD;AACvD,oCAAoC;AACpC,uDAAuD;AACvD,uFAAuF;AAC/D,sBAAS,GAAG,GAAG,GAAG,IAAI,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { IBatchMessage } from \"@fluidframework/container-definitions\";\nimport { ContainerRuntimeMessage } from \"./containerRuntime\";\n\n/**\n * Message type used by BatchManager\n */\nexport type BatchMessage = IBatchMessage & {\n localOpMetadata: unknown;\n deserializedContent: ContainerRuntimeMessage;\n referenceSequenceNumber: number;\n};\n\n/**\n * Helper class that manages partial batch & rollback.\n */\nexport class BatchManager {\n private pendingBatch: BatchMessage [] = [];\n private batchContentSize = 0;\n\n // The actual limit is 1Mb (socket.io and Kafka limits)\n // We can't estimate it fully, as we\n // - do not know what properties relay service will add\n // - we do not stringify final op, thus we do not know how much escaping will be added.\n private static readonly hardLimit = 950 * 1024;\n\n public get length() { return this.pendingBatch.length; }\n public get limit() { return BatchManager.hardLimit; }\n public static get limit() { return BatchManager.hardLimit; }\n\n constructor(public readonly softLimit?: number) {}\n\n public push(message: BatchMessage): boolean {\n const contentSize = this.batchContentSize + message.contents.length;\n const opCount = this.pendingBatch.length;\n\n // Attempt to estimate batch size, aka socket message size.\n // Each op has pretty large envelope, estimating to be 200 bytes.\n // Also content will be strigified, and that adds a lot of overhead due to a lot of escape characters.\n // Not taking it into account, as compression work should help there - compressed payload will be\n // initially stored as base64, and that requires only 2 extra escape characters.\n const socketMessageSize = contentSize + 200 * opCount;\n\n // If we were provided soft limit, check for exceeding it.\n // But only if we have any ops, as the intention here is to flush existing ops (on exceeding this limit)\n // and start over. That's not an option if we have no ops.\n if (this.softLimit !== undefined && this.length > 0 && socketMessageSize >= this.softLimit) {\n return false;\n }\n\n if (socketMessageSize >= this.limit) {\n return false;\n }\n\n this.batchContentSize = contentSize;\n this.pendingBatch.push(message);\n return true;\n }\n\n public get empty() { return this.pendingBatch.length === 0; }\n\n public popBatch() {\n const batch = this.pendingBatch;\n this.pendingBatch = [];\n this.batchContentSize = 0;\n return batch;\n }\n\n /**\n * Capture the pending state at this point\n */\n public checkpoint() {\n const startPoint = this.pendingBatch.length;\n return {\n rollback: (process: (message: BatchMessage) => void) => {\n for (let i = this.pendingBatch.length; i > startPoint;) {\n i--;\n const message = this.pendingBatch[i];\n this.batchContentSize -= message.contents.length;\n process(message);\n }\n\n this.pendingBatch.length = startPoint;\n },\n };\n }\n}\n"]}
@@ -17,10 +17,9 @@ export declare class BatchTracker {
17
17
  * Track batch sizes in terms of op counts and processing times
18
18
  *
19
19
  * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations
20
- * @param logger - logger
20
+ * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLogger}
21
21
  * @param batchLengthThreshold - threshold for the length of a batch when to send an error event
22
22
  * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics
23
- * @returns
24
23
  */
25
24
  export declare const BindBatchTracker: (batchEventEmitter: EventEmitter, logger: ITelemetryLogger, batchLengthThreshold?: number, batchCountSamplingRate?: number) => BatchTracker;
26
25
  //# sourceMappingURL=batchTracker.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"batchTracker.d.ts","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG;;AAEH,OAAO,YAAY,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAKtE,qBAAa,YAAY;IAOjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IANtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,wBAAwB,CAAqB;IACrD,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,6BAA6B,CAAqB;gBAGrC,iBAAiB,EAAE,YAAY,EAChD,MAAM,EAAE,gBAAgB,EACxB,oBAAoB,EAAE,MAAM,EAC5B,sBAAsB,EAAE,MAAM,EAC9B,gBAAgB,GAAE,MAAM,MAAgC;CAyC/D;AAED;;;;;;;;GAQG;AACH,eAAO,MAAM,gBAAgB,sBACN,YAAY,UACvB,gBAAgB,yBACF,MAAM,2BACJ,MAAM,iBAC4D,CAAC"}
1
+ {"version":3,"file":"batchTracker.d.ts","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG;;AAEH,OAAO,YAAY,MAAM,QAAQ,CAAC;AAClC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAKtE,qBAAa,YAAY;IAOjB,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IANtC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,wBAAwB,CAAqB;IACrD,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,6BAA6B,CAAqB;gBAGrC,iBAAiB,EAAE,YAAY,EAChD,MAAM,EAAE,gBAAgB,EACxB,oBAAoB,EAAE,MAAM,EAC5B,sBAAsB,EAAE,MAAM,EAC9B,gBAAgB,GAAE,MAAM,MAAgC;CAyC/D;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,sBACN,YAAY,UACvB,gBAAgB,yBACF,MAAM,2BACJ,MAAM,iBAC4D,CAAC"}
@@ -21,7 +21,7 @@ class BatchTracker {
21
21
  (0, common_utils_1.assert)(this.startBatchSequenceNumber !== undefined && this.batchProcessingStartTimeStamp !== undefined, 0x2ba /* "batchBegin must fire before batchEnd" */);
22
22
  const length = message.sequenceNumber - this.startBatchSequenceNumber + 1;
23
23
  if (length >= batchLengthThreshold) {
24
- this.logger.sendErrorEvent({
24
+ this.logger.sendPerformanceEvent({
25
25
  eventName: "LengthTooBig",
26
26
  length,
27
27
  threshold: batchLengthThreshold,
@@ -49,10 +49,9 @@ exports.BatchTracker = BatchTracker;
49
49
  * Track batch sizes in terms of op counts and processing times
50
50
  *
51
51
  * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations
52
- * @param logger - logger
52
+ * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLogger}
53
53
  * @param batchLengthThreshold - threshold for the length of a batch when to send an error event
54
54
  * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics
55
- * @returns
56
55
  */
57
56
  const BindBatchTracker = (batchEventEmitter, logger, batchLengthThreshold = 1000, batchCountSamplingRate = 1000) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);
58
57
  exports.BindBatchTracker = BindBatchTracker;
@@ -1 +1 @@
1
- {"version":3,"file":"batchTracker.js","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,+DAAmE;AAEnE,qEAA8D;AAE9D,MAAa,YAAY;IAMrB,YACqB,iBAA+B,EAChD,MAAwB,EACxB,oBAA4B,EAC5B,sBAA8B,EAC9B,mBAAiC,GAAG,EAAE,CAAC,0BAAW,CAAC,GAAG,EAAE;QAJvC,sBAAiB,GAAjB,iBAAiB,CAAc;QAJ5C,sBAAiB,GAAW,CAAC,CAAC;QAUlC,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAErD,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC3E,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC;YACvD,IAAI,CAAC,6BAA6B,GAAG,gBAAgB,EAAE,CAAC;YACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,KAAsB,EAAE,OAAkC,EAAE,EAAE;YACjG,IAAA,qBAAM,EACF,IAAI,CAAC,wBAAwB,KAAK,SAAS,IAAI,IAAI,CAAC,6BAA6B,KAAK,SAAS,EAC/F,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAExD,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC;YAC1E,IAAI,MAAM,IAAI,oBAAoB,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,cAAc,CAAC;oBACvB,SAAS,EAAE,cAAc;oBACzB,MAAM;oBACN,SAAS,EAAE,oBAAoB;oBAC/B,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;oBACjE,UAAU,EAAE,KAAK,KAAK,SAAS;iBAClC,CAAC,CAAC;aACN;YAED,IAAI,IAAI,CAAC,iBAAiB,GAAG,sBAAsB,KAAK,CAAC,EAAE;gBACvD,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAC7B,SAAS,EAAE,QAAQ;oBACnB,MAAM;oBACN,YAAY,EAAE,sBAAsB;oBACpC,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;iBACpE,CAAC,CAAC;aACN;YAED,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,6BAA6B,GAAG,SAAS,CAAC;QACnD,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AApDD,oCAoDC;AAED;;;;;;;;GAQG;AACI,MAAM,gBAAgB,GAAG,CAC5B,iBAA+B,EAC/B,MAAwB,EACxB,uBAA+B,IAAI,EACnC,yBAAiC,IAAI,EACvC,EAAE,CAAC,IAAI,YAAY,CAAC,iBAAiB,EAAE,MAAM,EAAE,oBAAoB,EAAE,sBAAsB,CAAC,CAAC;AALlF,QAAA,gBAAgB,oBAKkE","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport EventEmitter from \"events\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, performance } from \"@fluidframework/common-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\n\nexport class BatchTracker {\n private readonly logger: ITelemetryLogger;\n private startBatchSequenceNumber: number | undefined;\n private trackedBatchCount: number = 0;\n private batchProcessingStartTimeStamp: number | undefined;\n\n constructor(\n private readonly batchEventEmitter: EventEmitter,\n logger: ITelemetryLogger,\n batchLengthThreshold: number,\n batchCountSamplingRate: number,\n dateTimeProvider: () => number = () => performance.now(),\n ) {\n this.logger = ChildLogger.create(logger, \"Batching\");\n\n this.batchEventEmitter.on(\"batchBegin\", (message: ISequencedDocumentMessage) => {\n this.startBatchSequenceNumber = message.sequenceNumber;\n this.batchProcessingStartTimeStamp = dateTimeProvider();\n this.trackedBatchCount++;\n });\n\n this.batchEventEmitter.on(\"batchEnd\", (error: any | undefined, message: ISequencedDocumentMessage) => {\n assert(\n this.startBatchSequenceNumber !== undefined && this.batchProcessingStartTimeStamp !== undefined,\n 0x2ba /* \"batchBegin must fire before batchEnd\" */);\n\n const length = message.sequenceNumber - this.startBatchSequenceNumber + 1;\n if (length >= batchLengthThreshold) {\n this.logger.sendErrorEvent({\n eventName: \"LengthTooBig\",\n length,\n threshold: batchLengthThreshold,\n batchEndSequenceNumber: message.sequenceNumber,\n duration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n batchError: error !== undefined,\n });\n }\n\n if (this.trackedBatchCount % batchCountSamplingRate === 0) {\n this.logger.sendPerformanceEvent({\n eventName: \"Length\",\n length,\n samplingRate: batchCountSamplingRate,\n batchEndSequenceNumber: message.sequenceNumber,\n duration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n });\n }\n\n this.startBatchSequenceNumber = undefined;\n this.batchProcessingStartTimeStamp = undefined;\n });\n }\n}\n\n/**\n * Track batch sizes in terms of op counts and processing times\n *\n * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations\n * @param logger - logger\n * @param batchLengthThreshold - threshold for the length of a batch when to send an error event\n * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics\n * @returns\n */\nexport const BindBatchTracker = (\n batchEventEmitter: EventEmitter,\n logger: ITelemetryLogger,\n batchLengthThreshold: number = 1000,\n batchCountSamplingRate: number = 1000,\n) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);\n"]}
1
+ {"version":3,"file":"batchTracker.js","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAIH,+DAAmE;AAEnE,qEAA8D;AAE9D,MAAa,YAAY;IAMrB,YACqB,iBAA+B,EAChD,MAAwB,EACxB,oBAA4B,EAC5B,sBAA8B,EAC9B,mBAAiC,GAAG,EAAE,CAAC,0BAAW,CAAC,GAAG,EAAE;QAJvC,sBAAiB,GAAjB,iBAAiB,CAAc;QAJ5C,sBAAiB,GAAW,CAAC,CAAC;QAUlC,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAErD,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC3E,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC;YACvD,IAAI,CAAC,6BAA6B,GAAG,gBAAgB,EAAE,CAAC;YACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC7B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,UAAU,EAAE,CAAC,KAAsB,EAAE,OAAkC,EAAE,EAAE;YACjG,IAAA,qBAAM,EACF,IAAI,CAAC,wBAAwB,KAAK,SAAS,IAAI,IAAI,CAAC,6BAA6B,KAAK,SAAS,EAC/F,KAAK,CAAC,4CAA4C,CAAC,CAAC;YAExD,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC;YAC1E,IAAI,MAAM,IAAI,oBAAoB,EAAE;gBAChC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAC7B,SAAS,EAAE,cAAc;oBACzB,MAAM;oBACN,SAAS,EAAE,oBAAoB;oBAC/B,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;oBACjE,UAAU,EAAE,KAAK,KAAK,SAAS;iBAClC,CAAC,CAAC;aACN;YAED,IAAI,IAAI,CAAC,iBAAiB,GAAG,sBAAsB,KAAK,CAAC,EAAE;gBACvD,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAC7B,SAAS,EAAE,QAAQ;oBACnB,MAAM;oBACN,YAAY,EAAE,sBAAsB;oBACpC,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;iBACpE,CAAC,CAAC;aACN;YAED,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,6BAA6B,GAAG,SAAS,CAAC;QACnD,CAAC,CAAC,CAAC;IACP,CAAC;CACJ;AApDD,oCAoDC;AAED;;;;;;;GAOG;AACI,MAAM,gBAAgB,GAAG,CAC5B,iBAA+B,EAC/B,MAAwB,EACxB,uBAA+B,IAAI,EACnC,yBAAiC,IAAI,EACvC,EAAE,CAAC,IAAI,YAAY,CAAC,iBAAiB,EAAE,MAAM,EAAE,oBAAoB,EAAE,sBAAsB,CAAC,CAAC;AALlF,QAAA,gBAAgB,oBAKkE","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport EventEmitter from \"events\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, performance } from \"@fluidframework/common-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\n\nexport class BatchTracker {\n private readonly logger: ITelemetryLogger;\n private startBatchSequenceNumber: number | undefined;\n private trackedBatchCount: number = 0;\n private batchProcessingStartTimeStamp: number | undefined;\n\n constructor(\n private readonly batchEventEmitter: EventEmitter,\n logger: ITelemetryLogger,\n batchLengthThreshold: number,\n batchCountSamplingRate: number,\n dateTimeProvider: () => number = () => performance.now(),\n ) {\n this.logger = ChildLogger.create(logger, \"Batching\");\n\n this.batchEventEmitter.on(\"batchBegin\", (message: ISequencedDocumentMessage) => {\n this.startBatchSequenceNumber = message.sequenceNumber;\n this.batchProcessingStartTimeStamp = dateTimeProvider();\n this.trackedBatchCount++;\n });\n\n this.batchEventEmitter.on(\"batchEnd\", (error: any | undefined, message: ISequencedDocumentMessage) => {\n assert(\n this.startBatchSequenceNumber !== undefined && this.batchProcessingStartTimeStamp !== undefined,\n 0x2ba /* \"batchBegin must fire before batchEnd\" */);\n\n const length = message.sequenceNumber - this.startBatchSequenceNumber + 1;\n if (length >= batchLengthThreshold) {\n this.logger.sendPerformanceEvent({\n eventName: \"LengthTooBig\",\n length,\n threshold: batchLengthThreshold,\n batchEndSequenceNumber: message.sequenceNumber,\n duration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n batchError: error !== undefined,\n });\n }\n\n if (this.trackedBatchCount % batchCountSamplingRate === 0) {\n this.logger.sendPerformanceEvent({\n eventName: \"Length\",\n length,\n samplingRate: batchCountSamplingRate,\n batchEndSequenceNumber: message.sequenceNumber,\n duration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n });\n }\n\n this.startBatchSequenceNumber = undefined;\n this.batchProcessingStartTimeStamp = undefined;\n });\n }\n}\n\n/**\n * Track batch sizes in terms of op counts and processing times\n *\n * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations\n * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLogger}\n * @param batchLengthThreshold - threshold for the length of a batch when to send an error event\n * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics\n */\nexport const BindBatchTracker = (\n batchEventEmitter: EventEmitter,\n logger: ITelemetryLogger,\n batchLengthThreshold: number = 1000,\n batchCountSamplingRate: number = 1000,\n) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);\n"]}
@@ -4,9 +4,9 @@
4
4
  */
5
5
  import { IFluidHandle, IFluidHandleContext } from "@fluidframework/core-interfaces";
6
6
  import { IDocumentStorageService } from "@fluidframework/driver-definitions";
7
- import { ISnapshotTree } from "@fluidframework/protocol-definitions";
8
- import { ITelemetryLogger } from "@fluidframework/common-definitions";
9
- import { IContainerRuntime } from "@fluidframework/container-runtime-definitions";
7
+ import { ISequencedDocumentMessage, ISnapshotTree } from "@fluidframework/protocol-definitions";
8
+ import { TypedEventEmitter } from "@fluidframework/common-utils";
9
+ import { IContainerRuntime, IContainerRuntimeEvents } from "@fluidframework/container-runtime-definitions";
10
10
  import { IGarbageCollectionData, ISummaryTreeWithStats, ITelemetryContext } from "@fluidframework/runtime-definitions";
11
11
  /**
12
12
  * This class represents blob (long string)
@@ -34,47 +34,108 @@ export interface IBlobManagerLoadInfo {
34
34
  ids?: string[];
35
35
  redirectTable?: [string, string][];
36
36
  }
37
+ export declare type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "connected" | "logger"> & TypedEventEmitter<IContainerRuntimeEvents>;
38
+ export interface IPendingBlobs {
39
+ [id: string]: {
40
+ blob: string;
41
+ };
42
+ }
37
43
  export declare class BlobManager {
38
44
  private readonly routeContext;
39
45
  private readonly getStorage;
40
- private readonly attachBlobCallback;
46
+ /**
47
+ * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
48
+ * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
49
+ * will then not delete the blob as long as it is listed as referenced in future summaries.
50
+ * The summarizing client will know to include the storage ID in the summary when it sees the op.
51
+ *
52
+ * The op may also include a local ID to inform all clients of the relation to the storage
53
+ * ID, without knowledge of which they cannot request the blob from storage. This is also
54
+ * included in the redirect table in the summary.
55
+ */
56
+ private readonly sendBlobAttachOp;
41
57
  private readonly gcNodeUpdated;
42
58
  private readonly runtime;
43
- private readonly logger;
44
59
  static readonly basePath = "_blobs";
45
60
  private static readonly redirectTableBlobName;
46
- private readonly blobIds;
47
- private readonly pendingBlobIds;
48
- private readonly detachedBlobIds;
49
- private redirectTable;
50
- constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService, attachBlobCallback: (blobId: string) => void, gcNodeUpdated: (blobPath: string) => void, runtime: IContainerRuntime, logger: ITelemetryLogger);
51
- private hasBlob;
61
+ private readonly logger;
62
+ /**
63
+ * Map of local (offline/detached) IDs to storage IDs. Contains identity entries
64
+ * (id → id) for storage IDs, so all requested IDs should be a key in this map.
65
+ * Blobs created while the container is detached are stored in IDetachedBlobStorage
66
+ * which gives local IDs; the storage IDs are filled in at attach time.
67
+ */
68
+ private readonly redirectTable;
69
+ /**
70
+ * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
71
+ * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
72
+ * we must save it. This is true for both the online and offline flow.
73
+ */
74
+ private readonly pendingBlobs;
75
+ /**
76
+ * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
77
+ * don't include local ID in online flow.
78
+ */
79
+ private readonly opsInFlight;
80
+ private readonly retryThrottler;
81
+ constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService,
82
+ /**
83
+ * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
84
+ * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
85
+ * will then not delete the blob as long as it is listed as referenced in future summaries.
86
+ * The summarizing client will know to include the storage ID in the summary when it sees the op.
87
+ *
88
+ * The op may also include a local ID to inform all clients of the relation to the storage
89
+ * ID, without knowledge of which they cannot request the blob from storage. This is also
90
+ * included in the redirect table in the summary.
91
+ */
92
+ sendBlobAttachOp: (storageId?: string, localId?: string) => void, gcNodeUpdated: (blobPath: string) => void, runtime: IBlobManagerRuntime, stashedBlobs?: IPendingBlobs);
93
+ private get pendingOfflineUploads();
94
+ get hasPendingOfflineUploads(): boolean;
95
+ /**
96
+ * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
97
+ */
98
+ onConnected(): Promise<void>;
99
+ /**
100
+ * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
101
+ * again
102
+ */
103
+ private onDisconnected;
52
104
  /**
53
105
  * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
54
106
  * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
55
107
  * referenced by storing these handles in a referenced DDS.
56
108
  */
57
109
  private getBlobGCNodePath;
58
- getBlob(blobId: string): Promise<IFluidHandle<ArrayBufferLike>>;
110
+ /**
111
+ * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
112
+ * detached or there are no (non-pending) attachment blobs in the document
113
+ */
114
+ private get storageIds();
115
+ getBlob(blobId: string): Promise<ArrayBufferLike>;
116
+ private getBlobHandle;
117
+ private createBlobDetached;
59
118
  createBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>>;
60
- processBlobAttachOp(blobId: string, local: boolean): void;
119
+ private uploadBlob;
120
+ private onUploadResolve;
121
+ private onUploadReject;
122
+ private transitionToOffline;
123
+ /**
124
+ * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
125
+ * submitted to runtime while disconnected.
126
+ * @param metadata - op metadata containing storage and/or local IDs
127
+ */
128
+ reSubmit(metadata: Record<string, unknown> | undefined): void;
129
+ processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean): void;
61
130
  /**
62
131
  * Reads blobs needed to load BlobManager from storage.
132
+ * @param blobsTree - Tree containing IDs of previously attached blobs. We
133
+ * look for the IDs in the blob entries of the tree since the both the r11s
134
+ * and SPO drivers replace the attachment types returned in snapshot() with blobs.
63
135
  */
64
136
  static load(blobsTree: ISnapshotTree | undefined, tryFetchBlob: (id: string) => Promise<[string, string][]>): Promise<IBlobManagerLoadInfo>;
65
137
  /**
66
- * Load a set of previously attached blob IDs from a previous snapshot. Note
67
- * that BlobManager tracking and reporting attached blobs is a temporary
68
- * solution since storage expects attached blobs to be reported and any that
69
- * are not reported as attached may be GCed. In the future attached blob
70
- * IDs will be collected at summarization time, and runtime will not care
71
- * about the existence or specific formatting of this tree in returned
72
- * snapshots.
73
- *
74
- * @param blobsTree - Tree containing IDs of previously attached blobs. This
75
- * corresponds to snapshot() below. We look for the IDs in the blob entries
76
- * of the tree since the both the r11s and SPO drivers replace the
77
- * attachment types returned in snapshot() with blobs.
138
+ * Load a set of previously attached blob IDs and redirect table from a previous snapshot.
78
139
  */
79
140
  private load;
80
141
  /**
@@ -91,5 +152,6 @@ export declare class BlobManager {
91
152
  deleteUnusedRoutes(unusedRoutes: string[]): void;
92
153
  summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
93
154
  setRedirectTable(table: Map<string, string>): void;
155
+ getPendingBlobs(): IPendingBlobs;
94
156
  }
95
157
  //# sourceMappingURL=blobManager.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAErE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAEtE,OAAO,EAAE,iBAAiB,EAAE,MAAM,+CAA+C,CAAC;AAGlF,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAE7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAED;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAED,qBAAa,WAAW;IAchB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAGnC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,MAAM;IArB3B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IAEjE,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA0B;IAGlD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA0C;IAEzE,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;IAE1D,OAAO,CAAC,aAAa,CAAkC;gBAGlC,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB,EACzC,kBAAkB,EAAE,CAAC,MAAM,EAAE,MAAM,KAAK,IAAI,EAG5C,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,iBAAiB,EAC1B,MAAM,EAAE,gBAAgB;IAU7C,OAAO,CAAC,OAAO;IAIf;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIZ,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IAqB/D,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;IAgD/E,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,OAAO;IAUzD;;OAEG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,IAAI;IAeZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAqBhD,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAoBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;CAWrD"}
1
+ {"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAAuB,yBAAyB,EAAE,aAAa,EAAE,MAAM,sCAAsC,CAAC;AAGrH,OAAO,EAAoD,iBAAiB,EAAE,MAAM,8BAA8B,CAAC;AACnH,OAAO,EAAE,iBAAiB,EAAE,uBAAuB,EAAE,MAAM,+CAA+C,CAAC;AAG3G,OAAO,EACH,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACpB,MAAM,qCAAqC,CAAC;AAG7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAYxC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAblC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAAiB;IAExD,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGjB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAK3B,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGnC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACjC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACtC;AAID,oBAAY,mBAAmB,GAC3B,IAAI,CAAC,iBAAiB,EAAE,aAAa,GAAG,WAAW,GAAG,QAAQ,CAAC,GAAG,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAmBjH,MAAM,WAAW,aAAa;IAAG,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAC;KAAE,CAAC;CAAE;AAEnE,qBAAa,WAAW;IAkChB,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAlD5B,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAE1C;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAK5B;gBAGkB,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,OAAO,CAAC,EAAE,MAAM,KAAK,IAAI,EAGhE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EACzC,OAAO,EAAE,mBAAmB,EAC7C,YAAY,GAAE,aAAkB;IAkBpC,OAAO,KAAK,qBAAqB,GAGhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED;;OAEG;IACU,WAAW;IAWxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAYrB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IA8B9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAQnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA0BxE,UAAU;IAYxB,OAAO,CAAC,eAAe;YAoCT,cAAc;IAgB5B,OAAO,CAAC,mBAAmB;IAoB3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IActD,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAmC7E;;;;;OAKG;WACiB,IAAI,CACpB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GAC1D,OAAO,CAAC,oBAAoB,CAAC;IAchC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAyBjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAoBhD,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAqBtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAa3C,eAAe,IAAI,aAAa;CAO1C"}