@fluidframework/container-runtime 1.2.2 → 2.0.0-internal.1.0.0.82159

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/dist/blobManager.d.ts +81 -25
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +301 -100
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +65 -11
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +101 -82
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStore.d.ts +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +32 -26
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +3 -4
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +16 -23
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStores.d.ts +5 -2
  18. package/dist/dataStores.d.ts.map +1 -1
  19. package/dist/dataStores.js +11 -3
  20. package/dist/dataStores.js.map +1 -1
  21. package/dist/garbageCollection.d.ts.map +1 -1
  22. package/dist/garbageCollection.js +17 -12
  23. package/dist/garbageCollection.js.map +1 -1
  24. package/dist/opProperties.d.ts +7 -0
  25. package/dist/opProperties.d.ts.map +1 -0
  26. package/dist/opProperties.js +20 -0
  27. package/dist/opProperties.js.map +1 -0
  28. package/dist/packageVersion.d.ts +1 -1
  29. package/dist/packageVersion.d.ts.map +1 -1
  30. package/dist/packageVersion.js +1 -1
  31. package/dist/packageVersion.js.map +1 -1
  32. package/dist/runningSummarizer.d.ts +14 -4
  33. package/dist/runningSummarizer.d.ts.map +1 -1
  34. package/dist/runningSummarizer.js +68 -26
  35. package/dist/runningSummarizer.js.map +1 -1
  36. package/dist/summarizer.d.ts +0 -2
  37. package/dist/summarizer.d.ts.map +1 -1
  38. package/dist/summarizer.js +1 -12
  39. package/dist/summarizer.js.map +1 -1
  40. package/dist/summarizerHeuristics.d.ts +26 -4
  41. package/dist/summarizerHeuristics.d.ts.map +1 -1
  42. package/dist/summarizerHeuristics.js +95 -18
  43. package/dist/summarizerHeuristics.js.map +1 -1
  44. package/dist/summarizerTypes.d.ts +30 -10
  45. package/dist/summarizerTypes.d.ts.map +1 -1
  46. package/dist/summarizerTypes.js.map +1 -1
  47. package/dist/summaryCollection.js +1 -1
  48. package/dist/summaryCollection.js.map +1 -1
  49. package/dist/summaryFormat.d.ts +0 -5
  50. package/dist/summaryFormat.d.ts.map +1 -1
  51. package/dist/summaryFormat.js.map +1 -1
  52. package/dist/summaryGenerator.d.ts +1 -0
  53. package/dist/summaryGenerator.d.ts.map +1 -1
  54. package/dist/summaryGenerator.js +11 -9
  55. package/dist/summaryGenerator.js.map +1 -1
  56. package/lib/blobManager.d.ts +81 -25
  57. package/lib/blobManager.d.ts.map +1 -1
  58. package/lib/blobManager.js +302 -101
  59. package/lib/blobManager.js.map +1 -1
  60. package/lib/containerRuntime.d.ts +65 -11
  61. package/lib/containerRuntime.d.ts.map +1 -1
  62. package/lib/containerRuntime.js +103 -84
  63. package/lib/containerRuntime.js.map +1 -1
  64. package/lib/dataStore.d.ts +1 -1
  65. package/lib/dataStore.d.ts.map +1 -1
  66. package/lib/dataStore.js +32 -26
  67. package/lib/dataStore.js.map +1 -1
  68. package/lib/dataStoreContext.d.ts +3 -4
  69. package/lib/dataStoreContext.d.ts.map +1 -1
  70. package/lib/dataStoreContext.js +17 -24
  71. package/lib/dataStoreContext.js.map +1 -1
  72. package/lib/dataStores.d.ts +5 -2
  73. package/lib/dataStores.d.ts.map +1 -1
  74. package/lib/dataStores.js +11 -3
  75. package/lib/dataStores.js.map +1 -1
  76. package/lib/garbageCollection.d.ts.map +1 -1
  77. package/lib/garbageCollection.js +17 -12
  78. package/lib/garbageCollection.js.map +1 -1
  79. package/lib/opProperties.d.ts +7 -0
  80. package/lib/opProperties.d.ts.map +1 -0
  81. package/lib/opProperties.js +16 -0
  82. package/lib/opProperties.js.map +1 -0
  83. package/lib/packageVersion.d.ts +1 -1
  84. package/lib/packageVersion.d.ts.map +1 -1
  85. package/lib/packageVersion.js +1 -1
  86. package/lib/packageVersion.js.map +1 -1
  87. package/lib/runningSummarizer.d.ts +14 -4
  88. package/lib/runningSummarizer.d.ts.map +1 -1
  89. package/lib/runningSummarizer.js +68 -26
  90. package/lib/runningSummarizer.js.map +1 -1
  91. package/lib/summarizer.d.ts +0 -2
  92. package/lib/summarizer.d.ts.map +1 -1
  93. package/lib/summarizer.js +1 -12
  94. package/lib/summarizer.js.map +1 -1
  95. package/lib/summarizerHeuristics.d.ts +26 -4
  96. package/lib/summarizerHeuristics.d.ts.map +1 -1
  97. package/lib/summarizerHeuristics.js +95 -18
  98. package/lib/summarizerHeuristics.js.map +1 -1
  99. package/lib/summarizerTypes.d.ts +30 -10
  100. package/lib/summarizerTypes.d.ts.map +1 -1
  101. package/lib/summarizerTypes.js.map +1 -1
  102. package/lib/summaryCollection.js +1 -1
  103. package/lib/summaryCollection.js.map +1 -1
  104. package/lib/summaryFormat.d.ts +0 -5
  105. package/lib/summaryFormat.d.ts.map +1 -1
  106. package/lib/summaryFormat.js.map +1 -1
  107. package/lib/summaryGenerator.d.ts +1 -0
  108. package/lib/summaryGenerator.d.ts.map +1 -1
  109. package/lib/summaryGenerator.js +11 -9
  110. package/lib/summaryGenerator.js.map +1 -1
  111. package/package.json +45 -20
  112. package/src/blobManager.ts +360 -119
  113. package/src/containerRuntime.ts +203 -103
  114. package/src/dataStore.ts +53 -38
  115. package/src/dataStoreContext.ts +16 -23
  116. package/src/dataStores.ts +14 -3
  117. package/src/garbageCollection.ts +13 -7
  118. package/src/opProperties.ts +19 -0
  119. package/src/packageVersion.ts +1 -1
  120. package/src/runningSummarizer.ts +75 -22
  121. package/src/summarizer.ts +1 -18
  122. package/src/summarizerHeuristics.ts +133 -19
  123. package/src/summarizerTypes.ts +37 -10
  124. package/src/summaryCollection.ts +1 -1
  125. package/src/summaryFormat.ts +0 -6
  126. package/src/summaryGenerator.ts +40 -22
  127. package/dist/opTelemetry.d.ts +0 -22
  128. package/dist/opTelemetry.d.ts.map +0 -1
  129. package/dist/opTelemetry.js +0 -59
  130. package/dist/opTelemetry.js.map +0 -1
  131. package/lib/opTelemetry.d.ts +0 -22
  132. package/lib/opTelemetry.d.ts.map +0 -1
  133. package/lib/opTelemetry.js +0 -55
  134. package/lib/opTelemetry.js.map +0 -1
  135. package/src/opTelemetry.ts +0 -71
@@ -5,10 +5,12 @@
5
5
  */
6
6
  Object.defineProperty(exports, "__esModule", { value: true });
7
7
  exports.BlobManager = exports.BlobHandle = void 0;
8
+ const uuid_1 = require("uuid");
8
9
  const runtime_utils_1 = require("@fluidframework/runtime-utils");
9
10
  const common_utils_1 = require("@fluidframework/common-utils");
10
11
  const container_definitions_1 = require("@fluidframework/container-definitions");
11
12
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
13
+ const throttler_1 = require("./throttler");
12
14
  /**
13
15
  * This class represents blob (long string)
14
16
  * This object is used only when creating (writing) new blob and serialization purposes.
@@ -36,33 +38,101 @@ class BlobHandle {
36
38
  }
37
39
  }
38
40
  exports.BlobHandle = BlobHandle;
41
+ class CancellableThrottler {
42
+ constructor(throttler) {
43
+ this.throttler = throttler;
44
+ this.cancelP = new common_utils_1.Deferred();
45
+ }
46
+ async getDelay() {
47
+ return Promise.race([
48
+ this.cancelP.promise,
49
+ new Promise((resolve) => setTimeout(resolve, this.throttler.getDelay())),
50
+ ]);
51
+ }
52
+ cancel() {
53
+ this.cancelP.resolve();
54
+ this.cancelP = new common_utils_1.Deferred();
55
+ }
56
+ }
57
+ // Note that while offline we "submit" an op before uploading the blob, but we always
58
+ // expect blobs to be uploaded before we actually see the op round-trip
59
+ var PendingBlobStatus;
60
+ (function (PendingBlobStatus) {
61
+ PendingBlobStatus[PendingBlobStatus["OnlinePendingUpload"] = 0] = "OnlinePendingUpload";
62
+ PendingBlobStatus[PendingBlobStatus["OnlinePendingOp"] = 1] = "OnlinePendingOp";
63
+ PendingBlobStatus[PendingBlobStatus["OfflinePendingUpload"] = 2] = "OfflinePendingUpload";
64
+ PendingBlobStatus[PendingBlobStatus["OfflinePendingOp"] = 3] = "OfflinePendingOp";
65
+ })(PendingBlobStatus || (PendingBlobStatus = {}));
39
66
  class BlobManager {
40
- constructor(routeContext, snapshot, getStorage, attachBlobCallback,
67
+ constructor(routeContext, snapshot, getStorage,
68
+ /**
69
+ * Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
70
+ * the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
71
+ * will then not delete the blob as long as it is listed as referenced in future summaries.
72
+ * The summarizing client will know to include the storage ID in the summary when it sees the op.
73
+ *
74
+ * The op may also include a local ID to inform all clients of the relation to the storage
75
+ * ID, without knowledge of which they cannot request the blob from storage. This is also
76
+ * included in the redirect table in the summary.
77
+ */
78
+ sendBlobAttachOp,
41
79
  // To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's
42
80
  // of the format `/<BlobManager.basePath>/<blobId>`.
43
- gcNodeUpdated, runtime, logger) {
81
+ gcNodeUpdated, runtime) {
44
82
  this.routeContext = routeContext;
45
83
  this.getStorage = getStorage;
46
- this.attachBlobCallback = attachBlobCallback;
84
+ this.sendBlobAttachOp = sendBlobAttachOp;
47
85
  this.gcNodeUpdated = gcNodeUpdated;
48
86
  this.runtime = runtime;
49
- this.logger = logger;
50
- // uploaded blob IDs
51
- this.blobIds = new Set();
52
- // blobs for which upload is pending. maps to a promise that will resolve once the blob has been uploaded and a
53
- // BlobAttach op has round-tripped.
54
- this.pendingBlobIds = new Map();
55
- // blobs uploaded while detached; cleared upon attach
56
- this.detachedBlobIds = new Set();
57
- this.runtime.once("dispose", () => {
58
- for (const promise of this.pendingBlobIds.values()) {
59
- promise.reject(new Error("runtime disposed while blobAttach op in flight"));
60
- }
61
- });
62
- this.load(snapshot);
87
+ /**
88
+ * Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
89
+ * Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
90
+ * we must save it. This is true for both the online and offline flow.
91
+ */
92
+ this.pendingBlobs = new Map();
93
+ /**
94
+ * Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
95
+ * don't include local ID in online flow.
96
+ */
97
+ this.opsInFlight = new Map();
98
+ this.retryThrottler = new CancellableThrottler(new throttler_1.Throttler(60 * 1000, // 60 sec delay window
99
+ 30 * 1000, // 30 sec max delay
100
+ // throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
101
+ (0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })));
102
+ this.logger = telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager");
103
+ this.runtime.on("disconnected", () => this.onDisconnected());
104
+ this.redirectTable = this.load(snapshot);
63
105
  }
64
- hasBlob(id) {
65
- return this.blobIds.has(id) || this.detachedBlobIds.has(id);
106
+ get pendingOfflineUploads() {
107
+ return Array.from(this.pendingBlobs.values())
108
+ .filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
109
+ }
110
+ get hasPendingOfflineUploads() {
111
+ return this.pendingOfflineUploads.length > 0;
112
+ }
113
+ /**
114
+ * Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
115
+ */
116
+ async onConnected() {
117
+ this.retryThrottler.cancel();
118
+ const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);
119
+ await telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, {
120
+ eventName: "BlobUploadOnConnected",
121
+ count: pendingUploads.length,
122
+ }, async () => Promise.all(pendingUploads), { start: true, end: true });
123
+ }
124
+ /**
125
+ * Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
126
+ * again
127
+ */
128
+ onDisconnected() {
129
+ for (const [localId, entry] of this.pendingBlobs) {
130
+ if (entry.status === PendingBlobStatus.OnlinePendingOp) {
131
+ // This will submit another BlobAttach op for this blob. This is necessary because the one we sent
132
+ // already didn't have the local ID.
133
+ this.transitionToOffline(localId);
134
+ }
135
+ }
66
136
  }
67
137
  /**
68
138
  * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
@@ -72,61 +142,200 @@ class BlobManager {
72
142
  getBlobGCNodePath(blobId) {
73
143
  return `/${BlobManager.basePath}/${blobId}`;
74
144
  }
145
+ /**
146
+ * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
147
+ * detached or there are no (non-pending) attachment blobs in the document
148
+ */
149
+ get storageIds() {
150
+ const ids = new Set(this.redirectTable.values());
151
+ // If we are detached, we will not have storage IDs, only undefined
152
+ const undefinedValueInTable = ids.delete(undefined);
153
+ // For a detached container, entries are inserted into the redirect table with an undefined storage ID.
154
+ // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
155
+ (0, common_utils_1.assert)(!undefinedValueInTable || this.runtime.attachState === container_definitions_1.AttachState.Detached && ids.size === 0, "'redirectTable' must contain only undefined while detached / defined values while attached");
156
+ return ids;
157
+ }
75
158
  async getBlob(blobId) {
76
- var _a, _b;
77
- const storageId = (_b = (_a = this.redirectTable) === null || _a === void 0 ? void 0 : _a.get(blobId)) !== null && _b !== void 0 ? _b : blobId;
78
- (0, common_utils_1.assert)(this.hasBlob(storageId), 0x11f /* "requesting unknown blobs" */);
79
- // When this blob is retrieved, let the container runtime know that the corresponding GC node got updated.
159
+ const pending = this.pendingBlobs.get(blobId);
160
+ if (pending) {
161
+ return pending.blob;
162
+ }
163
+ let storageId;
164
+ if (this.runtime.attachState === container_definitions_1.AttachState.Detached) {
165
+ (0, common_utils_1.assert)(this.redirectTable.has(blobId), "requesting unknown blobs");
166
+ // Blobs created while the container is detached are stored in IDetachedBlobStorage.
167
+ // The 'IDocumentStorageService.readBlob()' call below will retrieve these via localId.
168
+ storageId = blobId;
169
+ }
170
+ else {
171
+ storageId = this.redirectTable.get(blobId);
172
+ (0, common_utils_1.assert)(!!storageId, 0x11f /* "requesting unknown blobs" */);
173
+ }
174
+ // When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
80
175
  this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
81
- return new BlobHandle(`${BlobManager.basePath}/${storageId}`, this.routeContext, async () => telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
176
+ return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
82
177
  return this.getStorage().readBlob(storageId);
83
- }, { end: true, cancel: "error" }));
178
+ }, { end: true, cancel: "error" });
179
+ }
180
+ getBlobHandle(id) {
181
+ (0, common_utils_1.assert)(this.redirectTable.has(id) || this.pendingBlobs.has(id), "requesting handle for unknown blob");
182
+ return new BlobHandle(`${BlobManager.basePath}/${id}`, this.routeContext, async () => this.getBlob(id));
183
+ }
184
+ async createBlobDetached(blob) {
185
+ // Blobs created while the container is detached are stored in IDetachedBlobStorage.
186
+ // The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
187
+ const response = await this.getStorage().createBlob(blob);
188
+ this.redirectTable.set(response.id, undefined);
189
+ return this.getBlobHandle(response.id);
84
190
  }
85
191
  async createBlob(blob) {
86
- var _a, _b;
192
+ if (this.runtime.attachState === container_definitions_1.AttachState.Detached) {
193
+ return this.createBlobDetached(blob);
194
+ }
87
195
  if (this.runtime.attachState === container_definitions_1.AttachState.Attaching) {
88
196
  // blob upload is not supported in "Attaching" state
89
197
  this.logger.sendTelemetryEvent({ eventName: "CreateBlobWhileAttaching" });
90
198
  await new Promise((resolve) => this.runtime.once("attached", resolve));
91
199
  }
92
- if (!this.runtime.connected && this.runtime.attachState === container_definitions_1.AttachState.Attached) {
93
- // see https://github.com/microsoft/FluidFramework/issues/8246
94
- // Avoid getting storage if we are offline since it might be undefined. In the future we will return
95
- // handles immediately while offline
96
- await new Promise((resolve) => this.runtime.once("connected", resolve));
97
- }
98
- const response = await telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: "error" });
99
- const handle = new BlobHandle(`${BlobManager.basePath}/${response.id}`, this.routeContext,
100
- // get() should go through BlobManager.getBlob() so handles created while detached can be redirected
101
- // to the correct storage id after they are uploaded
102
- async () => this.getBlob(response.id).then(async (h) => h.get()));
103
- if (this.runtime.attachState === container_definitions_1.AttachState.Detached) {
104
- this.detachedBlobIds.add(response.id);
105
- return handle;
200
+ (0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Attached, "For clarity and paranoid defense against adding future attachment states");
201
+ // Create a local ID for each blob. This is used to support blobs if/when the client goes
202
+ // offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.
203
+ const localId = (0, uuid_1.v4)();
204
+ const pendingEntry = {
205
+ blob,
206
+ status: PendingBlobStatus.OnlinePendingUpload,
207
+ handleP: new common_utils_1.Deferred(),
208
+ uploadP: this.uploadBlob(localId, blob),
209
+ };
210
+ this.pendingBlobs.set(localId, pendingEntry);
211
+ return pendingEntry.handleP.promise;
212
+ }
213
+ async uploadBlob(localId, blob) {
214
+ return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
215
+ }
216
+ onUploadResolve(localId, response) {
217
+ var _a;
218
+ const entry = this.pendingBlobs.get(localId);
219
+ (0, common_utils_1.assert)((entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OnlinePendingUpload ||
220
+ (entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OfflinePendingUpload, "Must have pending blob entry for uploaded blob");
221
+ entry.storageId = response.id;
222
+ if (this.runtime.connected) {
223
+ if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
224
+ if (this.storageIds.has(response.id)) {
225
+ // Storage may dedupe blobs and give us an ID we already know
226
+ // no need to submit BlobAttach op in this case
227
+ entry.handleP.resolve(this.getBlobHandle(response.id));
228
+ this.pendingBlobs.delete(localId);
229
+ }
230
+ else {
231
+ // Check for still-pending duplicates too; if an op is already in flight we can wait for that one
232
+ if (!this.opsInFlight.has(response.id)) {
233
+ this.sendBlobAttachOp(response.id);
234
+ }
235
+ this.opsInFlight.set(response.id, ((_a = this.opsInFlight.get(response.id)) !== null && _a !== void 0 ? _a : []).concat(localId));
236
+ entry.status = PendingBlobStatus.OnlinePendingOp;
237
+ }
238
+ }
239
+ else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {
240
+ // We already submitted a BlobAttach op for this blob when it was transitioned to offline flow
241
+ entry.status = PendingBlobStatus.OfflinePendingOp;
242
+ }
106
243
  }
107
- // Note - server will de-dup blobs, so we might get existing blobId!
108
- if (this.pendingBlobIds.has(response.id)) {
109
- await ((_a = this.pendingBlobIds.get(response.id)) === null || _a === void 0 ? void 0 : _a.promise);
244
+ else {
245
+ // connected to storage but not ordering service?
246
+ this.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
247
+ if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
248
+ this.transitionToOffline(localId);
249
+ }
250
+ entry.status = PendingBlobStatus.OfflinePendingOp;
110
251
  }
111
- else if (!this.blobIds.has(response.id)) {
112
- this.pendingBlobIds.set(response.id, new common_utils_1.Deferred());
113
- // send blob attach op and wait until we see it to return the handle
114
- this.attachBlobCallback(response.id);
115
- await ((_b = this.pendingBlobIds.get(response.id)) === null || _b === void 0 ? void 0 : _b.promise);
252
+ return response;
253
+ }
254
+ async onUploadReject(localId, error) {
255
+ const entry = this.pendingBlobs.get(localId);
256
+ (0, common_utils_1.assert)(!!entry, "Must have pending blob entry for blob which failed to upload");
257
+ if (!this.runtime.connected) {
258
+ if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
259
+ this.transitionToOffline(localId);
260
+ }
261
+ // we are probably not connected to storage but start another upload request in case we are
262
+ entry.uploadP = this.retryThrottler.getDelay().then(async () => this.uploadBlob(localId, entry.blob));
263
+ return entry.uploadP;
264
+ }
265
+ else {
266
+ entry.handleP.reject(error);
267
+ throw error;
268
+ }
269
+ }
270
+ transitionToOffline(localId) {
271
+ (0, common_utils_1.assert)(!this.runtime.connected, "Must only transition to offline flow while runtime is disconnected");
272
+ const entry = this.pendingBlobs.get(localId);
273
+ (0, common_utils_1.assert)(!!entry, "No pending blob entry");
274
+ (0, common_utils_1.assert)([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status), "Blob must be in online flow to transition to offline flow");
275
+ entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload
276
+ ? PendingBlobStatus.OfflinePendingUpload
277
+ : PendingBlobStatus.OfflinePendingOp;
278
+ // Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the
279
+ // BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be
280
+ // added to the document if the ops are not all successfully submitted upon reconnection.
281
+ // storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
282
+ // is called
283
+ this.sendBlobAttachOp(entry.storageId, localId);
284
+ entry.handleP.resolve(this.getBlobHandle(localId));
285
+ }
286
+ /**
287
+ * Resubmit a BlobAttach op. Used to add storage IDs to ops that were
288
+ * submitted to runtime while disconnected.
289
+ * @param metadata - op metadata containing storage and/or local IDs
290
+ */
291
+ reSubmit(metadata) {
292
+ (0, common_utils_1.assert)(!!metadata, "Resubmitted ops must have metadata");
293
+ const { blobId, localId } = metadata;
294
+ if (!blobId) {
295
+ (0, common_utils_1.assert)(!!localId, "Submitted BlobAttach ops must have a blobId or localId");
296
+ // We submitted this op while offline. The blob should have been uploaded by now.
297
+ const pendingEntry = this.pendingBlobs.get(localId);
298
+ (0, common_utils_1.assert)((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
299
+ !!(pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.storageId), "blob must be uploaded before resubmitting BlobAttach op");
300
+ return this.sendBlobAttachOp(pendingEntry.storageId, localId);
116
301
  }
117
- return handle;
302
+ return this.sendBlobAttachOp(blobId, localId);
118
303
  }
119
- processBlobAttachOp(blobId, local) {
304
+ processBlobAttachOp(message, local) {
305
+ var _a, _b;
306
+ (0, common_utils_1.assert)((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
307
+ if (message.metadata.localId !== undefined) {
308
+ this.redirectTable.set(message.metadata.localId, message.metadata.blobId);
309
+ }
310
+ // set identity (id -> id) entry
311
+ this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);
120
312
  if (local) {
121
- const pendingBlobP = this.pendingBlobIds.get(blobId);
122
- (0, common_utils_1.assert)(pendingBlobP !== undefined, 0x1f8 /* "local BlobAttach op with no pending blob" */);
123
- pendingBlobP.resolve();
124
- this.pendingBlobIds.delete(blobId);
313
+ if (message.metadata.localId === undefined) {
314
+ // Since there is no local ID, we know this op was submitted while online.
315
+ const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);
316
+ (0, common_utils_1.assert)(!!waitingBlobs, "local online BlobAttach op with no pending blob");
317
+ waitingBlobs.forEach((localId) => {
318
+ const pendingBlobEntry = this.pendingBlobs.get(localId);
319
+ (0, common_utils_1.assert)(pendingBlobEntry !== undefined, "local online BlobAttach op with no pending blob entry");
320
+ // It's possible we transitioned to offline flow while waiting for this op.
321
+ if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
322
+ pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));
323
+ this.pendingBlobs.delete(localId);
324
+ }
325
+ });
326
+ }
327
+ else {
328
+ // Each local ID is unique; get the pending blob entry and delete it
329
+ (0, common_utils_1.assert)(((_b = this.pendingBlobs.get(message.metadata.localId)) === null || _b === void 0 ? void 0 : _b.status) === PendingBlobStatus.OfflinePendingOp, 0x1f8 /* "local BlobAttach op with no pending blob" */);
330
+ this.pendingBlobs.delete(message.metadata.localId);
331
+ }
125
332
  }
126
- this.blobIds.add(blobId);
127
333
  }
128
334
  /**
129
335
  * Reads blobs needed to load BlobManager from storage.
336
+ * @param blobsTree - Tree containing IDs of previously attached blobs. We
337
+ * look for the IDs in the blob entries of the tree since the both the r11s
338
+ * and SPO drivers replace the attachment types returned in snapshot() with blobs.
130
339
  */
131
340
  static async load(blobsTree, tryFetchBlob) {
132
341
  if (!blobsTree) {
@@ -142,33 +351,23 @@ class BlobManager {
142
351
  return { ids, redirectTable };
143
352
  }
144
353
  /**
145
- * Load a set of previously attached blob IDs from a previous snapshot. Note
146
- * that BlobManager tracking and reporting attached blobs is a temporary
147
- * solution since storage expects attached blobs to be reported and any that
148
- * are not reported as attached may be GCed. In the future attached blob
149
- * IDs will be collected at summarization time, and runtime will not care
150
- * about the existence or specific formatting of this tree in returned
151
- * snapshots.
152
- *
153
- * @param blobsTree - Tree containing IDs of previously attached blobs. This
154
- * corresponds to snapshot() below. We look for the IDs in the blob entries
155
- * of the tree since the both the r11s and SPO drivers replace the
156
- * attachment types returned in snapshot() with blobs.
354
+ * Load a set of previously attached blob IDs and redirect table from a previous snapshot.
157
355
  */
158
356
  load(snapshot) {
159
357
  var _a, _b, _c;
160
- if (snapshot.ids) {
161
- const detached = this.runtime.attachState === container_definitions_1.AttachState.Detached;
162
- snapshot.ids.map((entry) => detached ? this.detachedBlobIds.add(entry) : this.blobIds.add(entry));
163
- }
164
- if (snapshot.redirectTable) {
165
- this.redirectTable = new Map(snapshot.redirectTable);
166
- }
167
358
  this.logger.sendTelemetryEvent({
168
359
  eventName: "AttachmentBlobsLoaded",
169
360
  count: (_b = (_a = snapshot.ids) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0,
170
361
  redirectTable: (_c = snapshot.redirectTable) === null || _c === void 0 ? void 0 : _c.length,
171
362
  });
363
+ const table = new Map(snapshot.redirectTable);
364
+ if (snapshot.ids) {
365
+ const detached = this.runtime.attachState === container_definitions_1.AttachState.Detached;
366
+ // If we are detached, we don't have storage IDs yet, so set to undefined
367
+ // Otherwise, set identity (id -> id) entries
368
+ snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));
369
+ }
370
+ return table;
172
371
  }
173
372
  /**
174
373
  * Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
@@ -178,22 +377,23 @@ class BlobManager {
178
377
  */
179
378
  getGCData(fullGC = false) {
180
379
  const gcData = { gcNodes: {} };
181
- this.blobIds.forEach((blobId) => {
380
+ /**
381
+ * The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
382
+ * by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
383
+ */
384
+ this.storageIds.forEach((blobId) => {
182
385
  gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];
183
386
  });
184
- /**
185
- * For all blobs in the redirect table, the handle returned on creation is based off of the localId. So, these
186
- * nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
187
- * must also be marked referenced. So, we add a route from the localId node to the storageId node.
188
- * Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
189
- * a blob may be referenced via its storageId handle.
190
- */
191
- if (this.redirectTable !== undefined) {
192
- for (const [localId, storageId] of this.redirectTable) {
193
- // Add node for the localId and add a route to the storageId node. The storageId node will have been
194
- // added above when adding nodes for this.blobIds.
195
- gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
196
- }
387
+ // For some blobs, the handle returned on creation is based off of the localId. So, these
388
+ // nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
389
+ // must also be marked referenced. So, we add a route from the localId node to the storageId node.
390
+ // Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
391
+ // a blob may be referenced via its storageId handle.
392
+ for (const [localId, storageId] of this.redirectTable) {
393
+ (0, common_utils_1.assert)(!!storageId, "Must be attached to get GC data");
394
+ // Add node for the localId and add a route to the storageId node. The storageId node will have been
395
+ // added above when adding nodes for this.blobIds.
396
+ gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
197
397
  }
198
398
  return gcData;
199
399
  }
@@ -215,32 +415,33 @@ class BlobManager {
215
415
  this.redirectTable.delete(blobId);
216
416
  continue;
217
417
  }
218
- this.blobIds.delete(blobId);
219
418
  }
220
419
  }
221
420
  summarize(telemetryContext) {
222
- // If we have a redirect table it means the container is about to transition to "Attaching" state, so we need
223
- // to return an actual snapshot containing all the real storage IDs we know about.
224
- const attachingOrAttached = !!this.redirectTable || this.runtime.attachState !== container_definitions_1.AttachState.Detached;
225
- const blobIds = attachingOrAttached ? this.blobIds : this.detachedBlobIds;
421
+ // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
422
+ const blobIds = this.storageIds.size > 0 ? Array.from(this.storageIds) : Array.from(this.redirectTable.keys());
226
423
  const builder = new runtime_utils_1.SummaryTreeBuilder();
227
424
  blobIds.forEach((blobId) => {
228
425
  builder.addAttachment(blobId);
229
426
  });
230
- if (this.redirectTable && this.redirectTable.size > 0) {
231
- builder.addBlob(BlobManager.redirectTableBlobName, JSON.stringify(Array.from(this.redirectTable.entries())));
427
+ // Any non-identity entries in the table need to be saved in the summary
428
+ if (this.redirectTable.size > blobIds.length) {
429
+ builder.addBlob(BlobManager.redirectTableBlobName,
430
+ // filter out identity entries
431
+ JSON.stringify(Array.from(this.redirectTable.entries())
432
+ .filter(([localId, storageId]) => localId !== storageId)));
232
433
  }
233
434
  return builder.getSummaryTree();
234
435
  }
235
436
  setRedirectTable(table) {
236
437
  (0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
237
- (0, common_utils_1.assert)(!this.redirectTable, 0x253 /* "redirect table already exists" */);
438
+ (0, common_utils_1.assert)(this.redirectTable.size === table.size, "Redirect table size must match BlobManager's local ID count");
238
439
  for (const [localId, storageId] of table) {
239
- (0, common_utils_1.assert)(this.detachedBlobIds.delete(localId), 0x254 /* "unrecognized id in redirect table" */);
240
- this.blobIds.add(storageId);
440
+ (0, common_utils_1.assert)(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
441
+ this.redirectTable.set(localId, storageId);
442
+ // set identity (id -> id) entry
443
+ this.redirectTable.set(storageId, storageId);
241
444
  }
242
- (0, common_utils_1.assert)(this.detachedBlobIds.size === 0, 0x255 /* "detached blob id absent in redirect table" */);
243
- this.redirectTable = table;
244
445
  }
245
446
  }
246
447
  exports.BlobManager = BlobManager;