@fluidframework/container-runtime 1.2.2 → 2.0.0-internal.1.0.0.82159
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts +81 -25
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +301 -100
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +65 -11
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +101 -82
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +32 -26
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -23
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +5 -2
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +11 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +17 -12
- package/dist/garbageCollection.js.map +1 -1
- package/dist/opProperties.d.ts +7 -0
- package/dist/opProperties.d.ts.map +1 -0
- package/dist/opProperties.js +20 -0
- package/dist/opProperties.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +14 -4
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +68 -26
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts +0 -2
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +1 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +26 -4
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +95 -18
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +30 -10
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.js +1 -1
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts +0 -5
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts +1 -0
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +11 -9
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts +81 -25
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +302 -101
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +65 -11
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +103 -84
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +32 -26
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +17 -24
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +5 -2
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +11 -3
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +17 -12
- package/lib/garbageCollection.js.map +1 -1
- package/lib/opProperties.d.ts +7 -0
- package/lib/opProperties.d.ts.map +1 -0
- package/lib/opProperties.js +16 -0
- package/lib/opProperties.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +14 -4
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +68 -26
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts +0 -2
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +1 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +26 -4
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +95 -18
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +30 -10
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.js +1 -1
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts +0 -5
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts +1 -0
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +11 -9
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +45 -20
- package/src/blobManager.ts +360 -119
- package/src/containerRuntime.ts +203 -103
- package/src/dataStore.ts +53 -38
- package/src/dataStoreContext.ts +16 -23
- package/src/dataStores.ts +14 -3
- package/src/garbageCollection.ts +13 -7
- package/src/opProperties.ts +19 -0
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +75 -22
- package/src/summarizer.ts +1 -18
- package/src/summarizerHeuristics.ts +133 -19
- package/src/summarizerTypes.ts +37 -10
- package/src/summaryCollection.ts +1 -1
- package/src/summaryFormat.ts +0 -6
- package/src/summaryGenerator.ts +40 -22
- package/dist/opTelemetry.d.ts +0 -22
- package/dist/opTelemetry.d.ts.map +0 -1
- package/dist/opTelemetry.js +0 -59
- package/dist/opTelemetry.js.map +0 -1
- package/lib/opTelemetry.d.ts +0 -22
- package/lib/opTelemetry.d.ts.map +0 -1
- package/lib/opTelemetry.js +0 -55
- package/lib/opTelemetry.js.map +0 -1
- package/src/opTelemetry.ts +0 -71
package/dist/blobManager.js
CHANGED
|
@@ -5,10 +5,12 @@
|
|
|
5
5
|
*/
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.BlobManager = exports.BlobHandle = void 0;
|
|
8
|
+
const uuid_1 = require("uuid");
|
|
8
9
|
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
9
10
|
const common_utils_1 = require("@fluidframework/common-utils");
|
|
10
11
|
const container_definitions_1 = require("@fluidframework/container-definitions");
|
|
11
12
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
13
|
+
const throttler_1 = require("./throttler");
|
|
12
14
|
/**
|
|
13
15
|
* This class represents blob (long string)
|
|
14
16
|
* This object is used only when creating (writing) new blob and serialization purposes.
|
|
@@ -36,33 +38,101 @@ class BlobHandle {
|
|
|
36
38
|
}
|
|
37
39
|
}
|
|
38
40
|
exports.BlobHandle = BlobHandle;
|
|
41
|
+
class CancellableThrottler {
|
|
42
|
+
constructor(throttler) {
|
|
43
|
+
this.throttler = throttler;
|
|
44
|
+
this.cancelP = new common_utils_1.Deferred();
|
|
45
|
+
}
|
|
46
|
+
async getDelay() {
|
|
47
|
+
return Promise.race([
|
|
48
|
+
this.cancelP.promise,
|
|
49
|
+
new Promise((resolve) => setTimeout(resolve, this.throttler.getDelay())),
|
|
50
|
+
]);
|
|
51
|
+
}
|
|
52
|
+
cancel() {
|
|
53
|
+
this.cancelP.resolve();
|
|
54
|
+
this.cancelP = new common_utils_1.Deferred();
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
// Note that while offline we "submit" an op before uploading the blob, but we always
|
|
58
|
+
// expect blobs to be uploaded before we actually see the op round-trip
|
|
59
|
+
var PendingBlobStatus;
|
|
60
|
+
(function (PendingBlobStatus) {
|
|
61
|
+
PendingBlobStatus[PendingBlobStatus["OnlinePendingUpload"] = 0] = "OnlinePendingUpload";
|
|
62
|
+
PendingBlobStatus[PendingBlobStatus["OnlinePendingOp"] = 1] = "OnlinePendingOp";
|
|
63
|
+
PendingBlobStatus[PendingBlobStatus["OfflinePendingUpload"] = 2] = "OfflinePendingUpload";
|
|
64
|
+
PendingBlobStatus[PendingBlobStatus["OfflinePendingOp"] = 3] = "OfflinePendingOp";
|
|
65
|
+
})(PendingBlobStatus || (PendingBlobStatus = {}));
|
|
39
66
|
class BlobManager {
|
|
40
|
-
constructor(routeContext, snapshot, getStorage,
|
|
67
|
+
constructor(routeContext, snapshot, getStorage,
|
|
68
|
+
/**
|
|
69
|
+
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
|
|
70
|
+
* the blob is deleted. The BlobAttach op notifies the server that blob is in use. The server
|
|
71
|
+
* will then not delete the blob as long as it is listed as referenced in future summaries.
|
|
72
|
+
* The summarizing client will know to include the storage ID in the summary when it sees the op.
|
|
73
|
+
*
|
|
74
|
+
* The op may also include a local ID to inform all clients of the relation to the storage
|
|
75
|
+
* ID, without knowledge of which they cannot request the blob from storage. This is also
|
|
76
|
+
* included in the redirect table in the summary.
|
|
77
|
+
*/
|
|
78
|
+
sendBlobAttachOp,
|
|
41
79
|
// To be called when a blob node is requested. blobPath is the path of the blob's node in GC's graph. It's
|
|
42
80
|
// of the format `/<BlobManager.basePath>/<blobId>`.
|
|
43
|
-
gcNodeUpdated, runtime
|
|
81
|
+
gcNodeUpdated, runtime) {
|
|
44
82
|
this.routeContext = routeContext;
|
|
45
83
|
this.getStorage = getStorage;
|
|
46
|
-
this.
|
|
84
|
+
this.sendBlobAttachOp = sendBlobAttachOp;
|
|
47
85
|
this.gcNodeUpdated = gcNodeUpdated;
|
|
48
86
|
this.runtime = runtime;
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
this.
|
|
55
|
-
|
|
56
|
-
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
87
|
+
/**
|
|
88
|
+
* Blobs which have not been uploaded or for which we have not yet seen a BlobAttach op round-trip.
|
|
89
|
+
* Until we see the op round-trip, there is a possibility we may need to re-upload the blob, so
|
|
90
|
+
* we must save it. This is true for both the online and offline flow.
|
|
91
|
+
*/
|
|
92
|
+
this.pendingBlobs = new Map();
|
|
93
|
+
/**
|
|
94
|
+
* Track ops in flight for online flow. Used to avoid searching pendingBlobs since BlobAttach ops
|
|
95
|
+
* don't include local ID in online flow.
|
|
96
|
+
*/
|
|
97
|
+
this.opsInFlight = new Map();
|
|
98
|
+
this.retryThrottler = new CancellableThrottler(new throttler_1.Throttler(60 * 1000, // 60 sec delay window
|
|
99
|
+
30 * 1000, // 30 sec max delay
|
|
100
|
+
// throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
|
|
101
|
+
(0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })));
|
|
102
|
+
this.logger = telemetry_utils_1.ChildLogger.create(this.runtime.logger, "BlobManager");
|
|
103
|
+
this.runtime.on("disconnected", () => this.onDisconnected());
|
|
104
|
+
this.redirectTable = this.load(snapshot);
|
|
63
105
|
}
|
|
64
|
-
|
|
65
|
-
return
|
|
106
|
+
get pendingOfflineUploads() {
|
|
107
|
+
return Array.from(this.pendingBlobs.values())
|
|
108
|
+
.filter((e) => e.status === PendingBlobStatus.OfflinePendingUpload);
|
|
109
|
+
}
|
|
110
|
+
get hasPendingOfflineUploads() {
|
|
111
|
+
return this.pendingOfflineUploads.length > 0;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
|
|
115
|
+
*/
|
|
116
|
+
async onConnected() {
|
|
117
|
+
this.retryThrottler.cancel();
|
|
118
|
+
const pendingUploads = this.pendingOfflineUploads.map(async (e) => e.uploadP);
|
|
119
|
+
await telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, {
|
|
120
|
+
eventName: "BlobUploadOnConnected",
|
|
121
|
+
count: pendingUploads.length,
|
|
122
|
+
}, async () => Promise.all(pendingUploads), { start: true, end: true });
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Transition online blobs waiting for BlobAttach op round-trip since we will not see the op until we are connected
|
|
126
|
+
* again
|
|
127
|
+
*/
|
|
128
|
+
onDisconnected() {
|
|
129
|
+
for (const [localId, entry] of this.pendingBlobs) {
|
|
130
|
+
if (entry.status === PendingBlobStatus.OnlinePendingOp) {
|
|
131
|
+
// This will submit another BlobAttach op for this blob. This is necessary because the one we sent
|
|
132
|
+
// already didn't have the local ID.
|
|
133
|
+
this.transitionToOffline(localId);
|
|
134
|
+
}
|
|
135
|
+
}
|
|
66
136
|
}
|
|
67
137
|
/**
|
|
68
138
|
* For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`
|
|
@@ -72,61 +142,200 @@ class BlobManager {
|
|
|
72
142
|
getBlobGCNodePath(blobId) {
|
|
73
143
|
return `/${BlobManager.basePath}/${blobId}`;
|
|
74
144
|
}
|
|
145
|
+
/**
|
|
146
|
+
* Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
|
|
147
|
+
* detached or there are no (non-pending) attachment blobs in the document
|
|
148
|
+
*/
|
|
149
|
+
get storageIds() {
|
|
150
|
+
const ids = new Set(this.redirectTable.values());
|
|
151
|
+
// If we are detached, we will not have storage IDs, only undefined
|
|
152
|
+
const undefinedValueInTable = ids.delete(undefined);
|
|
153
|
+
// For a detached container, entries are inserted into the redirect table with an undefined storage ID.
|
|
154
|
+
// For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
|
|
155
|
+
(0, common_utils_1.assert)(!undefinedValueInTable || this.runtime.attachState === container_definitions_1.AttachState.Detached && ids.size === 0, "'redirectTable' must contain only undefined while detached / defined values while attached");
|
|
156
|
+
return ids;
|
|
157
|
+
}
|
|
75
158
|
async getBlob(blobId) {
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
|
|
159
|
+
const pending = this.pendingBlobs.get(blobId);
|
|
160
|
+
if (pending) {
|
|
161
|
+
return pending.blob;
|
|
162
|
+
}
|
|
163
|
+
let storageId;
|
|
164
|
+
if (this.runtime.attachState === container_definitions_1.AttachState.Detached) {
|
|
165
|
+
(0, common_utils_1.assert)(this.redirectTable.has(blobId), "requesting unknown blobs");
|
|
166
|
+
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
167
|
+
// The 'IDocumentStorageService.readBlob()' call below will retrieve these via localId.
|
|
168
|
+
storageId = blobId;
|
|
169
|
+
}
|
|
170
|
+
else {
|
|
171
|
+
storageId = this.redirectTable.get(blobId);
|
|
172
|
+
(0, common_utils_1.assert)(!!storageId, 0x11f /* "requesting unknown blobs" */);
|
|
173
|
+
}
|
|
174
|
+
// When a GC-able (not pending) blob is retrieved, let runtime know that the corresponding GC node got updated.
|
|
80
175
|
this.gcNodeUpdated(this.getBlobGCNodePath(blobId));
|
|
81
|
-
return
|
|
176
|
+
return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "AttachmentReadBlob", id: storageId }, async () => {
|
|
82
177
|
return this.getStorage().readBlob(storageId);
|
|
83
|
-
}, { end: true, cancel: "error" })
|
|
178
|
+
}, { end: true, cancel: "error" });
|
|
179
|
+
}
|
|
180
|
+
getBlobHandle(id) {
|
|
181
|
+
(0, common_utils_1.assert)(this.redirectTable.has(id) || this.pendingBlobs.has(id), "requesting handle for unknown blob");
|
|
182
|
+
return new BlobHandle(`${BlobManager.basePath}/${id}`, this.routeContext, async () => this.getBlob(id));
|
|
183
|
+
}
|
|
184
|
+
async createBlobDetached(blob) {
|
|
185
|
+
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
186
|
+
// The 'IDocumentStorageService.createBlob()' call below will respond with a localId.
|
|
187
|
+
const response = await this.getStorage().createBlob(blob);
|
|
188
|
+
this.redirectTable.set(response.id, undefined);
|
|
189
|
+
return this.getBlobHandle(response.id);
|
|
84
190
|
}
|
|
85
191
|
async createBlob(blob) {
|
|
86
|
-
|
|
192
|
+
if (this.runtime.attachState === container_definitions_1.AttachState.Detached) {
|
|
193
|
+
return this.createBlobDetached(blob);
|
|
194
|
+
}
|
|
87
195
|
if (this.runtime.attachState === container_definitions_1.AttachState.Attaching) {
|
|
88
196
|
// blob upload is not supported in "Attaching" state
|
|
89
197
|
this.logger.sendTelemetryEvent({ eventName: "CreateBlobWhileAttaching" });
|
|
90
198
|
await new Promise((resolve) => this.runtime.once("attached", resolve));
|
|
91
199
|
}
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
104
|
-
|
|
105
|
-
|
|
200
|
+
(0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Attached, "For clarity and paranoid defense against adding future attachment states");
|
|
201
|
+
// Create a local ID for each blob. This is used to support blobs if/when the client goes
|
|
202
|
+
// offline since we don't have the ID from storage yet. If online flow succeeds this won't be used.
|
|
203
|
+
const localId = (0, uuid_1.v4)();
|
|
204
|
+
const pendingEntry = {
|
|
205
|
+
blob,
|
|
206
|
+
status: PendingBlobStatus.OnlinePendingUpload,
|
|
207
|
+
handleP: new common_utils_1.Deferred(),
|
|
208
|
+
uploadP: this.uploadBlob(localId, blob),
|
|
209
|
+
};
|
|
210
|
+
this.pendingBlobs.set(localId, pendingEntry);
|
|
211
|
+
return pendingEntry.handleP.promise;
|
|
212
|
+
}
|
|
213
|
+
async uploadBlob(localId, blob) {
|
|
214
|
+
return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.logger, { eventName: "createBlob" }, async () => this.getStorage().createBlob(blob), { end: true, cancel: this.runtime.connected ? "error" : "generic" }).then((response) => this.onUploadResolve(localId, response), async (err) => this.onUploadReject(localId, err));
|
|
215
|
+
}
|
|
216
|
+
onUploadResolve(localId, response) {
|
|
217
|
+
var _a;
|
|
218
|
+
const entry = this.pendingBlobs.get(localId);
|
|
219
|
+
(0, common_utils_1.assert)((entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OnlinePendingUpload ||
|
|
220
|
+
(entry === null || entry === void 0 ? void 0 : entry.status) === PendingBlobStatus.OfflinePendingUpload, "Must have pending blob entry for uploaded blob");
|
|
221
|
+
entry.storageId = response.id;
|
|
222
|
+
if (this.runtime.connected) {
|
|
223
|
+
if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
|
|
224
|
+
if (this.storageIds.has(response.id)) {
|
|
225
|
+
// Storage may dedupe blobs and give us an ID we already know
|
|
226
|
+
// no need to submit BlobAttach op in this case
|
|
227
|
+
entry.handleP.resolve(this.getBlobHandle(response.id));
|
|
228
|
+
this.pendingBlobs.delete(localId);
|
|
229
|
+
}
|
|
230
|
+
else {
|
|
231
|
+
// Check for still-pending duplicates too; if an op is already in flight we can wait for that one
|
|
232
|
+
if (!this.opsInFlight.has(response.id)) {
|
|
233
|
+
this.sendBlobAttachOp(response.id);
|
|
234
|
+
}
|
|
235
|
+
this.opsInFlight.set(response.id, ((_a = this.opsInFlight.get(response.id)) !== null && _a !== void 0 ? _a : []).concat(localId));
|
|
236
|
+
entry.status = PendingBlobStatus.OnlinePendingOp;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
else if (entry.status === PendingBlobStatus.OfflinePendingUpload) {
|
|
240
|
+
// We already submitted a BlobAttach op for this blob when it was transitioned to offline flow
|
|
241
|
+
entry.status = PendingBlobStatus.OfflinePendingOp;
|
|
242
|
+
}
|
|
106
243
|
}
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
|
|
244
|
+
else {
|
|
245
|
+
// connected to storage but not ordering service?
|
|
246
|
+
this.logger.sendTelemetryEvent({ eventName: "BlobUploadSuccessWhileDisconnected" });
|
|
247
|
+
if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
|
|
248
|
+
this.transitionToOffline(localId);
|
|
249
|
+
}
|
|
250
|
+
entry.status = PendingBlobStatus.OfflinePendingOp;
|
|
110
251
|
}
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
252
|
+
return response;
|
|
253
|
+
}
|
|
254
|
+
async onUploadReject(localId, error) {
|
|
255
|
+
const entry = this.pendingBlobs.get(localId);
|
|
256
|
+
(0, common_utils_1.assert)(!!entry, "Must have pending blob entry for blob which failed to upload");
|
|
257
|
+
if (!this.runtime.connected) {
|
|
258
|
+
if (entry.status === PendingBlobStatus.OnlinePendingUpload) {
|
|
259
|
+
this.transitionToOffline(localId);
|
|
260
|
+
}
|
|
261
|
+
// we are probably not connected to storage but start another upload request in case we are
|
|
262
|
+
entry.uploadP = this.retryThrottler.getDelay().then(async () => this.uploadBlob(localId, entry.blob));
|
|
263
|
+
return entry.uploadP;
|
|
264
|
+
}
|
|
265
|
+
else {
|
|
266
|
+
entry.handleP.reject(error);
|
|
267
|
+
throw error;
|
|
268
|
+
}
|
|
269
|
+
}
|
|
270
|
+
transitionToOffline(localId) {
|
|
271
|
+
(0, common_utils_1.assert)(!this.runtime.connected, "Must only transition to offline flow while runtime is disconnected");
|
|
272
|
+
const entry = this.pendingBlobs.get(localId);
|
|
273
|
+
(0, common_utils_1.assert)(!!entry, "No pending blob entry");
|
|
274
|
+
(0, common_utils_1.assert)([PendingBlobStatus.OnlinePendingUpload, PendingBlobStatus.OnlinePendingOp].includes(entry.status), "Blob must be in online flow to transition to offline flow");
|
|
275
|
+
entry.status = entry.status === PendingBlobStatus.OnlinePendingUpload
|
|
276
|
+
? PendingBlobStatus.OfflinePendingUpload
|
|
277
|
+
: PendingBlobStatus.OfflinePendingOp;
|
|
278
|
+
// Submit a BlobAttach op. It's important we submit this op now before returning the blob handle so the
|
|
279
|
+
// BlobAttach op is sequenced prior to any ops referencing the handle. Otherwise an invalid handle could be
|
|
280
|
+
// added to the document if the ops are not all successfully submitted upon reconnection.
|
|
281
|
+
// storageId may be undefined but since we are not connected we will have a chance to add it when reSubmit()
|
|
282
|
+
// is called
|
|
283
|
+
this.sendBlobAttachOp(entry.storageId, localId);
|
|
284
|
+
entry.handleP.resolve(this.getBlobHandle(localId));
|
|
285
|
+
}
|
|
286
|
+
/**
|
|
287
|
+
* Resubmit a BlobAttach op. Used to add storage IDs to ops that were
|
|
288
|
+
* submitted to runtime while disconnected.
|
|
289
|
+
* @param metadata - op metadata containing storage and/or local IDs
|
|
290
|
+
*/
|
|
291
|
+
reSubmit(metadata) {
|
|
292
|
+
(0, common_utils_1.assert)(!!metadata, "Resubmitted ops must have metadata");
|
|
293
|
+
const { blobId, localId } = metadata;
|
|
294
|
+
if (!blobId) {
|
|
295
|
+
(0, common_utils_1.assert)(!!localId, "Submitted BlobAttach ops must have a blobId or localId");
|
|
296
|
+
// We submitted this op while offline. The blob should have been uploaded by now.
|
|
297
|
+
const pendingEntry = this.pendingBlobs.get(localId);
|
|
298
|
+
(0, common_utils_1.assert)((pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.status) === PendingBlobStatus.OfflinePendingOp &&
|
|
299
|
+
!!(pendingEntry === null || pendingEntry === void 0 ? void 0 : pendingEntry.storageId), "blob must be uploaded before resubmitting BlobAttach op");
|
|
300
|
+
return this.sendBlobAttachOp(pendingEntry.storageId, localId);
|
|
116
301
|
}
|
|
117
|
-
return
|
|
302
|
+
return this.sendBlobAttachOp(blobId, localId);
|
|
118
303
|
}
|
|
119
|
-
processBlobAttachOp(
|
|
304
|
+
processBlobAttachOp(message, local) {
|
|
305
|
+
var _a, _b;
|
|
306
|
+
(0, common_utils_1.assert)((_a = message === null || message === void 0 ? void 0 : message.metadata) === null || _a === void 0 ? void 0 : _a.blobId, 0x12a /* "Missing blob id on metadata" */);
|
|
307
|
+
if (message.metadata.localId !== undefined) {
|
|
308
|
+
this.redirectTable.set(message.metadata.localId, message.metadata.blobId);
|
|
309
|
+
}
|
|
310
|
+
// set identity (id -> id) entry
|
|
311
|
+
this.redirectTable.set(message.metadata.blobId, message.metadata.blobId);
|
|
120
312
|
if (local) {
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
313
|
+
if (message.metadata.localId === undefined) {
|
|
314
|
+
// Since there is no local ID, we know this op was submitted while online.
|
|
315
|
+
const waitingBlobs = this.opsInFlight.get(message.metadata.blobId);
|
|
316
|
+
(0, common_utils_1.assert)(!!waitingBlobs, "local online BlobAttach op with no pending blob");
|
|
317
|
+
waitingBlobs.forEach((localId) => {
|
|
318
|
+
const pendingBlobEntry = this.pendingBlobs.get(localId);
|
|
319
|
+
(0, common_utils_1.assert)(pendingBlobEntry !== undefined, "local online BlobAttach op with no pending blob entry");
|
|
320
|
+
// It's possible we transitioned to offline flow while waiting for this op.
|
|
321
|
+
if (pendingBlobEntry.status === PendingBlobStatus.OnlinePendingOp) {
|
|
322
|
+
pendingBlobEntry.handleP.resolve(this.getBlobHandle(message.metadata.blobId));
|
|
323
|
+
this.pendingBlobs.delete(localId);
|
|
324
|
+
}
|
|
325
|
+
});
|
|
326
|
+
}
|
|
327
|
+
else {
|
|
328
|
+
// Each local ID is unique; get the pending blob entry and delete it
|
|
329
|
+
(0, common_utils_1.assert)(((_b = this.pendingBlobs.get(message.metadata.localId)) === null || _b === void 0 ? void 0 : _b.status) === PendingBlobStatus.OfflinePendingOp, 0x1f8 /* "local BlobAttach op with no pending blob" */);
|
|
330
|
+
this.pendingBlobs.delete(message.metadata.localId);
|
|
331
|
+
}
|
|
125
332
|
}
|
|
126
|
-
this.blobIds.add(blobId);
|
|
127
333
|
}
|
|
128
334
|
/**
|
|
129
335
|
* Reads blobs needed to load BlobManager from storage.
|
|
336
|
+
* @param blobsTree - Tree containing IDs of previously attached blobs. We
|
|
337
|
+
* look for the IDs in the blob entries of the tree since the both the r11s
|
|
338
|
+
* and SPO drivers replace the attachment types returned in snapshot() with blobs.
|
|
130
339
|
*/
|
|
131
340
|
static async load(blobsTree, tryFetchBlob) {
|
|
132
341
|
if (!blobsTree) {
|
|
@@ -142,33 +351,23 @@ class BlobManager {
|
|
|
142
351
|
return { ids, redirectTable };
|
|
143
352
|
}
|
|
144
353
|
/**
|
|
145
|
-
* Load a set of previously attached blob IDs from a previous snapshot.
|
|
146
|
-
* that BlobManager tracking and reporting attached blobs is a temporary
|
|
147
|
-
* solution since storage expects attached blobs to be reported and any that
|
|
148
|
-
* are not reported as attached may be GCed. In the future attached blob
|
|
149
|
-
* IDs will be collected at summarization time, and runtime will not care
|
|
150
|
-
* about the existence or specific formatting of this tree in returned
|
|
151
|
-
* snapshots.
|
|
152
|
-
*
|
|
153
|
-
* @param blobsTree - Tree containing IDs of previously attached blobs. This
|
|
154
|
-
* corresponds to snapshot() below. We look for the IDs in the blob entries
|
|
155
|
-
* of the tree since the both the r11s and SPO drivers replace the
|
|
156
|
-
* attachment types returned in snapshot() with blobs.
|
|
354
|
+
* Load a set of previously attached blob IDs and redirect table from a previous snapshot.
|
|
157
355
|
*/
|
|
158
356
|
load(snapshot) {
|
|
159
357
|
var _a, _b, _c;
|
|
160
|
-
if (snapshot.ids) {
|
|
161
|
-
const detached = this.runtime.attachState === container_definitions_1.AttachState.Detached;
|
|
162
|
-
snapshot.ids.map((entry) => detached ? this.detachedBlobIds.add(entry) : this.blobIds.add(entry));
|
|
163
|
-
}
|
|
164
|
-
if (snapshot.redirectTable) {
|
|
165
|
-
this.redirectTable = new Map(snapshot.redirectTable);
|
|
166
|
-
}
|
|
167
358
|
this.logger.sendTelemetryEvent({
|
|
168
359
|
eventName: "AttachmentBlobsLoaded",
|
|
169
360
|
count: (_b = (_a = snapshot.ids) === null || _a === void 0 ? void 0 : _a.length) !== null && _b !== void 0 ? _b : 0,
|
|
170
361
|
redirectTable: (_c = snapshot.redirectTable) === null || _c === void 0 ? void 0 : _c.length,
|
|
171
362
|
});
|
|
363
|
+
const table = new Map(snapshot.redirectTable);
|
|
364
|
+
if (snapshot.ids) {
|
|
365
|
+
const detached = this.runtime.attachState === container_definitions_1.AttachState.Detached;
|
|
366
|
+
// If we are detached, we don't have storage IDs yet, so set to undefined
|
|
367
|
+
// Otherwise, set identity (id -> id) entries
|
|
368
|
+
snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));
|
|
369
|
+
}
|
|
370
|
+
return table;
|
|
172
371
|
}
|
|
173
372
|
/**
|
|
174
373
|
* Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
|
|
@@ -178,22 +377,23 @@ class BlobManager {
|
|
|
178
377
|
*/
|
|
179
378
|
getGCData(fullGC = false) {
|
|
180
379
|
const gcData = { gcNodes: {} };
|
|
181
|
-
|
|
380
|
+
/**
|
|
381
|
+
* The node path is of the format `/_blobs/blobId`. This path must match the path of the blob handle returned
|
|
382
|
+
* by the createBlob API because blobs are marked referenced by storing these handles in a referenced DDS.
|
|
383
|
+
*/
|
|
384
|
+
this.storageIds.forEach((blobId) => {
|
|
182
385
|
gcData.gcNodes[this.getBlobGCNodePath(blobId)] = [];
|
|
183
386
|
});
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
// added above when adding nodes for this.blobIds.
|
|
195
|
-
gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
|
|
196
|
-
}
|
|
387
|
+
// For some blobs, the handle returned on creation is based off of the localId. So, these
|
|
388
|
+
// nodes can be referenced by storing the localId handle. When that happens, the corresponding storageId node
|
|
389
|
+
// must also be marked referenced. So, we add a route from the localId node to the storageId node.
|
|
390
|
+
// Note that because of de-duping, there can be multiple localIds that all redirect to the same storageId or
|
|
391
|
+
// a blob may be referenced via its storageId handle.
|
|
392
|
+
for (const [localId, storageId] of this.redirectTable) {
|
|
393
|
+
(0, common_utils_1.assert)(!!storageId, "Must be attached to get GC data");
|
|
394
|
+
// Add node for the localId and add a route to the storageId node. The storageId node will have been
|
|
395
|
+
// added above when adding nodes for this.blobIds.
|
|
396
|
+
gcData.gcNodes[this.getBlobGCNodePath(localId)] = [this.getBlobGCNodePath(storageId)];
|
|
197
397
|
}
|
|
198
398
|
return gcData;
|
|
199
399
|
}
|
|
@@ -215,32 +415,33 @@ class BlobManager {
|
|
|
215
415
|
this.redirectTable.delete(blobId);
|
|
216
416
|
continue;
|
|
217
417
|
}
|
|
218
|
-
this.blobIds.delete(blobId);
|
|
219
418
|
}
|
|
220
419
|
}
|
|
221
420
|
summarize(telemetryContext) {
|
|
222
|
-
//
|
|
223
|
-
|
|
224
|
-
const attachingOrAttached = !!this.redirectTable || this.runtime.attachState !== container_definitions_1.AttachState.Detached;
|
|
225
|
-
const blobIds = attachingOrAttached ? this.blobIds : this.detachedBlobIds;
|
|
421
|
+
// if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
|
|
422
|
+
const blobIds = this.storageIds.size > 0 ? Array.from(this.storageIds) : Array.from(this.redirectTable.keys());
|
|
226
423
|
const builder = new runtime_utils_1.SummaryTreeBuilder();
|
|
227
424
|
blobIds.forEach((blobId) => {
|
|
228
425
|
builder.addAttachment(blobId);
|
|
229
426
|
});
|
|
230
|
-
|
|
231
|
-
|
|
427
|
+
// Any non-identity entries in the table need to be saved in the summary
|
|
428
|
+
if (this.redirectTable.size > blobIds.length) {
|
|
429
|
+
builder.addBlob(BlobManager.redirectTableBlobName,
|
|
430
|
+
// filter out identity entries
|
|
431
|
+
JSON.stringify(Array.from(this.redirectTable.entries())
|
|
432
|
+
.filter(([localId, storageId]) => localId !== storageId)));
|
|
232
433
|
}
|
|
233
434
|
return builder.getSummaryTree();
|
|
234
435
|
}
|
|
235
436
|
setRedirectTable(table) {
|
|
236
437
|
(0, common_utils_1.assert)(this.runtime.attachState === container_definitions_1.AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
|
|
237
|
-
(0, common_utils_1.assert)(
|
|
438
|
+
(0, common_utils_1.assert)(this.redirectTable.size === table.size, "Redirect table size must match BlobManager's local ID count");
|
|
238
439
|
for (const [localId, storageId] of table) {
|
|
239
|
-
(0, common_utils_1.assert)(this.
|
|
240
|
-
this.
|
|
440
|
+
(0, common_utils_1.assert)(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
|
|
441
|
+
this.redirectTable.set(localId, storageId);
|
|
442
|
+
// set identity (id -> id) entry
|
|
443
|
+
this.redirectTable.set(storageId, storageId);
|
|
241
444
|
}
|
|
242
|
-
(0, common_utils_1.assert)(this.detachedBlobIds.size === 0, 0x255 /* "detached blob id absent in redirect table" */);
|
|
243
|
-
this.redirectTable = table;
|
|
244
445
|
}
|
|
245
446
|
}
|
|
246
447
|
exports.BlobManager = BlobManager;
|