@fluidframework/container-runtime 2.51.0 → 2.53.0-350190
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/api-report/container-runtime.legacy.alpha.api.md +1 -2
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +15 -7
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +72 -186
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/containerCompatibility.d.ts +34 -0
- package/dist/containerCompatibility.d.ts.map +1 -0
- package/dist/containerCompatibility.js +125 -0
- package/dist/containerCompatibility.js.map +1 -0
- package/dist/containerRuntime.d.ts +27 -15
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +175 -136
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +6 -6
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +5 -0
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +5 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/metadata.d.ts +3 -2
- package/dist/metadata.d.ts.map +1 -1
- package/dist/metadata.js +7 -1
- package/dist/metadata.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +40 -5
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -1
- package/dist/storageServiceWithAttachBlobs.js +56 -5
- package/dist/storageServiceWithAttachBlobs.js.map +1 -1
- package/dist/summary/documentSchema.d.ts +1 -1
- package/dist/summary/documentSchema.d.ts.map +1 -1
- package/dist/summary/documentSchema.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +3 -3
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +15 -7
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +39 -153
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/containerCompatibility.d.ts +34 -0
- package/lib/containerCompatibility.d.ts.map +1 -0
- package/lib/containerCompatibility.js +120 -0
- package/lib/containerCompatibility.js.map +1 -0
- package/lib/containerRuntime.d.ts +27 -15
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +103 -64
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +6 -6
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +1 -1
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +5 -0
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +5 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/metadata.d.ts +3 -2
- package/lib/metadata.d.ts.map +1 -1
- package/lib/metadata.js +5 -0
- package/lib/metadata.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +40 -5
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -1
- package/lib/storageServiceWithAttachBlobs.js +56 -5
- package/lib/storageServiceWithAttachBlobs.js.map +1 -1
- package/lib/summary/documentSchema.d.ts +1 -1
- package/lib/summary/documentSchema.d.ts.map +1 -1
- package/lib/summary/documentSchema.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +3 -3
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +20 -20
- package/src/blobManager/blobManager.ts +53 -195
- package/src/containerCompatibility.ts +176 -0
- package/src/containerRuntime.ts +157 -122
- package/src/dataStoreContext.ts +13 -5
- package/src/gc/garbageCollection.ts +6 -0
- package/src/index.ts +6 -1
- package/src/metadata.ts +10 -2
- package/src/packageVersion.ts +1 -1
- package/src/storageServiceWithAttachBlobs.ts +92 -10
- package/src/summary/documentSchema.ts +1 -1
- package/src/summary/summaryFormat.ts +2 -2
- package/dist/compatUtils.d.ts +0 -106
- package/dist/compatUtils.d.ts.map +0 -1
- package/dist/compatUtils.js +0 -251
- package/dist/compatUtils.js.map +0 -1
- package/lib/compatUtils.d.ts +0 -106
- package/lib/compatUtils.d.ts.map +0 -1
- package/lib/compatUtils.js +0 -242
- package/lib/compatUtils.js.map +0 -1
- package/src/compatUtils.ts +0 -365
|
@@ -3,8 +3,11 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
6
|
+
import { createEmitter } from "@fluid-internal/client-utils";
|
|
7
|
+
import {
|
|
8
|
+
AttachState,
|
|
9
|
+
type IContainerStorageService,
|
|
10
|
+
} from "@fluidframework/container-definitions/internal";
|
|
8
11
|
import {
|
|
9
12
|
IContainerRuntime,
|
|
10
13
|
IContainerRuntimeEvents,
|
|
@@ -20,10 +23,7 @@ import type {
|
|
|
20
23
|
PayloadState,
|
|
21
24
|
} from "@fluidframework/core-interfaces/internal";
|
|
22
25
|
import { assert, Deferred } from "@fluidframework/core-utils/internal";
|
|
23
|
-
import {
|
|
24
|
-
IDocumentStorageService,
|
|
25
|
-
ICreateBlobResponse,
|
|
26
|
-
} from "@fluidframework/driver-definitions/internal";
|
|
26
|
+
import { ICreateBlobResponse } from "@fluidframework/driver-definitions/internal";
|
|
27
27
|
import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils/internal";
|
|
28
28
|
import {
|
|
29
29
|
IGarbageCollectionData,
|
|
@@ -41,12 +41,13 @@ import {
|
|
|
41
41
|
LoggingError,
|
|
42
42
|
MonitoringContext,
|
|
43
43
|
PerformanceEvent,
|
|
44
|
+
UsageError,
|
|
44
45
|
createChildMonitoringContext,
|
|
45
46
|
wrapError,
|
|
46
47
|
} from "@fluidframework/telemetry-utils/internal";
|
|
47
48
|
import { v4 as uuid } from "uuid";
|
|
48
49
|
|
|
49
|
-
import {
|
|
50
|
+
import { isBlobMetadata } from "../metadata.js";
|
|
50
51
|
|
|
51
52
|
import {
|
|
52
53
|
getStorageIds,
|
|
@@ -148,7 +149,6 @@ interface PendingBlob {
|
|
|
148
149
|
attached?: boolean;
|
|
149
150
|
acked?: boolean;
|
|
150
151
|
abortSignal?: AbortSignal;
|
|
151
|
-
stashedUpload?: boolean;
|
|
152
152
|
}
|
|
153
153
|
|
|
154
154
|
export interface IPendingBlobs {
|
|
@@ -171,16 +171,6 @@ interface IBlobManagerInternalEvents {
|
|
|
171
171
|
processedBlobAttach: (localId: string, storageId: string) => void;
|
|
172
172
|
}
|
|
173
173
|
|
|
174
|
-
const stashedPendingBlobOverrides: Pick<
|
|
175
|
-
PendingBlob,
|
|
176
|
-
"stashedUpload" | "storageId" | "minTTLInSeconds" | "uploadTime"
|
|
177
|
-
> = {
|
|
178
|
-
stashedUpload: true,
|
|
179
|
-
storageId: undefined,
|
|
180
|
-
minTTLInSeconds: undefined,
|
|
181
|
-
uploadTime: undefined,
|
|
182
|
-
} as const;
|
|
183
|
-
|
|
184
174
|
export const blobManagerBasePath = "_blobs" as const;
|
|
185
175
|
|
|
186
176
|
export class BlobManager {
|
|
@@ -213,11 +203,10 @@ export class BlobManager {
|
|
|
213
203
|
*/
|
|
214
204
|
private readonly opsInFlight: Map<string, Set<string>> = new Map();
|
|
215
205
|
|
|
216
|
-
private readonly sendBlobAttachOp: (localId: string, storageId
|
|
217
|
-
private stopAttaching: boolean = false;
|
|
206
|
+
private readonly sendBlobAttachOp: (localId: string, storageId: string) => void;
|
|
218
207
|
|
|
219
208
|
private readonly routeContext: IFluidHandleContext;
|
|
220
|
-
private readonly storage:
|
|
209
|
+
private readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
|
|
221
210
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
222
211
|
// blobPath's format - `/<basePath>/<blobId>`.
|
|
223
212
|
private readonly blobRequested: (blobPath: string) => void;
|
|
@@ -226,9 +215,6 @@ export class BlobManager {
|
|
|
226
215
|
private readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
227
216
|
private readonly runtime: IBlobManagerRuntime;
|
|
228
217
|
private readonly localBlobIdGenerator: () => string;
|
|
229
|
-
private readonly pendingStashedBlobs: Map<string, Promise<ICreateBlobResponse | void>> =
|
|
230
|
-
new Map();
|
|
231
|
-
public readonly stashedBlobsUploadP: Promise<(void | ICreateBlobResponse)[]>;
|
|
232
218
|
|
|
233
219
|
private readonly createBlobPayloadPending: boolean;
|
|
234
220
|
|
|
@@ -236,7 +222,7 @@ export class BlobManager {
|
|
|
236
222
|
readonly routeContext: IFluidHandleContext;
|
|
237
223
|
|
|
238
224
|
blobManagerLoadInfo: IBlobManagerLoadInfo;
|
|
239
|
-
readonly storage:
|
|
225
|
+
readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
|
|
240
226
|
/**
|
|
241
227
|
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
|
|
242
228
|
* deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
|
|
@@ -247,7 +233,7 @@ export class BlobManager {
|
|
|
247
233
|
* knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
|
|
248
234
|
* before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
|
|
249
235
|
*/
|
|
250
|
-
sendBlobAttachOp: (localId: string, storageId
|
|
236
|
+
sendBlobAttachOp: (localId: string, storageId: string) => void;
|
|
251
237
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
252
238
|
// blobPath's format - `/<basePath>/<blobId>`.
|
|
253
239
|
readonly blobRequested: (blobPath: string) => void;
|
|
@@ -267,7 +253,6 @@ export class BlobManager {
|
|
|
267
253
|
blobRequested,
|
|
268
254
|
isBlobDeleted,
|
|
269
255
|
runtime,
|
|
270
|
-
stashedBlobs,
|
|
271
256
|
localBlobIdGenerator,
|
|
272
257
|
createBlobPayloadPending,
|
|
273
258
|
} = props;
|
|
@@ -290,48 +275,7 @@ export class BlobManager {
|
|
|
290
275
|
this.runtime.attachState,
|
|
291
276
|
);
|
|
292
277
|
|
|
293
|
-
|
|
294
|
-
for (const [localId, entry] of Object.entries(stashedBlobs ?? {})) {
|
|
295
|
-
const { acked, storageId, minTTLInSeconds, uploadTime } = entry;
|
|
296
|
-
const blob = stringToBuffer(entry.blob, "base64");
|
|
297
|
-
const pendingEntry: PendingBlob = {
|
|
298
|
-
blob,
|
|
299
|
-
opsent: true,
|
|
300
|
-
handleP: new Deferred(),
|
|
301
|
-
storageId,
|
|
302
|
-
uploadP: undefined,
|
|
303
|
-
uploadTime,
|
|
304
|
-
minTTLInSeconds,
|
|
305
|
-
attached: true,
|
|
306
|
-
acked,
|
|
307
|
-
};
|
|
308
|
-
this.pendingBlobs.set(localId, pendingEntry);
|
|
309
|
-
|
|
310
|
-
if (storageId !== undefined && minTTLInSeconds && uploadTime) {
|
|
311
|
-
const timeLapseSinceLocalUpload = (Date.now() - uploadTime) / 1000;
|
|
312
|
-
// stashed entries with more than half-life in storage will not be reuploaded
|
|
313
|
-
if (minTTLInSeconds - timeLapseSinceLocalUpload > minTTLInSeconds / 2) {
|
|
314
|
-
continue;
|
|
315
|
-
}
|
|
316
|
-
}
|
|
317
|
-
this.pendingStashedBlobs.set(localId, this.uploadBlob(localId, blob));
|
|
318
|
-
this.pendingBlobs.set(localId, {
|
|
319
|
-
...pendingEntry,
|
|
320
|
-
...stashedPendingBlobOverrides,
|
|
321
|
-
uploadP: this.pendingStashedBlobs.get(localId),
|
|
322
|
-
});
|
|
323
|
-
}
|
|
324
|
-
|
|
325
|
-
this.stashedBlobsUploadP = PerformanceEvent.timedExecAsync(
|
|
326
|
-
this.mc.logger,
|
|
327
|
-
{ eventName: "BlobUploadProcessStashedChanges", count: this.pendingStashedBlobs.size },
|
|
328
|
-
async () => Promise.all(this.pendingStashedBlobs.values()),
|
|
329
|
-
{ start: true, end: true },
|
|
330
|
-
).finally(() => {
|
|
331
|
-
this.pendingStashedBlobs.clear();
|
|
332
|
-
});
|
|
333
|
-
|
|
334
|
-
this.sendBlobAttachOp = (localId: string, blobId?: string) => {
|
|
278
|
+
this.sendBlobAttachOp = (localId: string, blobId: string) => {
|
|
335
279
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
336
280
|
assert(
|
|
337
281
|
pendingEntry !== undefined,
|
|
@@ -387,10 +331,6 @@ export class BlobManager {
|
|
|
387
331
|
});
|
|
388
332
|
}
|
|
389
333
|
|
|
390
|
-
public hasPendingStashedUploads(): boolean {
|
|
391
|
-
return [...this.pendingBlobs.values()].some((e) => e.stashedUpload === true);
|
|
392
|
-
}
|
|
393
|
-
|
|
394
334
|
public hasBlob(blobId: string): boolean {
|
|
395
335
|
return this.redirectTable.get(blobId) !== undefined;
|
|
396
336
|
}
|
|
@@ -420,7 +360,7 @@ export class BlobManager {
|
|
|
420
360
|
assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
|
|
421
361
|
|
|
422
362
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
423
|
-
// The '
|
|
363
|
+
// The 'IContainerStorageService.readBlob()' call below will retrieve these via localId.
|
|
424
364
|
storageId = blobId;
|
|
425
365
|
} else {
|
|
426
366
|
const attachedStorageId = this.redirectTable.get(blobId);
|
|
@@ -491,7 +431,7 @@ export class BlobManager {
|
|
|
491
431
|
blob: ArrayBufferLike,
|
|
492
432
|
): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
|
|
493
433
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
494
|
-
// The '
|
|
434
|
+
// The 'IContainerStorageService.createBlob()' call below will respond with a localId.
|
|
495
435
|
const response = await this.storage.createBlob(blob);
|
|
496
436
|
this.setRedirection(response.id, undefined);
|
|
497
437
|
return this.getBlobHandle(response.id);
|
|
@@ -671,18 +611,9 @@ export class BlobManager {
|
|
|
671
611
|
response: ICreateBlobResponseWithTTL,
|
|
672
612
|
): ICreateBlobResponseWithTTL | undefined {
|
|
673
613
|
const entry = this.pendingBlobs.get(localId);
|
|
674
|
-
if (entry === undefined && this.pendingStashedBlobs.has(localId)) {
|
|
675
|
-
// The blob was already processed and deleted. This can happen if the blob was reuploaded by
|
|
676
|
-
// the stashing process and the original upload was processed before the stashed upload.
|
|
677
|
-
this.mc.logger.sendTelemetryEvent({
|
|
678
|
-
eventName: "StashedBlobAlreadyProcessed",
|
|
679
|
-
localId,
|
|
680
|
-
});
|
|
681
|
-
return;
|
|
682
|
-
}
|
|
683
614
|
|
|
684
615
|
assert(entry !== undefined, 0x6c8 /* pending blob entry not found for uploaded blob */);
|
|
685
|
-
if (
|
|
616
|
+
if (entry.abortSignal?.aborted === true && !entry.opsent) {
|
|
686
617
|
this.mc.logger.sendTelemetryEvent({
|
|
687
618
|
eventName: "BlobAborted",
|
|
688
619
|
localId,
|
|
@@ -694,7 +625,6 @@ export class BlobManager {
|
|
|
694
625
|
entry.storageId === undefined,
|
|
695
626
|
0x386 /* Must have pending blob entry for uploaded blob */,
|
|
696
627
|
);
|
|
697
|
-
entry.stashedUpload = undefined;
|
|
698
628
|
entry.storageId = response.id;
|
|
699
629
|
entry.uploadTime = Date.now();
|
|
700
630
|
entry.minTTLInSeconds = response.minTTLInSeconds;
|
|
@@ -738,46 +668,35 @@ export class BlobManager {
|
|
|
738
668
|
* @param metadata - op metadata containing storage and/or local IDs
|
|
739
669
|
*/
|
|
740
670
|
public reSubmit(metadata: Record<string, unknown> | undefined): void {
|
|
741
|
-
assert(
|
|
742
|
-
const { localId, blobId }
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
);
|
|
752
|
-
return this.sendBlobAttachOp(localId, pendingEntry?.storageId);
|
|
671
|
+
assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
|
|
672
|
+
const { localId, blobId } = metadata;
|
|
673
|
+
// Any blob that we're actively trying to advance to attached state must have a
|
|
674
|
+
// pendingBlobs entry. Decline to resubmit for anything else.
|
|
675
|
+
// For example, we might be asked to resubmit stashed ops for blobs that never had
|
|
676
|
+
// their handle attached - these won't have a pendingBlobs entry and we shouldn't
|
|
677
|
+
// try to attach them since they won't be accessible to the customer and would just
|
|
678
|
+
// be considered garbage immediately.
|
|
679
|
+
if (this.pendingBlobs.has(localId)) {
|
|
680
|
+
this.sendBlobAttachOp(localId, blobId);
|
|
753
681
|
}
|
|
754
|
-
return this.sendBlobAttachOp(localId, blobId);
|
|
755
682
|
}
|
|
756
683
|
|
|
757
684
|
public processBlobAttachMessage(message: ISequencedMessageEnvelope, local: boolean): void {
|
|
758
|
-
|
|
759
|
-
|
|
760
|
-
|
|
761
|
-
|
|
762
|
-
|
|
763
|
-
|
|
764
|
-
|
|
765
|
-
|
|
766
|
-
|
|
767
|
-
}
|
|
768
|
-
assert(blobId !== undefined, 0x12a /* "Missing blob id on metadata" */);
|
|
769
|
-
|
|
770
|
-
// Set up a mapping from local ID to storage ID. This is crucial since without this the blob cannot be
|
|
771
|
-
// requested from the server.
|
|
772
|
-
// Note: The check for undefined is needed for back-compat when localId was not part of the BlobAttach op that
|
|
773
|
-
// was sent when online.
|
|
774
|
-
if (localId !== undefined) {
|
|
775
|
-
this.setRedirection(localId, blobId);
|
|
685
|
+
assert(
|
|
686
|
+
isBlobMetadata(message.metadata),
|
|
687
|
+
0xc02 /* Expected blob metadata for a BlobAttach op */,
|
|
688
|
+
);
|
|
689
|
+
const { localId, blobId } = message.metadata;
|
|
690
|
+
const pendingEntry = this.pendingBlobs.get(localId);
|
|
691
|
+
if (pendingEntry?.abortSignal?.aborted) {
|
|
692
|
+
this.deletePendingBlob(localId);
|
|
693
|
+
return;
|
|
776
694
|
}
|
|
695
|
+
|
|
696
|
+
this.setRedirection(localId, blobId);
|
|
777
697
|
// set identity (id -> id) entry
|
|
778
698
|
this.setRedirection(blobId, blobId);
|
|
779
699
|
|
|
780
|
-
assert(localId !== undefined, 0x50e /* local ID not present in blob attach message */);
|
|
781
700
|
if (local) {
|
|
782
701
|
const waitingBlobs = this.opsInFlight.get(blobId);
|
|
783
702
|
if (waitingBlobs !== undefined) {
|
|
@@ -950,6 +869,21 @@ export class BlobManager {
|
|
|
950
869
|
}
|
|
951
870
|
}
|
|
952
871
|
|
|
872
|
+
/**
|
|
873
|
+
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
874
|
+
* pending upload and/or their BlobAttach op, which can be given to a new BlobManager to
|
|
875
|
+
* resume work.
|
|
876
|
+
*
|
|
877
|
+
* @privateRemarks
|
|
878
|
+
* For now, we don't track any pending blobs since the getPendingBlobs flow doesn't enable
|
|
879
|
+
* restoring to a state where an accessible handle has been stored by the customer (and we'll
|
|
880
|
+
* just drop any BlobAttach ops on the ground during reSubmit). However, once we add support
|
|
881
|
+
* for payload-pending handles, this will return the blobs associated with those handles.
|
|
882
|
+
*/
|
|
883
|
+
public getPendingBlobs(): IPendingBlobs | undefined {
|
|
884
|
+
return undefined;
|
|
885
|
+
}
|
|
886
|
+
|
|
953
887
|
/**
|
|
954
888
|
* Part of container serialization when imminent closure is enabled (Currently when calling closeAndGetPendingLocalState).
|
|
955
889
|
* This asynchronous function resolves all pending createBlob calls and waits for each blob
|
|
@@ -963,83 +897,7 @@ export class BlobManager {
|
|
|
963
897
|
public async attachAndGetPendingBlobs(
|
|
964
898
|
stopBlobAttachingSignal?: AbortSignal,
|
|
965
899
|
): Promise<IPendingBlobs | undefined> {
|
|
966
|
-
|
|
967
|
-
this.mc.logger,
|
|
968
|
-
{ eventName: "GetPendingBlobs" },
|
|
969
|
-
async () => {
|
|
970
|
-
if (this.pendingBlobs.size === 0) {
|
|
971
|
-
return;
|
|
972
|
-
}
|
|
973
|
-
const blobs = {};
|
|
974
|
-
const localBlobs = new Set<PendingBlob>();
|
|
975
|
-
// This while is used to stash blobs created while attaching and getting blobs
|
|
976
|
-
while (localBlobs.size < this.pendingBlobs.size) {
|
|
977
|
-
const attachHandlesP: Promise<void>[] = [];
|
|
978
|
-
for (const [localId, entry] of this.pendingBlobs) {
|
|
979
|
-
if (!localBlobs.has(entry)) {
|
|
980
|
-
localBlobs.add(entry);
|
|
981
|
-
// In order to follow natural blob creation flow we need to:
|
|
982
|
-
// 1 send the blob attach op
|
|
983
|
-
// 2 resolve the blob handle
|
|
984
|
-
// 3 wait for op referencing the blob
|
|
985
|
-
if (!entry.opsent) {
|
|
986
|
-
this.sendBlobAttachOp(localId, entry.storageId);
|
|
987
|
-
}
|
|
988
|
-
// Resolving the blob handle to let hosts continue with their operations (it will resolve
|
|
989
|
-
// original createBlob call) and let them attach the blob. This is a lie we told since the upload
|
|
990
|
-
// hasn't finished yet, but it's fine since we will retry on rehydration.
|
|
991
|
-
entry.handleP.resolve(this.getBlobHandle(localId));
|
|
992
|
-
// Array of promises that will resolve when handles get attached.
|
|
993
|
-
attachHandlesP.push(
|
|
994
|
-
new Promise<void>((resolve, reject) => {
|
|
995
|
-
stopBlobAttachingSignal?.addEventListener(
|
|
996
|
-
"abort",
|
|
997
|
-
() => {
|
|
998
|
-
this.stopAttaching = true;
|
|
999
|
-
reject(new Error("Operation aborted"));
|
|
1000
|
-
},
|
|
1001
|
-
{ once: true },
|
|
1002
|
-
);
|
|
1003
|
-
const onHandleAttached = (attachedEntry: PendingBlob): void => {
|
|
1004
|
-
if (attachedEntry === entry) {
|
|
1005
|
-
this.internalEvents.off("handleAttached", onHandleAttached);
|
|
1006
|
-
resolve();
|
|
1007
|
-
}
|
|
1008
|
-
};
|
|
1009
|
-
if (entry.attached) {
|
|
1010
|
-
resolve();
|
|
1011
|
-
} else {
|
|
1012
|
-
this.internalEvents.on("handleAttached", onHandleAttached);
|
|
1013
|
-
}
|
|
1014
|
-
}),
|
|
1015
|
-
);
|
|
1016
|
-
}
|
|
1017
|
-
}
|
|
1018
|
-
// Wait for all blobs to be attached. This is important, otherwise serialized container
|
|
1019
|
-
// could send the blobAttach op without any op that references the blob, making it useless.
|
|
1020
|
-
await Promise.allSettled(attachHandlesP);
|
|
1021
|
-
}
|
|
1022
|
-
|
|
1023
|
-
for (const [localId, entry] of this.pendingBlobs) {
|
|
1024
|
-
if (stopBlobAttachingSignal?.aborted && !entry.attached) {
|
|
1025
|
-
this.mc.logger.sendTelemetryEvent({
|
|
1026
|
-
eventName: "UnableToStashBlob",
|
|
1027
|
-
id: localId,
|
|
1028
|
-
});
|
|
1029
|
-
continue;
|
|
1030
|
-
}
|
|
1031
|
-
assert(entry.attached === true, 0x790 /* stashed blob should be attached */);
|
|
1032
|
-
blobs[localId] = {
|
|
1033
|
-
blob: bufferToString(entry.blob, "base64"),
|
|
1034
|
-
storageId: entry.storageId,
|
|
1035
|
-
acked: entry.acked,
|
|
1036
|
-
minTTLInSeconds: entry.minTTLInSeconds,
|
|
1037
|
-
uploadTime: entry.uploadTime,
|
|
1038
|
-
};
|
|
1039
|
-
}
|
|
1040
|
-
return Object.keys(blobs).length > 0 ? blobs : undefined;
|
|
1041
|
-
},
|
|
1042
|
-
);
|
|
900
|
+
throw new UsageError("attachAndGetPendingBlobs is no longer supported");
|
|
1043
901
|
}
|
|
1044
902
|
}
|
|
1045
903
|
|
|
@@ -0,0 +1,176 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
FlushMode,
|
|
8
|
+
type MinimumVersionForCollab,
|
|
9
|
+
} from "@fluidframework/runtime-definitions/internal";
|
|
10
|
+
import {
|
|
11
|
+
configValueToMinVersionForCollab,
|
|
12
|
+
getConfigsForMinVersionForCollab,
|
|
13
|
+
getValidationForRuntimeOptions,
|
|
14
|
+
type ConfigMap,
|
|
15
|
+
type ConfigValidationMap,
|
|
16
|
+
} from "@fluidframework/runtime-utils/internal";
|
|
17
|
+
|
|
18
|
+
import {
|
|
19
|
+
disabledCompressionConfig,
|
|
20
|
+
enabledCompressionConfig,
|
|
21
|
+
} from "./compressionDefinitions.js";
|
|
22
|
+
import type { ContainerRuntimeOptionsInternal } from "./containerRuntime.js";
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Subset of the {@link ContainerRuntimeOptionsInternal} properties which
|
|
26
|
+
* affect {@link IDocumentSchemaFeatures}.
|
|
27
|
+
*
|
|
28
|
+
* @remarks
|
|
29
|
+
* When a new option is added to {@link ContainerRuntimeOptionsInternal}, we
|
|
30
|
+
* must consider if it changes the DocumentSchema. If so, then a corresponding
|
|
31
|
+
* entry must be added to {@link runtimeOptionsAffectingDocSchemaConfigMap}
|
|
32
|
+
* below. If not, then it must be omitted from this type.
|
|
33
|
+
*
|
|
34
|
+
* Note: `Omit` is used instead of `Pick` to ensure that all new options are
|
|
35
|
+
* included in this type by default. If any new properties are added to
|
|
36
|
+
* {@link ContainerRuntimeOptionsInternal}, they will be included in this
|
|
37
|
+
* type unless explicitly omitted. This will prevent us from forgetting to
|
|
38
|
+
* account for any new properties in the future.
|
|
39
|
+
*/
|
|
40
|
+
export type RuntimeOptionsAffectingDocSchema = Omit<
|
|
41
|
+
ContainerRuntimeOptionsInternal,
|
|
42
|
+
| "chunkSizeInBytes"
|
|
43
|
+
| "maxBatchSizeInBytes"
|
|
44
|
+
| "loadSequenceNumberVerification"
|
|
45
|
+
| "summaryOptions"
|
|
46
|
+
>;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Mapping of RuntimeOptionsAffectingDocSchema to their compatibility related configs.
|
|
50
|
+
*
|
|
51
|
+
* Each key in this map corresponds to a property in RuntimeOptionsAffectingDocSchema. The value is an object that maps MinimumVersionForCollab
|
|
52
|
+
* to the appropriate default value for that property to supporting that MinimumVersionForCollab. If clients running MinimumVersionForCollab X are able to understand
|
|
53
|
+
* the format changes introduced by the property, then the default value for that MinimumVersionForCollab will enable the feature associated with the property.
|
|
54
|
+
* Otherwise, the feature will be disabled.
|
|
55
|
+
*
|
|
56
|
+
* For example if the minVersionForCollab is a 1.x version (i.e. "1.5.0"), then the default value for `enableGroupedBatching` will be false since 1.x
|
|
57
|
+
* clients do not understand the document format when batching is enabled. If the minVersionForCollab is a 2.x client (i.e. "2.0.0" or later), then the
|
|
58
|
+
* default value for `enableGroupedBatching` will be true because clients running 2.0 or later will be able to understand the format changes associated
|
|
59
|
+
* with the batching feature.
|
|
60
|
+
*/
|
|
61
|
+
const runtimeOptionsAffectingDocSchemaConfigMap = {
|
|
62
|
+
enableGroupedBatching: {
|
|
63
|
+
"1.0.0": false,
|
|
64
|
+
"2.0.0-defaults": true,
|
|
65
|
+
},
|
|
66
|
+
compressionOptions: {
|
|
67
|
+
"1.0.0": disabledCompressionConfig,
|
|
68
|
+
"2.0.0-defaults": enabledCompressionConfig,
|
|
69
|
+
},
|
|
70
|
+
enableRuntimeIdCompressor: {
|
|
71
|
+
// For IdCompressorMode, `undefined` represents a logical state (off).
|
|
72
|
+
// However, to satisfy the Required<> constraint while
|
|
73
|
+
// `exactOptionalPropertyTypes` is `false` (TODO: AB#8215), we need
|
|
74
|
+
// to have it defined, so we trick the type checker here.
|
|
75
|
+
"1.0.0": undefined,
|
|
76
|
+
// We do not yet want to enable idCompressor by default since it will
|
|
77
|
+
// increase bundle sizes, and not all customers will benefit from it.
|
|
78
|
+
// Therefore, we will require customers to explicitly enable it. We
|
|
79
|
+
// are keeping it as a DocSchema affecting option for now as this may
|
|
80
|
+
// change in the future.
|
|
81
|
+
},
|
|
82
|
+
explicitSchemaControl: {
|
|
83
|
+
"1.0.0": false,
|
|
84
|
+
// This option's intention is to prevent 1.x clients from joining sessions
|
|
85
|
+
// when enabled. This is set to true when the minVersionForCollab is set
|
|
86
|
+
// to >=2.0.0 (explicitly). This is different than other 2.0 defaults
|
|
87
|
+
// because it was not enabled by default prior to the implementation of
|
|
88
|
+
// `minVersionForCollab`.
|
|
89
|
+
// `defaultMinVersionForCollab` is set to "2.0.0-defaults" which "2.0.0"
|
|
90
|
+
// does not satisfy to avoiding enabling this option by default as of
|
|
91
|
+
// `minVersionForCollab` introduction, which could be unexpected.
|
|
92
|
+
// Only enable as a default when `minVersionForCollab` is specified at
|
|
93
|
+
// 2.0.0+.
|
|
94
|
+
"2.0.0": true,
|
|
95
|
+
},
|
|
96
|
+
flushMode: {
|
|
97
|
+
// Note: 1.x clients are compatible with TurnBased flushing, but here we elect to remain on Immediate flush mode
|
|
98
|
+
// as a work-around for inability to send batches larger than 1Mb. Immediate flushing keeps batches smaller as
|
|
99
|
+
// fewer messages will be included per flush.
|
|
100
|
+
"1.0.0": FlushMode.Immediate,
|
|
101
|
+
"2.0.0-defaults": FlushMode.TurnBased,
|
|
102
|
+
},
|
|
103
|
+
gcOptions: {
|
|
104
|
+
"1.0.0": {},
|
|
105
|
+
// Although sweep is supported in 2.x, it is disabled by default until minVersionForCollab>=3.0.0 to be extra safe.
|
|
106
|
+
"3.0.0": { enableGCSweep: true },
|
|
107
|
+
},
|
|
108
|
+
createBlobPayloadPending: {
|
|
109
|
+
// This feature is new and disabled by default. In the future we will enable it by default, but we have not
|
|
110
|
+
// closed on the version where that will happen yet. Probably a .10 release since blob functionality is not
|
|
111
|
+
// exposed on the `@public` API surface.
|
|
112
|
+
"1.0.0": undefined,
|
|
113
|
+
},
|
|
114
|
+
} as const satisfies ConfigMap<RuntimeOptionsAffectingDocSchema>;
|
|
115
|
+
|
|
116
|
+
const runtimeOptionsAffectingDocSchemaConfigValidationMap = {
|
|
117
|
+
enableGroupedBatching: configValueToMinVersionForCollab([
|
|
118
|
+
[false, "1.0.0"],
|
|
119
|
+
[true, "2.0.0-defaults"],
|
|
120
|
+
]),
|
|
121
|
+
compressionOptions: configValueToMinVersionForCollab([
|
|
122
|
+
[{ ...disabledCompressionConfig }, "1.0.0"],
|
|
123
|
+
[{ ...enabledCompressionConfig }, "2.0.0-defaults"],
|
|
124
|
+
]),
|
|
125
|
+
enableRuntimeIdCompressor: configValueToMinVersionForCollab([
|
|
126
|
+
[undefined, "1.0.0"],
|
|
127
|
+
["on", "2.0.0-defaults"],
|
|
128
|
+
["delayed", "2.0.0-defaults"],
|
|
129
|
+
]),
|
|
130
|
+
explicitSchemaControl: configValueToMinVersionForCollab([
|
|
131
|
+
[false, "1.0.0"],
|
|
132
|
+
[true, "2.0.0-defaults"],
|
|
133
|
+
]),
|
|
134
|
+
flushMode: configValueToMinVersionForCollab([
|
|
135
|
+
[FlushMode.Immediate, "1.0.0"],
|
|
136
|
+
[FlushMode.TurnBased, "2.0.0-defaults"],
|
|
137
|
+
]),
|
|
138
|
+
gcOptions: configValueToMinVersionForCollab([
|
|
139
|
+
[{ enableGCSweep: undefined }, "1.0.0"],
|
|
140
|
+
[{ enableGCSweep: true }, "2.0.0-defaults"],
|
|
141
|
+
]),
|
|
142
|
+
createBlobPayloadPending: configValueToMinVersionForCollab([
|
|
143
|
+
[undefined, "1.0.0"],
|
|
144
|
+
[true, "2.40.0"],
|
|
145
|
+
]),
|
|
146
|
+
} as const satisfies ConfigValidationMap<RuntimeOptionsAffectingDocSchema>;
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Returns the default RuntimeOptionsAffectingDocSchema configuration for a given minVersionForCollab.
|
|
150
|
+
*/
|
|
151
|
+
export function getMinVersionForCollabDefaults(
|
|
152
|
+
minVersionForCollab: MinimumVersionForCollab,
|
|
153
|
+
): RuntimeOptionsAffectingDocSchema {
|
|
154
|
+
return getConfigsForMinVersionForCollab(
|
|
155
|
+
minVersionForCollab,
|
|
156
|
+
runtimeOptionsAffectingDocSchemaConfigMap,
|
|
157
|
+
// This is a bad cast away from Partial that getConfigsForCompatMode provides.
|
|
158
|
+
// ConfigMap should be restructured to provide RuntimeOptionsAffectingDocSchema guarantee.
|
|
159
|
+
) as RuntimeOptionsAffectingDocSchema;
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
/**
|
|
163
|
+
* Validates if the runtime options passed in from the user are compatible with the minVersionForCollab.
|
|
164
|
+
* For example, if a user sets the `enableGroupedBatching` option to true, but the minVersionForCollab
|
|
165
|
+
* is set to "1.0.0", then we should throw a UsageError since 1.x clients do not support batching.
|
|
166
|
+
* */
|
|
167
|
+
export function validateRuntimeOptions(
|
|
168
|
+
minVersionForCollab: MinimumVersionForCollab,
|
|
169
|
+
runtimeOptions: Partial<ContainerRuntimeOptionsInternal>,
|
|
170
|
+
): void {
|
|
171
|
+
getValidationForRuntimeOptions<RuntimeOptionsAffectingDocSchema>(
|
|
172
|
+
minVersionForCollab,
|
|
173
|
+
runtimeOptions as Partial<RuntimeOptionsAffectingDocSchema>,
|
|
174
|
+
runtimeOptionsAffectingDocSchemaConfigValidationMap,
|
|
175
|
+
);
|
|
176
|
+
}
|