@fluidframework/container-runtime 2.1.0-276326 → 2.1.0-276985
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +5 -5
- package/container-runtime.test-files.tar +0 -0
- package/dist/{blobManager.d.ts → blobManager/blobManager.d.ts} +9 -29
- package/dist/blobManager/blobManager.d.ts.map +1 -0
- package/dist/{blobManager.js → blobManager/blobManager.js} +23 -83
- package/dist/blobManager/blobManager.js.map +1 -0
- package/dist/blobManager/blobManagerSnapSum.d.ts +30 -0
- package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -0
- package/dist/blobManager/blobManagerSnapSum.js +82 -0
- package/dist/blobManager/blobManagerSnapSum.js.map +1 -0
- package/dist/blobManager/index.d.ts +7 -0
- package/dist/blobManager/index.d.ts.map +1 -0
- package/dist/blobManager/index.js +16 -0
- package/dist/blobManager/index.js.map +1 -0
- package/dist/containerRuntime.d.ts +1 -5
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +68 -75
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +1 -0
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +7 -2
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/gc/garbageCollection.js +2 -2
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +17 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +43 -5
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +3 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +16 -1
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/index.d.ts +1 -1
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +1 -2
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +0 -1
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +3 -3
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/{blobManager.d.ts → blobManager/blobManager.d.ts} +9 -29
- package/lib/blobManager/blobManager.d.ts.map +1 -0
- package/lib/{blobManager.js → blobManager/blobManager.js} +21 -83
- package/lib/blobManager/blobManager.js.map +1 -0
- package/lib/blobManager/blobManagerSnapSum.d.ts +30 -0
- package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -0
- package/lib/blobManager/blobManagerSnapSum.js +75 -0
- package/lib/blobManager/blobManagerSnapSum.js.map +1 -0
- package/lib/blobManager/index.d.ts +7 -0
- package/lib/blobManager/index.d.ts.map +1 -0
- package/lib/blobManager/index.js +7 -0
- package/lib/blobManager/index.js.map +1 -0
- package/lib/containerRuntime.d.ts +1 -5
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +17 -24
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +1 -0
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +7 -2
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/gc/garbageCollection.js +2 -2
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +17 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +41 -3
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +3 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +17 -2
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/index.d.ts +1 -1
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +1 -1
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +0 -1
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +1 -1
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +21 -21
- package/src/{blobManager.ts → blobManager/blobManager.ts} +38 -123
- package/src/blobManager/blobManagerSnapSum.ts +133 -0
- package/src/blobManager/index.ts +19 -0
- package/src/containerRuntime.ts +34 -37
- package/src/dataStoreContext.ts +8 -2
- package/src/gc/garbageCollection.ts +2 -2
- package/src/index.ts +1 -1
- package/src/opLifecycle/remoteMessageProcessor.ts +63 -6
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +18 -0
- package/src/summary/index.ts +0 -1
- package/src/summary/summaryFormat.ts +1 -1
- package/dist/blobManager.d.ts.map +0 -1
- package/dist/blobManager.js.map +0 -1
- package/lib/blobManager.d.ts.map +0 -1
- package/lib/blobManager.js.map +0 -1
|
@@ -21,7 +21,6 @@ import { assert, Deferred } from "@fluidframework/core-utils/internal";
|
|
|
21
21
|
import {
|
|
22
22
|
IDocumentStorageService,
|
|
23
23
|
ICreateBlobResponse,
|
|
24
|
-
ISnapshotTree,
|
|
25
24
|
ISequencedDocumentMessage,
|
|
26
25
|
} from "@fluidframework/driver-definitions/internal";
|
|
27
26
|
import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils/internal";
|
|
@@ -32,7 +31,6 @@ import {
|
|
|
32
31
|
} from "@fluidframework/runtime-definitions/internal";
|
|
33
32
|
import {
|
|
34
33
|
FluidHandleBase,
|
|
35
|
-
SummaryTreeBuilder,
|
|
36
34
|
createResponseError,
|
|
37
35
|
generateHandleContextPath,
|
|
38
36
|
responseToException,
|
|
@@ -47,7 +45,14 @@ import {
|
|
|
47
45
|
} from "@fluidframework/telemetry-utils/internal";
|
|
48
46
|
import { v4 as uuid } from "uuid";
|
|
49
47
|
|
|
50
|
-
import { IBlobMetadata } from "
|
|
48
|
+
import { IBlobMetadata } from "../metadata.js";
|
|
49
|
+
|
|
50
|
+
import {
|
|
51
|
+
getStorageIds,
|
|
52
|
+
summarizeBlobManagerState,
|
|
53
|
+
toRedirectTable,
|
|
54
|
+
type IBlobManagerLoadInfo,
|
|
55
|
+
} from "./blobManagerSnapSum.js";
|
|
51
56
|
|
|
52
57
|
/**
|
|
53
58
|
* This class represents blob (long string)
|
|
@@ -87,16 +92,6 @@ export class BlobHandle extends FluidHandleBase<ArrayBufferLike> {
|
|
|
87
92
|
}
|
|
88
93
|
}
|
|
89
94
|
|
|
90
|
-
/**
|
|
91
|
-
* Information from a snapshot needed to load BlobManager
|
|
92
|
-
* @legacy
|
|
93
|
-
* @alpha
|
|
94
|
-
*/
|
|
95
|
-
export interface IBlobManagerLoadInfo {
|
|
96
|
-
ids?: string[];
|
|
97
|
-
redirectTable?: [string, string][];
|
|
98
|
-
}
|
|
99
|
-
|
|
100
95
|
// Restrict the IContainerRuntime interface to the subset required by BlobManager. This helps to make
|
|
101
96
|
// the contract explicit and reduces the amount of mocking required for tests.
|
|
102
97
|
export type IBlobManagerRuntime = Pick<
|
|
@@ -146,9 +141,9 @@ const stashedPendingBlobOverrides: Pick<
|
|
|
146
141
|
uploadTime: undefined,
|
|
147
142
|
} as const;
|
|
148
143
|
|
|
144
|
+
export const blobManagerBasePath = "_blobs" as const;
|
|
145
|
+
|
|
149
146
|
export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
150
|
-
public static readonly basePath = "_blobs";
|
|
151
|
-
private static readonly redirectTableBlobName = ".redirectTable";
|
|
152
147
|
private readonly mc: MonitoringContext;
|
|
153
148
|
|
|
154
149
|
/**
|
|
@@ -178,10 +173,10 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
178
173
|
private readonly routeContext: IFluidHandleContext;
|
|
179
174
|
private readonly getStorage: () => IDocumentStorageService;
|
|
180
175
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
181
|
-
// blobPath's format - `/<
|
|
176
|
+
// blobPath's format - `/<basePath>/<blobId>`.
|
|
182
177
|
private readonly blobRequested: (blobPath: string) => void;
|
|
183
178
|
// Called to check if a blob has been deleted by GC.
|
|
184
|
-
// blobPath's format - `/<
|
|
179
|
+
// blobPath's format - `/<basePath>/<blobId>`.
|
|
185
180
|
private readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
186
181
|
private readonly runtime: IBlobManagerRuntime;
|
|
187
182
|
private readonly closeContainer: (error?: ICriticalContainerError) => void;
|
|
@@ -202,10 +197,10 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
202
197
|
*/
|
|
203
198
|
sendBlobAttachOp: (localId: string, storageId?: string) => void;
|
|
204
199
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
205
|
-
// blobPath's format - `/<
|
|
200
|
+
// blobPath's format - `/<basePath>/<blobId>`.
|
|
206
201
|
readonly blobRequested: (blobPath: string) => void;
|
|
207
202
|
// Called to check if a blob has been deleted by GC.
|
|
208
|
-
// blobPath's format - `/<
|
|
203
|
+
// blobPath's format - `/<basePath>/<blobId>`.
|
|
209
204
|
readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
210
205
|
readonly runtime: IBlobManagerRuntime;
|
|
211
206
|
stashedBlobs: IPendingBlobs | undefined;
|
|
@@ -235,7 +230,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
235
230
|
namespace: "BlobManager",
|
|
236
231
|
});
|
|
237
232
|
|
|
238
|
-
this.redirectTable = this.
|
|
233
|
+
this.redirectTable = toRedirectTable(snapshot, this.mc.logger, this.runtime.attachState);
|
|
239
234
|
|
|
240
235
|
// Begin uploading stashed blobs from previous container instance
|
|
241
236
|
Object.entries(stashedBlobs ?? {}).forEach(([localId, entry]) => {
|
|
@@ -347,27 +342,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
347
342
|
);
|
|
348
343
|
}
|
|
349
344
|
|
|
350
|
-
/**
|
|
351
|
-
* Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
|
|
352
|
-
* detached or there are no (non-pending) attachment blobs in the document
|
|
353
|
-
*/
|
|
354
|
-
private get storageIds(): Set<string> {
|
|
355
|
-
const ids = new Set<string | undefined>(this.redirectTable.values());
|
|
356
|
-
|
|
357
|
-
// If we are detached, we will not have storage IDs, only undefined
|
|
358
|
-
const undefinedValueInTable = ids.delete(undefined);
|
|
359
|
-
|
|
360
|
-
// For a detached container, entries are inserted into the redirect table with an undefined storage ID.
|
|
361
|
-
// For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
|
|
362
|
-
assert(
|
|
363
|
-
!undefinedValueInTable ||
|
|
364
|
-
(this.runtime.attachState === AttachState.Detached && ids.size === 0),
|
|
365
|
-
0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */,
|
|
366
|
-
);
|
|
367
|
-
|
|
368
|
-
return ids as Set<string>;
|
|
369
|
-
}
|
|
370
|
-
|
|
371
345
|
public async getBlob(blobId: string): Promise<ArrayBufferLike> {
|
|
372
346
|
// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
|
|
373
347
|
// an error, failing the call.
|
|
@@ -419,7 +393,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
419
393
|
}
|
|
420
394
|
: undefined;
|
|
421
395
|
return new BlobHandle(
|
|
422
|
-
|
|
396
|
+
getGCNodePathFromBlobId(id),
|
|
423
397
|
this.routeContext,
|
|
424
398
|
async () => this.getBlob(id),
|
|
425
399
|
callback,
|
|
@@ -569,7 +543,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
569
543
|
if (!entry.opsent) {
|
|
570
544
|
this.sendBlobAttachOp(localId, response.id);
|
|
571
545
|
}
|
|
572
|
-
|
|
546
|
+
const storageIds = getStorageIds(this.redirectTable, this.runtime.attachState);
|
|
547
|
+
if (storageIds.has(response.id)) {
|
|
573
548
|
// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
|
|
574
549
|
// an existing blob, we don't have to wait for the op to be ack'd since this step has already
|
|
575
550
|
// happened before and so, the server won't delete it.
|
|
@@ -662,74 +637,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
662
637
|
}
|
|
663
638
|
}
|
|
664
639
|
|
|
665
|
-
/**
|
|
666
|
-
* Reads blobs needed to load BlobManager from storage.
|
|
667
|
-
* @param blobsTree - Tree containing IDs of previously attached blobs. We
|
|
668
|
-
* look for the IDs in the blob entries of the tree since the both the r11s
|
|
669
|
-
* and SPO drivers replace the attachment types returned in snapshot() with blobs.
|
|
670
|
-
*/
|
|
671
|
-
public static async load(
|
|
672
|
-
blobsTree: ISnapshotTree | undefined,
|
|
673
|
-
tryFetchBlob: (id: string) => Promise<[string, string][]>,
|
|
674
|
-
): Promise<IBlobManagerLoadInfo> {
|
|
675
|
-
if (!blobsTree) {
|
|
676
|
-
return {};
|
|
677
|
-
}
|
|
678
|
-
let redirectTable;
|
|
679
|
-
const tableId = blobsTree.blobs[this.redirectTableBlobName];
|
|
680
|
-
if (tableId) {
|
|
681
|
-
redirectTable = await tryFetchBlob(tableId);
|
|
682
|
-
}
|
|
683
|
-
const ids = Object.entries(blobsTree.blobs)
|
|
684
|
-
.filter(([k, _]) => k !== this.redirectTableBlobName)
|
|
685
|
-
.map(([_, v]) => v);
|
|
686
|
-
return { ids, redirectTable };
|
|
687
|
-
}
|
|
688
|
-
|
|
689
|
-
/**
|
|
690
|
-
* Load a set of previously attached blob IDs and redirect table from a previous snapshot.
|
|
691
|
-
*/
|
|
692
|
-
private load(snapshot: IBlobManagerLoadInfo): Map<string, string | undefined> {
|
|
693
|
-
this.mc.logger.sendTelemetryEvent({
|
|
694
|
-
eventName: "AttachmentBlobsLoaded",
|
|
695
|
-
count: snapshot.ids?.length ?? 0,
|
|
696
|
-
redirectTable: snapshot.redirectTable?.length,
|
|
697
|
-
});
|
|
698
|
-
const table = new Map<string, string | undefined>(snapshot.redirectTable);
|
|
699
|
-
if (snapshot.ids) {
|
|
700
|
-
const detached = this.runtime.attachState === AttachState.Detached;
|
|
701
|
-
// If we are detached, we don't have storage IDs yet, so set to undefined
|
|
702
|
-
// Otherwise, set identity (id -> id) entries
|
|
703
|
-
snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));
|
|
704
|
-
}
|
|
705
|
-
return table;
|
|
706
|
-
}
|
|
707
|
-
|
|
708
640
|
public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
|
|
709
|
-
|
|
710
|
-
const blobIds =
|
|
711
|
-
this.storageIds.size > 0
|
|
712
|
-
? Array.from(this.storageIds)
|
|
713
|
-
: Array.from(this.redirectTable.keys());
|
|
714
|
-
const builder = new SummaryTreeBuilder();
|
|
715
|
-
blobIds.forEach((blobId) => {
|
|
716
|
-
builder.addAttachment(blobId);
|
|
717
|
-
});
|
|
718
|
-
|
|
719
|
-
// Any non-identity entries in the table need to be saved in the summary
|
|
720
|
-
if (this.redirectTable.size > blobIds.length) {
|
|
721
|
-
builder.addBlob(
|
|
722
|
-
BlobManager.redirectTableBlobName,
|
|
723
|
-
// filter out identity entries
|
|
724
|
-
JSON.stringify(
|
|
725
|
-
Array.from(this.redirectTable.entries()).filter(
|
|
726
|
-
([localId, storageId]) => localId !== storageId,
|
|
727
|
-
),
|
|
728
|
-
),
|
|
729
|
-
);
|
|
730
|
-
}
|
|
731
|
-
|
|
732
|
-
return builder.getSummaryTree();
|
|
641
|
+
return summarizeBlobManagerState(this.redirectTable, this.runtime.attachState);
|
|
733
642
|
}
|
|
734
643
|
|
|
735
644
|
/**
|
|
@@ -766,7 +675,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
766
675
|
|
|
767
676
|
/**
|
|
768
677
|
* Delete blobs with the given routes from the redirect table.
|
|
769
|
-
* The routes are GC nodes paths of format -`/<
|
|
678
|
+
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<blobId>`. The blob ids are all local ids.
|
|
770
679
|
* Deleting the blobs involves 2 steps:
|
|
771
680
|
* 1. The redirect table entry for the local ids are deleted.
|
|
772
681
|
* 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
|
|
@@ -839,7 +748,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
839
748
|
this.mc.logger.sendErrorEvent(
|
|
840
749
|
{
|
|
841
750
|
eventName: "GC_Deleted_Blob_Requested",
|
|
842
|
-
pkg:
|
|
751
|
+
pkg: blobManagerBasePath,
|
|
843
752
|
},
|
|
844
753
|
error,
|
|
845
754
|
);
|
|
@@ -936,22 +845,28 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
|
936
845
|
}
|
|
937
846
|
|
|
938
847
|
/**
|
|
939
|
-
* For a blobId, returns its path in GC's graph. The node path is of the format `/<
|
|
848
|
+
* For a blobId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<blobId>`.
|
|
940
849
|
* This path must match the path of the blob handle returned by the createBlob API because blobs are marked
|
|
941
850
|
* referenced by storing these handles in a referenced DDS.
|
|
942
851
|
*/
|
|
943
|
-
|
|
944
|
-
return `/${BlobManager.basePath}/${blobId}`;
|
|
945
|
-
}
|
|
852
|
+
const getGCNodePathFromBlobId = (blobId: string) => `/${blobManagerBasePath}/${blobId}`;
|
|
946
853
|
|
|
947
854
|
/**
|
|
948
|
-
* For a given GC node path, return the blobId. The node path is of the format `/<
|
|
855
|
+
* For a given GC node path, return the blobId. The node path is of the format `/<basePath>/<blobId>`.
|
|
949
856
|
*/
|
|
950
|
-
|
|
857
|
+
const getBlobIdFromGCNodePath = (nodePath: string) => {
|
|
951
858
|
const pathParts = nodePath.split("/");
|
|
952
|
-
assert(
|
|
953
|
-
pathParts.length === 3 && pathParts[1] === BlobManager.basePath,
|
|
954
|
-
0x5bd /* Invalid blob node path */,
|
|
955
|
-
);
|
|
859
|
+
assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
|
|
956
860
|
return pathParts[2];
|
|
957
|
-
}
|
|
861
|
+
};
|
|
862
|
+
|
|
863
|
+
/**
|
|
864
|
+
* Returns whether a given path is for attachment blobs that are in the format - "/blobManagerBasePath/...".
|
|
865
|
+
*/
|
|
866
|
+
export const isBlobPath = (path: string): path is `/${typeof blobManagerBasePath}/${string}` =>
|
|
867
|
+
areBlobPathParts(path.split("/"));
|
|
868
|
+
|
|
869
|
+
export const areBlobPathParts = (
|
|
870
|
+
pathParts: string[],
|
|
871
|
+
): pathParts is ["", typeof blobManagerBasePath, string] =>
|
|
872
|
+
pathParts.length === 3 && pathParts[1] === blobManagerBasePath;
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import {
|
|
7
|
+
AttachState,
|
|
8
|
+
type IContainerContext,
|
|
9
|
+
} from "@fluidframework/container-definitions/internal";
|
|
10
|
+
import { assert } from "@fluidframework/core-utils/internal";
|
|
11
|
+
import { readAndParse } from "@fluidframework/driver-utils/internal";
|
|
12
|
+
import type { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions/internal";
|
|
13
|
+
import { SummaryTreeBuilder } from "@fluidframework/runtime-utils/internal";
|
|
14
|
+
import type { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/internal";
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Information from a snapshot needed to load BlobManager
|
|
18
|
+
* @legacy
|
|
19
|
+
* @alpha
|
|
20
|
+
*/
|
|
21
|
+
export interface IBlobManagerLoadInfo {
|
|
22
|
+
ids?: string[];
|
|
23
|
+
redirectTable?: [string, string][];
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
export const redirectTableBlobName = ".redirectTable";
|
|
27
|
+
|
|
28
|
+
/**
|
|
29
|
+
* @internal
|
|
30
|
+
*/
|
|
31
|
+
export const blobsTreeName = ".blobs";
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Reads blobs needed to load BlobManager from storage.
|
|
35
|
+
*
|
|
36
|
+
*/
|
|
37
|
+
export const loadBlobManagerLoadInfo = async (
|
|
38
|
+
context: Pick<IContainerContext, "baseSnapshot" | "storage" | "attachState">,
|
|
39
|
+
): Promise<IBlobManagerLoadInfo> => loadV1(context);
|
|
40
|
+
|
|
41
|
+
const loadV1 = async (
|
|
42
|
+
context: Pick<IContainerContext, "baseSnapshot" | "storage" | "attachState">,
|
|
43
|
+
): Promise<IBlobManagerLoadInfo> => {
|
|
44
|
+
const blobsTree = context.baseSnapshot?.trees[blobsTreeName];
|
|
45
|
+
|
|
46
|
+
if (!blobsTree) {
|
|
47
|
+
return {};
|
|
48
|
+
}
|
|
49
|
+
let redirectTableEntries: [string, string][] = [];
|
|
50
|
+
const tableId = blobsTree.blobs[redirectTableBlobName];
|
|
51
|
+
if (tableId) {
|
|
52
|
+
redirectTableEntries = await readAndParse(context.storage, tableId);
|
|
53
|
+
}
|
|
54
|
+
const ids = Object.entries(blobsTree.blobs)
|
|
55
|
+
.filter(([k, _]) => k !== redirectTableBlobName)
|
|
56
|
+
.map(([_, v]) => v);
|
|
57
|
+
|
|
58
|
+
return { ids, redirectTable: redirectTableEntries };
|
|
59
|
+
};
|
|
60
|
+
|
|
61
|
+
export const toRedirectTable = (
|
|
62
|
+
snapshot: IBlobManagerLoadInfo,
|
|
63
|
+
logger: ITelemetryLoggerExt,
|
|
64
|
+
attachState: AttachState,
|
|
65
|
+
): Map<string, string | undefined> => {
|
|
66
|
+
logger.sendTelemetryEvent({
|
|
67
|
+
eventName: "AttachmentBlobsLoaded",
|
|
68
|
+
count: snapshot.ids?.length ?? 0,
|
|
69
|
+
redirectTable: snapshot.redirectTable?.length,
|
|
70
|
+
});
|
|
71
|
+
const redirectTable = new Map<string, string | undefined>(snapshot.redirectTable);
|
|
72
|
+
const detached = attachState !== AttachState.Attached;
|
|
73
|
+
if (snapshot.ids) {
|
|
74
|
+
// If we are detached, we don't have storage IDs yet, so set to undefined
|
|
75
|
+
// Otherwise, set identity (id -> id) entries.
|
|
76
|
+
snapshot.ids.forEach((entry) => redirectTable.set(entry, detached ? undefined : entry));
|
|
77
|
+
}
|
|
78
|
+
return redirectTable;
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
export const summarizeBlobManagerState = (
|
|
82
|
+
redirectTable: Map<string, string | undefined>,
|
|
83
|
+
attachState: AttachState,
|
|
84
|
+
): ISummaryTreeWithStats => summarizeV1(redirectTable, attachState);
|
|
85
|
+
|
|
86
|
+
const summarizeV1 = (
|
|
87
|
+
redirectTable: Map<string, string | undefined>,
|
|
88
|
+
attachState: AttachState,
|
|
89
|
+
): ISummaryTreeWithStats => {
|
|
90
|
+
const storageIds = getStorageIds(redirectTable, attachState);
|
|
91
|
+
|
|
92
|
+
// if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
|
|
93
|
+
const blobIds =
|
|
94
|
+
storageIds.size > 0 ? Array.from(storageIds) : Array.from(redirectTable.keys());
|
|
95
|
+
const builder = new SummaryTreeBuilder();
|
|
96
|
+
blobIds.forEach((blobId) => {
|
|
97
|
+
builder.addAttachment(blobId);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
// Any non-identity entries in the table need to be saved in the summary
|
|
101
|
+
if (redirectTable.size > blobIds.length) {
|
|
102
|
+
builder.addBlob(
|
|
103
|
+
redirectTableBlobName,
|
|
104
|
+
// filter out identity entries
|
|
105
|
+
JSON.stringify(
|
|
106
|
+
Array.from(redirectTable.entries()).filter(
|
|
107
|
+
([localId, storageId]) => localId !== storageId,
|
|
108
|
+
),
|
|
109
|
+
),
|
|
110
|
+
);
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
return builder.getSummaryTree();
|
|
114
|
+
};
|
|
115
|
+
|
|
116
|
+
export const getStorageIds = (
|
|
117
|
+
redirectTable: Map<string, string | undefined>,
|
|
118
|
+
attachState: AttachState,
|
|
119
|
+
) => {
|
|
120
|
+
const ids = new Set<string | undefined>(redirectTable.values());
|
|
121
|
+
|
|
122
|
+
// If we are detached, we will not have storage IDs, only undefined
|
|
123
|
+
const undefinedValueInTable = ids.delete(undefined);
|
|
124
|
+
|
|
125
|
+
// For a detached container, entries are inserted into the redirect table with an undefined storage ID.
|
|
126
|
+
// For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
|
|
127
|
+
assert(
|
|
128
|
+
!undefinedValueInTable || (attachState === AttachState.Detached && ids.size === 0),
|
|
129
|
+
0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */,
|
|
130
|
+
);
|
|
131
|
+
|
|
132
|
+
return ids as Set<string>;
|
|
133
|
+
};
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
export {
|
|
7
|
+
BlobManager,
|
|
8
|
+
IPendingBlobs,
|
|
9
|
+
IBlobManagerRuntime,
|
|
10
|
+
IBlobManagerEvents,
|
|
11
|
+
blobManagerBasePath,
|
|
12
|
+
isBlobPath,
|
|
13
|
+
} from "./blobManager.js";
|
|
14
|
+
export {
|
|
15
|
+
loadBlobManagerLoadInfo,
|
|
16
|
+
IBlobManagerLoadInfo,
|
|
17
|
+
blobsTreeName,
|
|
18
|
+
redirectTableBlobName,
|
|
19
|
+
} from "./blobManagerSnapSum.js";
|
package/src/containerRuntime.ts
CHANGED
|
@@ -126,7 +126,15 @@ import {
|
|
|
126
126
|
import { v4 as uuid } from "uuid";
|
|
127
127
|
|
|
128
128
|
import { BindBatchTracker } from "./batchTracker.js";
|
|
129
|
-
import {
|
|
129
|
+
import {
|
|
130
|
+
BlobManager,
|
|
131
|
+
IPendingBlobs,
|
|
132
|
+
blobManagerBasePath,
|
|
133
|
+
blobsTreeName,
|
|
134
|
+
isBlobPath,
|
|
135
|
+
loadBlobManagerLoadInfo,
|
|
136
|
+
type IBlobManagerLoadInfo,
|
|
137
|
+
} from "./blobManager/index.js";
|
|
130
138
|
import {
|
|
131
139
|
ChannelCollection,
|
|
132
140
|
getSummaryForDatastores,
|
|
@@ -211,7 +219,6 @@ import {
|
|
|
211
219
|
SummaryCollection,
|
|
212
220
|
SummaryManager,
|
|
213
221
|
aliasBlobName,
|
|
214
|
-
blobsTreeName,
|
|
215
222
|
chunksBlobName,
|
|
216
223
|
createRootSummarizerNodeWithGC,
|
|
217
224
|
electedSummarizerBlobName,
|
|
@@ -677,23 +684,26 @@ export const makeLegacySendBatchFn =
|
|
|
677
684
|
};
|
|
678
685
|
|
|
679
686
|
/** Helper type for type constraints passed through several functions.
|
|
687
|
+
* local - Did this client send the op?
|
|
688
|
+
* savedOp - Is this op being replayed after being serialized (having been sequenced previously)
|
|
689
|
+
* batchStartCsn - The clientSequenceNumber given on submit to the start of this batch
|
|
680
690
|
* message - The unpacked message. Likely a TypedContainerRuntimeMessage, but could also be a system op
|
|
681
691
|
* modernRuntimeMessage - Does this appear like a current TypedContainerRuntimeMessage?
|
|
682
|
-
* local - Did this client send the op?
|
|
683
692
|
*/
|
|
684
|
-
type MessageWithContext =
|
|
693
|
+
type MessageWithContext = {
|
|
694
|
+
local: boolean;
|
|
695
|
+
savedOp?: boolean;
|
|
696
|
+
batchStartCsn: number;
|
|
697
|
+
} & (
|
|
685
698
|
| {
|
|
686
699
|
message: InboundSequencedContainerRuntimeMessage;
|
|
687
700
|
modernRuntimeMessage: true;
|
|
688
|
-
local: boolean;
|
|
689
|
-
savedOp?: boolean;
|
|
690
701
|
}
|
|
691
702
|
| {
|
|
692
703
|
message: InboundSequencedContainerRuntimeMessageOrSystemMessage;
|
|
693
704
|
modernRuntimeMessage: false;
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
};
|
|
705
|
+
}
|
|
706
|
+
);
|
|
697
707
|
|
|
698
708
|
const summarizerRequestUrl = "_summarizer";
|
|
699
709
|
|
|
@@ -857,18 +867,7 @@ export class ContainerRuntime
|
|
|
857
867
|
]);
|
|
858
868
|
|
|
859
869
|
// read snapshot blobs needed for BlobManager to load
|
|
860
|
-
const blobManagerSnapshot = await
|
|
861
|
-
context.baseSnapshot?.trees[blobsTreeName],
|
|
862
|
-
async (id) => {
|
|
863
|
-
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
864
|
-
// So once we release 0.40 container-defn package we can remove this check.
|
|
865
|
-
assert(
|
|
866
|
-
context.storage !== undefined,
|
|
867
|
-
0x256 /* "storage undefined in attached container" */,
|
|
868
|
-
);
|
|
869
|
-
return readAndParse(context.storage, id);
|
|
870
|
-
},
|
|
871
|
-
);
|
|
870
|
+
const blobManagerSnapshot = await loadBlobManagerLoadInfo(context);
|
|
872
871
|
|
|
873
872
|
const messageAtLastSummary = lastMessageFromMetadata(metadata);
|
|
874
873
|
|
|
@@ -2221,7 +2220,7 @@ export class ContainerRuntime
|
|
|
2221
2220
|
return this.resolveHandle(requestParser.createSubRequest(1));
|
|
2222
2221
|
}
|
|
2223
2222
|
|
|
2224
|
-
if (id ===
|
|
2223
|
+
if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
|
|
2225
2224
|
const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
|
|
2226
2225
|
return blob
|
|
2227
2226
|
? {
|
|
@@ -2619,7 +2618,13 @@ export class ContainerRuntime
|
|
|
2619
2618
|
// but will not modify the contents object (likely it will replace it on the message).
|
|
2620
2619
|
const messageCopy = { ...messageArg };
|
|
2621
2620
|
const savedOp = (messageCopy.metadata as ISavedOpMetadata)?.savedOp;
|
|
2622
|
-
|
|
2621
|
+
const processResult = this.remoteMessageProcessor.process(messageCopy);
|
|
2622
|
+
if (processResult === undefined) {
|
|
2623
|
+
// This means the incoming message is an incomplete part of a message or batch
|
|
2624
|
+
// and we need to process more messages before the rest of the system can understand it.
|
|
2625
|
+
return;
|
|
2626
|
+
}
|
|
2627
|
+
for (const message of processResult.messages) {
|
|
2623
2628
|
const msg: MessageWithContext = modernRuntimeMessage
|
|
2624
2629
|
? {
|
|
2625
2630
|
// Cast it since we expect it to be this based on modernRuntimeMessage computation above.
|
|
@@ -2629,12 +2634,14 @@ export class ContainerRuntime
|
|
|
2629
2634
|
message: message as InboundSequencedContainerRuntimeMessage,
|
|
2630
2635
|
local,
|
|
2631
2636
|
modernRuntimeMessage,
|
|
2637
|
+
batchStartCsn: processResult.batchStartCsn,
|
|
2632
2638
|
}
|
|
2633
2639
|
: // Unrecognized message will be ignored.
|
|
2634
2640
|
{
|
|
2635
2641
|
message,
|
|
2636
2642
|
local,
|
|
2637
2643
|
modernRuntimeMessage,
|
|
2644
|
+
batchStartCsn: processResult.batchStartCsn,
|
|
2638
2645
|
};
|
|
2639
2646
|
msg.savedOp = savedOp;
|
|
2640
2647
|
|
|
@@ -2682,6 +2689,7 @@ export class ContainerRuntime
|
|
|
2682
2689
|
if (local && messageWithContext.modernRuntimeMessage) {
|
|
2683
2690
|
localOpMetadata = this.pendingStateManager.processPendingLocalMessage(
|
|
2684
2691
|
messageWithContext.message,
|
|
2692
|
+
messageWithContext.batchStartCsn,
|
|
2685
2693
|
);
|
|
2686
2694
|
}
|
|
2687
2695
|
|
|
@@ -3360,7 +3368,7 @@ export class ContainerRuntime
|
|
|
3360
3368
|
* blob manager.
|
|
3361
3369
|
*/
|
|
3362
3370
|
public getNodeType(nodePath: string): GCNodeType {
|
|
3363
|
-
if (
|
|
3371
|
+
if (isBlobPath(nodePath)) {
|
|
3364
3372
|
return GCNodeType.Blob;
|
|
3365
3373
|
}
|
|
3366
3374
|
return this.channelCollection.getGCNodeType(nodePath) ?? GCNodeType.Other;
|
|
@@ -3379,7 +3387,7 @@ export class ContainerRuntime
|
|
|
3379
3387
|
|
|
3380
3388
|
switch (this.getNodeType(nodePath)) {
|
|
3381
3389
|
case GCNodeType.Blob:
|
|
3382
|
-
return [
|
|
3390
|
+
return [blobManagerBasePath];
|
|
3383
3391
|
case GCNodeType.DataStore:
|
|
3384
3392
|
case GCNodeType.SubDataStore:
|
|
3385
3393
|
return this.channelCollection.getDataStorePackagePath(nodePath);
|
|
@@ -3388,17 +3396,6 @@ export class ContainerRuntime
|
|
|
3388
3396
|
}
|
|
3389
3397
|
}
|
|
3390
3398
|
|
|
3391
|
-
/**
|
|
3392
|
-
* Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
|
|
3393
|
-
*/
|
|
3394
|
-
private isBlobPath(path: string): boolean {
|
|
3395
|
-
const pathParts = path.split("/");
|
|
3396
|
-
if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
|
|
3397
|
-
return false;
|
|
3398
|
-
}
|
|
3399
|
-
return true;
|
|
3400
|
-
}
|
|
3401
|
-
|
|
3402
3399
|
/**
|
|
3403
3400
|
* From a given list of routes, separate and return routes that belong to blob manager and data stores.
|
|
3404
3401
|
* @param routes - A list of routes that can belong to data stores or blob manager.
|
|
@@ -3409,7 +3406,7 @@ export class ContainerRuntime
|
|
|
3409
3406
|
const blobManagerRoutes: string[] = [];
|
|
3410
3407
|
const dataStoreRoutes: string[] = [];
|
|
3411
3408
|
for (const route of routes) {
|
|
3412
|
-
if (
|
|
3409
|
+
if (isBlobPath(route)) {
|
|
3413
3410
|
blobManagerRoutes.push(route);
|
|
3414
3411
|
} else {
|
|
3415
3412
|
dataStoreRoutes.push(route);
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -1058,6 +1058,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
1058
1058
|
private snapshotFetchRequired: boolean | undefined;
|
|
1059
1059
|
private readonly runtime: IContainerRuntimeBase;
|
|
1060
1060
|
private readonly blobContents: Map<string, ArrayBuffer> | undefined;
|
|
1061
|
+
private readonly isSnapshotInISnapshotFormat: boolean | undefined;
|
|
1061
1062
|
|
|
1062
1063
|
constructor(props: IRemoteFluidDataStoreContextProps) {
|
|
1063
1064
|
super(props, true /* existing */, false /* isLocalDataStore */, () => {
|
|
@@ -1068,8 +1069,10 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
1068
1069
|
if (isInstanceOfISnapshot(props.snapshot)) {
|
|
1069
1070
|
this.blobContents = props.snapshot.blobContents;
|
|
1070
1071
|
this._baseSnapshot = props.snapshot.snapshotTree;
|
|
1072
|
+
this.isSnapshotInISnapshotFormat = true;
|
|
1071
1073
|
} else {
|
|
1072
1074
|
this._baseSnapshot = props.snapshot;
|
|
1075
|
+
this.isSnapshotInISnapshotFormat = false;
|
|
1073
1076
|
}
|
|
1074
1077
|
if (this._baseSnapshot !== undefined) {
|
|
1075
1078
|
this.summarizerNode.updateBaseSummaryState(this._baseSnapshot);
|
|
@@ -1091,10 +1094,13 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
1091
1094
|
private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
|
|
1092
1095
|
// Sequence number of the snapshot.
|
|
1093
1096
|
let sequenceNumber: number | undefined;
|
|
1094
|
-
// Check whether we need to fetch the snapshot first to load.
|
|
1097
|
+
// Check whether we need to fetch the snapshot first to load. The snapshot should be in new format to see
|
|
1098
|
+
// whether we want to evaluate to fetch snapshot or not for loadingGroupId. Otherwise, the snapshot
|
|
1099
|
+
// will contain all the blobs.
|
|
1095
1100
|
if (
|
|
1096
1101
|
this.snapshotFetchRequired === undefined &&
|
|
1097
|
-
this._baseSnapshot?.groupId !== undefined
|
|
1102
|
+
this._baseSnapshot?.groupId !== undefined &&
|
|
1103
|
+
this.isSnapshotInISnapshotFormat
|
|
1098
1104
|
) {
|
|
1099
1105
|
assert(
|
|
1100
1106
|
this.blobContents !== undefined,
|
|
@@ -26,7 +26,7 @@ import {
|
|
|
26
26
|
tagCodeArtifacts,
|
|
27
27
|
} from "@fluidframework/telemetry-utils/internal";
|
|
28
28
|
|
|
29
|
-
import {
|
|
29
|
+
import { blobManagerBasePath } from "../blobManager/index.js";
|
|
30
30
|
import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
|
|
31
31
|
import { ClientSessionExpiredError } from "../error.js";
|
|
32
32
|
import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
|
|
@@ -1272,7 +1272,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1272
1272
|
// be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
|
|
1273
1273
|
const getDeletedNodeType = (nodeId: string): GCNodeType => {
|
|
1274
1274
|
const pathParts = nodeId.split("/");
|
|
1275
|
-
if (pathParts[1] ===
|
|
1275
|
+
if (pathParts[1] === blobManagerBasePath) {
|
|
1276
1276
|
return GCNodeType.Blob;
|
|
1277
1277
|
}
|
|
1278
1278
|
if (pathParts.length === 2) {
|
package/src/index.ts
CHANGED
|
@@ -30,7 +30,7 @@ export {
|
|
|
30
30
|
RecentlyAddedContainerRuntimeMessageDetails,
|
|
31
31
|
UnknownContainerRuntimeMessage,
|
|
32
32
|
} from "./messageTypes.js";
|
|
33
|
-
export { IBlobManagerLoadInfo } from "./blobManager.js";
|
|
33
|
+
export { IBlobManagerLoadInfo } from "./blobManager/index.js";
|
|
34
34
|
export { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
|
|
35
35
|
export {
|
|
36
36
|
detectOutboundReferences,
|