@fluidframework/container-runtime 2.60.0 → 2.61.0-355516
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mocharc.cjs +1 -2
- package/api-report/container-runtime.legacy.beta.api.md +2 -1
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +33 -16
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +126 -106
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.js +30 -33
- package/dist/blobManager/blobManagerSnapSum.js.map +1 -1
- package/dist/channelCollection.d.ts +6 -2
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +1 -0
- package/dist/channelCollection.js.map +1 -1
- package/dist/containerCompatibility.d.ts +18 -0
- package/dist/containerCompatibility.d.ts.map +1 -1
- package/dist/containerCompatibility.js +23 -1
- package/dist/containerCompatibility.js.map +1 -1
- package/dist/containerRuntime.d.ts +15 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +75 -52
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +5 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/legacy.d.ts +2 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +33 -16
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +126 -106
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.js +26 -29
- package/lib/blobManager/blobManagerSnapSum.js.map +1 -1
- package/lib/channelCollection.d.ts +6 -2
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +1 -0
- package/lib/channelCollection.js.map +1 -1
- package/lib/containerCompatibility.d.ts +18 -0
- package/lib/containerCompatibility.d.ts.map +1 -1
- package/lib/containerCompatibility.js +22 -0
- package/lib/containerCompatibility.js.map +1 -1
- package/lib/containerRuntime.d.ts +15 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +26 -3
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +5 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/legacy.d.ts +2 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/tsdoc-metadata.json +1 -1
- package/package.json +27 -27
- package/src/blobManager/blobManager.ts +138 -123
- package/src/blobManager/blobManagerSnapSum.ts +31 -53
- package/src/channelCollection.ts +9 -1
- package/src/containerCompatibility.ts +56 -0
- package/src/containerRuntime.ts +35 -4
- package/src/dataStoreContext.ts +7 -0
- package/src/packageVersion.ts +1 -1
package/.mocharc.cjs
CHANGED
|
@@ -7,8 +7,7 @@
|
|
|
7
7
|
|
|
8
8
|
const getFluidTestMochaConfig = require("@fluid-internal/mocha-test-setup/mocharc-common");
|
|
9
9
|
|
|
10
|
-
const
|
|
11
|
-
const config = getFluidTestMochaConfig(packageDir);
|
|
10
|
+
const config = getFluidTestMochaConfig(__dirname);
|
|
12
11
|
// TODO: figure out why this package needs the --exit flag, tests might not be cleaning up correctly after themselves
|
|
13
12
|
// AB#7856
|
|
14
13
|
config.exit = true;
|
|
@@ -346,7 +346,8 @@ export interface LoadContainerRuntimeParams {
|
|
|
346
346
|
runtimeOptions?: IContainerRuntimeOptions;
|
|
347
347
|
}
|
|
348
348
|
|
|
349
|
-
|
|
349
|
+
// @beta @legacy
|
|
350
|
+
export type MinimumVersionForCollab = `${1 | 2}.${bigint}.${bigint}` | `${1 | 2}.${bigint}.${bigint}-${string}`;
|
|
350
351
|
|
|
351
352
|
// @beta @deprecated @legacy (undocumented)
|
|
352
353
|
export type OmitAttributesVersions<T> = Omit<T, "snapshotFormatVersion" | "summaryFormatVersion">;
|
|
Binary file
|
|
@@ -39,7 +39,7 @@ export declare class BlobHandle extends FluidHandleBase<ArrayBufferLike> impleme
|
|
|
39
39
|
readonly notifyFailed: (error: unknown) => void;
|
|
40
40
|
attachGraph(): void;
|
|
41
41
|
}
|
|
42
|
-
export type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "
|
|
42
|
+
export type IBlobManagerRuntime = Pick<IContainerRuntime, "attachState" | "baseLogger" | "disposed"> & IEventProvider<IContainerRuntimeEvents>;
|
|
43
43
|
export interface IPendingBlobs {
|
|
44
44
|
[localId: string]: {
|
|
45
45
|
blob: string;
|
|
@@ -59,11 +59,10 @@ export declare class BlobManager {
|
|
|
59
59
|
get events(): Listenable<IBlobManagerEvents>;
|
|
60
60
|
private readonly internalEvents;
|
|
61
61
|
/**
|
|
62
|
-
* Map of local IDs to storage IDs.
|
|
63
|
-
* be a key in this map. Blobs created while the container is detached are
|
|
64
|
-
* gives
|
|
65
|
-
*
|
|
66
|
-
* that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
|
|
62
|
+
* Map of local IDs to storage IDs. Also includes identity mappings of storage ID to storage ID for all known
|
|
63
|
+
* storage IDs. All requested IDs must be a key in this map. Blobs created while the container is detached are
|
|
64
|
+
* stored in IDetachedBlobStorage which gives pseudo storage IDs; the real storage IDs are filled in at attach
|
|
65
|
+
* time via setRedirectTable().
|
|
67
66
|
*/
|
|
68
67
|
private readonly redirectTable;
|
|
69
68
|
/**
|
|
@@ -82,7 +81,7 @@ export declare class BlobManager {
|
|
|
82
81
|
private readonly blobRequested;
|
|
83
82
|
private readonly isBlobDeleted;
|
|
84
83
|
private readonly runtime;
|
|
85
|
-
private readonly
|
|
84
|
+
private readonly localIdGenerator;
|
|
86
85
|
private readonly createBlobPayloadPending;
|
|
87
86
|
constructor(props: {
|
|
88
87
|
readonly routeContext: IFluidHandleContext;
|
|
@@ -103,20 +102,31 @@ export declare class BlobManager {
|
|
|
103
102
|
readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
104
103
|
readonly runtime: IBlobManagerRuntime;
|
|
105
104
|
stashedBlobs: IPendingBlobs | undefined;
|
|
106
|
-
readonly
|
|
105
|
+
readonly localIdGenerator?: (() => string) | undefined;
|
|
107
106
|
readonly createBlobPayloadPending: boolean;
|
|
108
107
|
});
|
|
109
108
|
get allBlobsAttached(): boolean;
|
|
110
109
|
get hasPendingBlobs(): boolean;
|
|
111
110
|
private createAbortError;
|
|
112
|
-
hasBlob(
|
|
111
|
+
hasBlob(localId: string): boolean;
|
|
112
|
+
/**
|
|
113
|
+
* Lookup the blob storage ID for a given local blob id.
|
|
114
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
115
|
+
* @returns The storage ID if found and the blob is not pending, undefined otherwise.
|
|
116
|
+
* @remarks
|
|
117
|
+
* For blobs with pending payloads (localId exists but upload hasn't finished), this is expected to return undefined.
|
|
118
|
+
* Consumers should use the observability APIs on the handle (handle.payloadState, payloadShared event)
|
|
119
|
+
* to understand/wait for storage ID availability.
|
|
120
|
+
* Similarly, when the runtime is detached, this will return undefined as no blobs have been uploaded to storage.
|
|
121
|
+
*/
|
|
122
|
+
lookupTemporaryBlobStorageId(localId: string): string | undefined;
|
|
113
123
|
/**
|
|
114
124
|
* Retrieve the blob with the given local blob id.
|
|
115
|
-
* @param
|
|
125
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
116
126
|
* @param payloadPending - Whether we suspect the payload may be pending and not available yet.
|
|
117
127
|
* @returns A promise which resolves to the blob contents
|
|
118
128
|
*/
|
|
119
|
-
getBlob(
|
|
129
|
+
getBlob(localId: string, payloadPending: boolean): Promise<ArrayBufferLike>;
|
|
120
130
|
private getBlobHandle;
|
|
121
131
|
private createBlobDetached;
|
|
122
132
|
createBlob(blob: ArrayBufferLike, signal?: AbortSignal): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>>;
|
|
@@ -163,16 +173,17 @@ export declare class BlobManager {
|
|
|
163
173
|
* Delete blobs with the given routes from the redirect table.
|
|
164
174
|
*
|
|
165
175
|
* @remarks
|
|
166
|
-
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<
|
|
176
|
+
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
|
|
167
177
|
* Deleting the blobs involves 2 steps:
|
|
168
178
|
*
|
|
169
179
|
* 1. The redirect table entry for the local ids are deleted.
|
|
170
180
|
*
|
|
171
|
-
* 2. If the storage ids corresponding to the deleted local ids are not
|
|
172
|
-
*
|
|
181
|
+
* 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
|
|
182
|
+
* identity mappings in the redirect table are deleted as well.
|
|
173
183
|
*
|
|
174
184
|
* Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
|
|
175
|
-
* will
|
|
185
|
+
* will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
|
|
186
|
+
* some time in the future.
|
|
176
187
|
*/
|
|
177
188
|
private deleteBlobsFromRedirectTable;
|
|
178
189
|
/**
|
|
@@ -180,7 +191,13 @@ export declare class BlobManager {
|
|
|
180
191
|
* log an error and throw if necessary.
|
|
181
192
|
*/
|
|
182
193
|
private verifyBlobNotDeleted;
|
|
183
|
-
|
|
194
|
+
/**
|
|
195
|
+
* Called in detached state just prior to attaching, this will update the redirect table by
|
|
196
|
+
* converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
|
|
197
|
+
* The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
|
|
198
|
+
* @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
|
|
199
|
+
*/
|
|
200
|
+
readonly patchRedirectTable: (detachedStorageTable: Map<string, string>) => void;
|
|
184
201
|
/**
|
|
185
202
|
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
186
203
|
* pending upload and/or their BlobAttach op, which can be given to a new BlobManager to
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../../src/blobManager/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAEN,KAAK,wBAAwB,EAC7B,MAAM,gDAAgD,CAAC;AACxD,OAAO,KAAK,EACX,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,wDAAwD,CAAC;AAChE,OAAO,KAAK,EAEX,cAAc,EACd,mBAAmB,EACnB,kCAAkC,EAClC,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,MAAM,0CAA0C,CAAC;AAGlD,OAAO,KAAK,EACX,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACN,eAAe,EAIf,MAAM,wCAAwC,CAAC;AAYhD,OAAO,EAIN,KAAK,oBAAoB,EACzB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,qBAAa,UACZ,SAAQ,eAAe,CAAC,eAAe,CACvC,YACC,iBAAiB,CAAC,eAAe,CAAC,EAClC,kCAAkC,CAAC,eAAe,CAAC;aAgCnC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC;aAC1B,cAAc,EAAE,OAAO;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAlChC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,OAAO,CAAC,OAAO,CAEF;IACb,IAAW,MAAM,IAAI,UAAU,CAAC,uBAAuB,CAAC,CAEvD;IAED,OAAO,CAAC,MAAM,CAA2B;IACzC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED;;;OAGG;IACH,OAAO,CAAC,kBAAkB,CAAU;IACpC,IAAW,iBAAiB,IAAI,OAAO,CAEtC;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,EAC1B,cAAc,EAAE,OAAO,EACtB,aAAa,CAAC,SAAQ,IAAI,aAAA;IAM5C,SAAgB,YAAY,QAAO,IAAI,CAGrC;IAEF,SAAgB,YAAY,UAAW,OAAO,KAAG,IAAI,CAGnD;IAEK,WAAW,IAAI,IAAI;CAM1B;AAID,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,
|
|
1
|
+
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../../src/blobManager/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAEN,KAAK,wBAAwB,EAC7B,MAAM,gDAAgD,CAAC;AACxD,OAAO,KAAK,EACX,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,wDAAwD,CAAC;AAChE,OAAO,KAAK,EAEX,cAAc,EACd,mBAAmB,EACnB,kCAAkC,EAClC,iBAAiB,EACjB,uBAAuB,EACvB,UAAU,EACV,YAAY,EACZ,MAAM,0CAA0C,CAAC;AAGlD,OAAO,KAAK,EACX,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,yBAAyB,EACzB,MAAM,8CAA8C,CAAC;AACtD,OAAO,EACN,eAAe,EAIf,MAAM,wCAAwC,CAAC;AAYhD,OAAO,EAIN,KAAK,oBAAoB,EACzB,MAAM,yBAAyB,CAAC;AAEjC;;;;;;GAMG;AACH,qBAAa,UACZ,SAAQ,eAAe,CAAC,eAAe,CACvC,YACC,iBAAiB,CAAC,eAAe,CAAC,EAClC,kCAAkC,CAAC,eAAe,CAAC;aAgCnC,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC;aAC1B,cAAc,EAAE,OAAO;IACvC,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAC;IAlChC,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,OAAO,CAAC,OAAO,CAEF;IACb,IAAW,MAAM,IAAI,UAAU,CAAC,uBAAuB,CAAC,CAEvD;IAED,OAAO,CAAC,MAAM,CAA2B;IACzC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED;;;OAGG;IACH,OAAO,CAAC,kBAAkB,CAAU;IACpC,IAAW,iBAAiB,IAAI,OAAO,CAEtC;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,eAAe,CAAC,EAC1B,cAAc,EAAE,OAAO,EACtB,aAAa,CAAC,SAAQ,IAAI,aAAA;IAM5C,SAAgB,YAAY,QAAO,IAAI,CAGrC;IAEF,SAAgB,YAAY,UAAW,OAAO,KAAG,IAAI,CAGnD;IAEK,WAAW,IAAI,IAAI;CAM1B;AAID,MAAM,MAAM,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,YAAY,GAAG,UAAU,CACzC,GACA,cAAc,CAAC,uBAAuB,CAAC,CAAC;AAkBzC,MAAM,WAAW,aAAa;IAC7B,CAAC,OAAO,EAAE,MAAM,GAAG;QAClB,IAAI,EAAE,MAAM,CAAC;QACb,SAAS,CAAC,EAAE,MAAM,CAAC;QACnB,UAAU,CAAC,EAAE,MAAM,CAAC;QACpB,eAAe,CAAC,EAAE,MAAM,CAAC;QACzB,KAAK,CAAC,EAAE,OAAO,CAAC;KAChB,CAAC;CACF;AAED,MAAM,WAAW,kBAAkB;IAClC,cAAc,EAAE,MAAM,IAAI,CAAC;CAC3B;AAQD,eAAO,MAAM,mBAAmB,UAAoB,CAAC;AAErD,qBAAa,WAAW;IACvB,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IACpE,IAAW,MAAM,IAAI,UAAU,CAAC,kBAAkB,CAAC,CAElD;IACD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAA+C;IAE9E;;;;;OAKG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAsB;IAEpD;;OAEG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAuC;IAEnE,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAA+C;IAEhF,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAsB;IACnD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4D;IAGpF,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA6B;IAG3D,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAgC;IAC9D,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsB;IAC9C,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAe;IAEhD,OAAO,CAAC,QAAQ,CAAC,wBAAwB,CAAU;gBAEhC,KAAK,EAAE;QACzB,QAAQ,CAAC,YAAY,EAAE,mBAAmB,CAAC;QAE3C,mBAAmB,EAAE,oBAAoB,CAAC;QAC1C,QAAQ,CAAC,OAAO,EAAE,IAAI,CAAC,wBAAwB,EAAE,YAAY,GAAG,UAAU,CAAC,CAAC;QAC5E;;;;;;;;;WASG;QACH,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,KAAK,IAAI,CAAC;QAG/D,QAAQ,CAAC,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,CAAC;QAGnD,QAAQ,CAAC,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;QACtD,QAAQ,CAAC,OAAO,EAAE,mBAAmB,CAAC;QACtC,YAAY,EAAE,aAAa,GAAG,SAAS,CAAC;QACxC,QAAQ,CAAC,gBAAgB,CAAC,EAAE,CAAC,MAAM,MAAM,CAAC,GAAG,SAAS,CAAC;QACvD,QAAQ,CAAC,wBAAwB,EAAE,OAAO,CAAC;KAC3C;IA4DD,IAAW,gBAAgB,IAAI,OAAO,CAOrC;IAED,IAAW,eAAe,IAAI,OAAO,CAKpC;IAED,OAAO,CAAC,gBAAgB;IAOjB,OAAO,CAAC,OAAO,EAAE,MAAM,GAAG,OAAO;IAIxC;;;;;;;;;OASG;IACI,4BAA4B,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,GAAG,SAAS;IAQxE;;;;;OAKG;IACU,OAAO,CAAC,OAAO,EAAE,MAAM,EAAE,cAAc,EAAE,OAAO,GAAG,OAAO,CAAC,eAAe,CAAC;IAoDxF,OAAO,CAAC,aAAa;YAwBP,kBAAkB;IAYnB,UAAU,CACtB,IAAI,EAAE,eAAe,EACrB,MAAM,CAAC,EAAE,WAAW,GAClB,OAAO,CAAC,kCAAkC,CAAC,eAAe,CAAC,CAAC;YAmBjD,gBAAgB;IAkC9B,OAAO,CAAC,4BAA4B;IA0CpC;;;;;OAKG;YACW,UAAU;IAsCxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAItB,OAAO,CAAC,sBAAsB;IAS9B,OAAO,CAAC,iBAAiB;IAMzB,OAAO,CAAC,eAAe;IAwDvB;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS,GAAG,IAAI;IAc7D,wBAAwB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO,GAAG,IAAI;IAiDlF,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IAI7E;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IAcjE;;;;;OAKG;IACI,qBAAqB,CAAC,oBAAoB,EAAE,SAAS,MAAM,EAAE,GAAG,SAAS,MAAM,EAAE;IAKxF;;;;;;;;;;;;;;;OAeG;IACH,OAAO,CAAC,4BAA4B;IA0CpC;;;OAGG;IACH,OAAO,CAAC,oBAAoB;IAqB5B;;;;;OAKG;IACH,SAAgB,kBAAkB,yBAA0B,IAAI,MAAM,EAAE,MAAM,CAAC,KAAG,IAAI,CAuBpF;IAEF;;;;;;;;;;OAUG;IACI,eAAe,IAAI,aAAa,GAAG,SAAS;IAInD;;;;;;;;;OASG;IACU,wBAAwB,CACpC,uBAAuB,CAAC,EAAE,WAAW,GACnC,OAAO,CAAC,aAAa,GAAG,SAAS,CAAC;CAGrC;AAmBD;;GAEG;AACH,eAAO,MAAM,UAAU,SAAU,MAAM,gCACL,CAAC"}
|
|
@@ -78,20 +78,44 @@ class BlobManager {
|
|
|
78
78
|
* because we know that the server will not delete the blob corresponding to that storage ID.
|
|
79
79
|
*/
|
|
80
80
|
this.opsInFlight = new Map();
|
|
81
|
-
|
|
81
|
+
/**
|
|
82
|
+
* Called in detached state just prior to attaching, this will update the redirect table by
|
|
83
|
+
* converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
|
|
84
|
+
* The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
|
|
85
|
+
* @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
|
|
86
|
+
*/
|
|
87
|
+
this.patchRedirectTable = (detachedStorageTable) => {
|
|
88
|
+
(0, internal_2.assert)(this.runtime.attachState === internal_1.AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
|
|
89
|
+
// The values of the redirect table are the pseudo storage IDs, which are the keys of the
|
|
90
|
+
// detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
|
|
91
|
+
// storage IDs (many in the case that the storage dedupes the blob).
|
|
92
|
+
(0, internal_2.assert)(new Set(this.redirectTable.values()).size === detachedStorageTable.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
|
|
93
|
+
// Taking a snapshot of the redirect table entries before iterating, because
|
|
94
|
+
// we will be adding identity mappings to the the redirect table as we iterate
|
|
95
|
+
// and we don't want to include those in the iteration.
|
|
96
|
+
const redirectTableEntries = [...this.redirectTable.entries()];
|
|
97
|
+
for (const [localId, detachedStorageId] of redirectTableEntries) {
|
|
98
|
+
const newStorageId = detachedStorageTable.get(detachedStorageId);
|
|
99
|
+
(0, internal_2.assert)(newStorageId !== undefined, "Couldn't find a matching storage ID");
|
|
100
|
+
this.setRedirection(localId, newStorageId);
|
|
101
|
+
// set identity (id -> id) entry
|
|
102
|
+
this.setRedirection(newStorageId, newStorageId);
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
const { routeContext, blobManagerLoadInfo, storage, sendBlobAttachOp, blobRequested, isBlobDeleted, runtime, localIdGenerator, createBlobPayloadPending, } = props;
|
|
82
106
|
this.routeContext = routeContext;
|
|
83
107
|
this.storage = storage;
|
|
84
108
|
this.blobRequested = blobRequested;
|
|
85
109
|
this.isBlobDeleted = isBlobDeleted;
|
|
86
110
|
this.runtime = runtime;
|
|
87
|
-
this.
|
|
111
|
+
this.localIdGenerator = localIdGenerator ?? uuid_1.v4;
|
|
88
112
|
this.createBlobPayloadPending = createBlobPayloadPending;
|
|
89
113
|
this.mc = (0, internal_4.createChildMonitoringContext)({
|
|
90
114
|
logger: this.runtime.baseLogger,
|
|
91
115
|
namespace: "BlobManager",
|
|
92
116
|
});
|
|
93
|
-
this.redirectTable = (0, blobManagerSnapSum_js_1.toRedirectTable)(blobManagerLoadInfo, this.mc.logger
|
|
94
|
-
this.sendBlobAttachOp = (localId,
|
|
117
|
+
this.redirectTable = (0, blobManagerSnapSum_js_1.toRedirectTable)(blobManagerLoadInfo, this.mc.logger);
|
|
118
|
+
this.sendBlobAttachOp = (localId, storageId) => {
|
|
95
119
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
96
120
|
(0, internal_2.assert)(pendingEntry !== undefined, 0x725 /* Must have pending blob entry for upcoming op */);
|
|
97
121
|
if (pendingEntry?.uploadTime && pendingEntry?.minTTLInSeconds) {
|
|
@@ -117,7 +141,7 @@ class BlobManager {
|
|
|
117
141
|
}
|
|
118
142
|
}
|
|
119
143
|
pendingEntry.opsent = true;
|
|
120
|
-
sendBlobAttachOp(localId,
|
|
144
|
+
sendBlobAttachOp(localId, storageId);
|
|
121
145
|
};
|
|
122
146
|
}
|
|
123
147
|
get allBlobsAttached() {
|
|
@@ -138,56 +162,63 @@ class BlobManager {
|
|
|
138
162
|
uploadTime: pending?.uploadTime,
|
|
139
163
|
});
|
|
140
164
|
}
|
|
141
|
-
hasBlob(
|
|
142
|
-
return this.redirectTable.get(
|
|
165
|
+
hasBlob(localId) {
|
|
166
|
+
return this.redirectTable.get(localId) !== undefined;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* Lookup the blob storage ID for a given local blob id.
|
|
170
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
171
|
+
* @returns The storage ID if found and the blob is not pending, undefined otherwise.
|
|
172
|
+
* @remarks
|
|
173
|
+
* For blobs with pending payloads (localId exists but upload hasn't finished), this is expected to return undefined.
|
|
174
|
+
* Consumers should use the observability APIs on the handle (handle.payloadState, payloadShared event)
|
|
175
|
+
* to understand/wait for storage ID availability.
|
|
176
|
+
* Similarly, when the runtime is detached, this will return undefined as no blobs have been uploaded to storage.
|
|
177
|
+
*/
|
|
178
|
+
lookupTemporaryBlobStorageId(localId) {
|
|
179
|
+
if (this.runtime.attachState === internal_1.AttachState.Detached) {
|
|
180
|
+
return undefined;
|
|
181
|
+
}
|
|
182
|
+
// Get the storage ID from the redirect table
|
|
183
|
+
return this.redirectTable.get(localId);
|
|
143
184
|
}
|
|
144
185
|
/**
|
|
145
186
|
* Retrieve the blob with the given local blob id.
|
|
146
|
-
* @param
|
|
187
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
147
188
|
* @param payloadPending - Whether we suspect the payload may be pending and not available yet.
|
|
148
189
|
* @returns A promise which resolves to the blob contents
|
|
149
190
|
*/
|
|
150
|
-
async getBlob(
|
|
191
|
+
async getBlob(localId, payloadPending) {
|
|
151
192
|
// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
|
|
152
193
|
// an error, failing the call.
|
|
153
|
-
this.verifyBlobNotDeleted(
|
|
194
|
+
this.verifyBlobNotDeleted(localId);
|
|
154
195
|
// Let runtime know that the corresponding GC node was requested.
|
|
155
196
|
// Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
|
|
156
197
|
// is configured.
|
|
157
|
-
this.blobRequested(
|
|
158
|
-
const pending = this.pendingBlobs.get(
|
|
198
|
+
this.blobRequested(getGCNodePathFromLocalId(localId));
|
|
199
|
+
const pending = this.pendingBlobs.get(localId);
|
|
159
200
|
if (pending) {
|
|
160
201
|
return pending.blob;
|
|
161
202
|
}
|
|
162
|
-
let storageId;
|
|
163
|
-
if (
|
|
164
|
-
|
|
165
|
-
//
|
|
166
|
-
//
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
attachedStorageId ??
|
|
182
|
-
(await new Promise((resolve) => {
|
|
183
|
-
const onProcessBlobAttach = (localId, _storageId) => {
|
|
184
|
-
if (localId === blobId) {
|
|
185
|
-
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
186
|
-
resolve(_storageId);
|
|
187
|
-
}
|
|
188
|
-
};
|
|
189
|
-
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
190
|
-
}));
|
|
203
|
+
let storageId = this.redirectTable.get(localId);
|
|
204
|
+
if (storageId === undefined) {
|
|
205
|
+
// Only blob handles explicitly marked with pending payload are permitted to exist without
|
|
206
|
+
// yet knowing their storage id. Otherwise they must already be associated with a storage id.
|
|
207
|
+
// Handles for detached blobs are not payload pending.
|
|
208
|
+
(0, internal_2.assert)(payloadPending, 0x11f /* "requesting unknown blobs" */);
|
|
209
|
+
// If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
|
|
210
|
+
// eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
|
|
211
|
+
// enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
|
|
212
|
+
// client failed the upload and doesn't exist anymore.
|
|
213
|
+
storageId = await new Promise((resolve) => {
|
|
214
|
+
const onProcessBlobAttach = (_localId, _storageId) => {
|
|
215
|
+
if (_localId === localId) {
|
|
216
|
+
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
217
|
+
resolve(_storageId);
|
|
218
|
+
}
|
|
219
|
+
};
|
|
220
|
+
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
221
|
+
});
|
|
191
222
|
}
|
|
192
223
|
return internal_4.PerformanceEvent.timedExecAsync(this.mc.logger, { eventName: "AttachmentReadBlob", id: storageId }, async (event) => {
|
|
193
224
|
return this.storage.readBlob(storageId).catch((error) => {
|
|
@@ -211,15 +242,17 @@ class BlobManager {
|
|
|
211
242
|
this.deletePendingBlobMaybe(localId);
|
|
212
243
|
}
|
|
213
244
|
: undefined;
|
|
214
|
-
return new BlobHandle(
|
|
245
|
+
return new BlobHandle(getGCNodePathFromLocalId(localId), this.routeContext, async () => this.getBlob(localId, false), false, // payloadPending
|
|
215
246
|
callback);
|
|
216
247
|
}
|
|
217
248
|
async createBlobDetached(blob) {
|
|
249
|
+
const localId = this.localIdGenerator();
|
|
218
250
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
219
|
-
// The 'IContainerStorageService.createBlob()' call below will respond with a
|
|
220
|
-
|
|
221
|
-
this.
|
|
222
|
-
|
|
251
|
+
// The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
|
|
252
|
+
// That pseudo storage ID will be replaced with the real storage ID at attach time.
|
|
253
|
+
const { id: detachedStorageId } = await this.storage.createBlob(blob);
|
|
254
|
+
this.setRedirection(localId, detachedStorageId);
|
|
255
|
+
return this.getBlobHandle(localId);
|
|
223
256
|
}
|
|
224
257
|
async createBlob(blob, signal) {
|
|
225
258
|
if (this.runtime.attachState === internal_1.AttachState.Detached) {
|
|
@@ -241,7 +274,7 @@ class BlobManager {
|
|
|
241
274
|
}
|
|
242
275
|
// Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
|
|
243
276
|
// storage ID mapping is created.
|
|
244
|
-
const localId = this.
|
|
277
|
+
const localId = this.localIdGenerator();
|
|
245
278
|
const pendingEntry = {
|
|
246
279
|
blob,
|
|
247
280
|
handleP: new internal_2.Deferred(),
|
|
@@ -263,8 +296,8 @@ class BlobManager {
|
|
|
263
296
|
});
|
|
264
297
|
}
|
|
265
298
|
createBlobWithPayloadPending(blob) {
|
|
266
|
-
const localId = this.
|
|
267
|
-
const blobHandle = new BlobHandle(
|
|
299
|
+
const localId = this.localIdGenerator();
|
|
300
|
+
const blobHandle = new BlobHandle(getGCNodePathFromLocalId(localId), this.routeContext, async () => blob, true, // payloadPending
|
|
268
301
|
() => {
|
|
269
302
|
const pendingEntry = {
|
|
270
303
|
blob,
|
|
@@ -379,7 +412,7 @@ class BlobManager {
|
|
|
379
412
|
if (!entry.opsent) {
|
|
380
413
|
this.sendBlobAttachOp(localId, response.id);
|
|
381
414
|
}
|
|
382
|
-
const storageIds = (0, blobManagerSnapSum_js_1.getStorageIds)(this.redirectTable
|
|
415
|
+
const storageIds = (0, blobManagerSnapSum_js_1.getStorageIds)(this.redirectTable);
|
|
383
416
|
if (storageIds.has(response.id)) {
|
|
384
417
|
// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
|
|
385
418
|
// an existing blob, we don't have to wait for the op to be ack'd since this step has already
|
|
@@ -412,7 +445,7 @@ class BlobManager {
|
|
|
412
445
|
*/
|
|
413
446
|
reSubmit(metadata) {
|
|
414
447
|
(0, internal_2.assert)((0, metadata_js_1.isBlobMetadata)(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
|
|
415
|
-
const { localId, blobId } = metadata;
|
|
448
|
+
const { localId, blobId: storageId } = metadata;
|
|
416
449
|
// Any blob that we're actively trying to advance to attached state must have a
|
|
417
450
|
// pendingBlobs entry. Decline to resubmit for anything else.
|
|
418
451
|
// For example, we might be asked to resubmit stashed ops for blobs that never had
|
|
@@ -420,22 +453,22 @@ class BlobManager {
|
|
|
420
453
|
// try to attach them since they won't be accessible to the customer and would just
|
|
421
454
|
// be considered garbage immediately.
|
|
422
455
|
if (this.pendingBlobs.has(localId)) {
|
|
423
|
-
this.sendBlobAttachOp(localId,
|
|
456
|
+
this.sendBlobAttachOp(localId, storageId);
|
|
424
457
|
}
|
|
425
458
|
}
|
|
426
459
|
processBlobAttachMessage(message, local) {
|
|
427
460
|
(0, internal_2.assert)((0, metadata_js_1.isBlobMetadata)(message.metadata), 0xc02 /* Expected blob metadata for a BlobAttach op */);
|
|
428
|
-
const { localId, blobId } = message.metadata;
|
|
461
|
+
const { localId, blobId: storageId } = message.metadata;
|
|
429
462
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
430
463
|
if (pendingEntry?.abortSignal?.aborted) {
|
|
431
464
|
this.deletePendingBlob(localId);
|
|
432
465
|
return;
|
|
433
466
|
}
|
|
434
|
-
this.setRedirection(localId,
|
|
467
|
+
this.setRedirection(localId, storageId);
|
|
435
468
|
// set identity (id -> id) entry
|
|
436
|
-
this.setRedirection(
|
|
469
|
+
this.setRedirection(storageId, storageId);
|
|
437
470
|
if (local) {
|
|
438
|
-
const waitingBlobs = this.opsInFlight.get(
|
|
471
|
+
const waitingBlobs = this.opsInFlight.get(storageId);
|
|
439
472
|
if (waitingBlobs !== undefined) {
|
|
440
473
|
// For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
|
|
441
474
|
// This is safe because the server will keep the blob alive and the op containing the local ID to
|
|
@@ -443,14 +476,14 @@ class BlobManager {
|
|
|
443
476
|
for (const pendingLocalId of waitingBlobs) {
|
|
444
477
|
const entry = this.pendingBlobs.get(pendingLocalId);
|
|
445
478
|
(0, internal_2.assert)(entry !== undefined, 0x38f /* local online BlobAttach op with no pending blob entry */);
|
|
446
|
-
this.setRedirection(pendingLocalId,
|
|
479
|
+
this.setRedirection(pendingLocalId, storageId);
|
|
447
480
|
entry.acked = true;
|
|
448
481
|
const blobHandle = this.getBlobHandle(pendingLocalId);
|
|
449
482
|
blobHandle.notifyShared();
|
|
450
483
|
entry.handleP.resolve(blobHandle);
|
|
451
484
|
this.deletePendingBlobMaybe(pendingLocalId);
|
|
452
485
|
}
|
|
453
|
-
this.opsInFlight.delete(
|
|
486
|
+
this.opsInFlight.delete(storageId);
|
|
454
487
|
}
|
|
455
488
|
const localEntry = this.pendingBlobs.get(localId);
|
|
456
489
|
if (localEntry) {
|
|
@@ -461,10 +494,10 @@ class BlobManager {
|
|
|
461
494
|
this.deletePendingBlobMaybe(localId);
|
|
462
495
|
}
|
|
463
496
|
}
|
|
464
|
-
this.internalEvents.emit("processedBlobAttach", localId,
|
|
497
|
+
this.internalEvents.emit("processedBlobAttach", localId, storageId);
|
|
465
498
|
}
|
|
466
499
|
summarize(telemetryContext) {
|
|
467
|
-
return (0, blobManagerSnapSum_js_1.summarizeBlobManagerState)(this.redirectTable
|
|
500
|
+
return (0, blobManagerSnapSum_js_1.summarizeBlobManagerState)(this.redirectTable);
|
|
468
501
|
}
|
|
469
502
|
/**
|
|
470
503
|
* Generates data used for garbage collection. Each blob uploaded represents a node in the GC graph as it can be
|
|
@@ -475,13 +508,12 @@ class BlobManager {
|
|
|
475
508
|
getGCData(fullGC = false) {
|
|
476
509
|
const gcData = { gcNodes: {} };
|
|
477
510
|
for (const [localId, storageId] of this.redirectTable) {
|
|
478
|
-
|
|
479
|
-
//
|
|
480
|
-
// id entries have the same key and value, ignore them.
|
|
481
|
-
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
482
|
-
// by adding its handle to a referenced DDS.
|
|
511
|
+
// Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
|
|
512
|
+
// IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
|
|
483
513
|
if (localId !== storageId) {
|
|
484
|
-
|
|
514
|
+
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
515
|
+
// by adding its handle to a referenced DDS.
|
|
516
|
+
gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
|
|
485
517
|
}
|
|
486
518
|
}
|
|
487
519
|
return gcData;
|
|
@@ -500,55 +532,53 @@ class BlobManager {
|
|
|
500
532
|
* Delete blobs with the given routes from the redirect table.
|
|
501
533
|
*
|
|
502
534
|
* @remarks
|
|
503
|
-
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<
|
|
535
|
+
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
|
|
504
536
|
* Deleting the blobs involves 2 steps:
|
|
505
537
|
*
|
|
506
538
|
* 1. The redirect table entry for the local ids are deleted.
|
|
507
539
|
*
|
|
508
|
-
* 2. If the storage ids corresponding to the deleted local ids are not
|
|
509
|
-
*
|
|
540
|
+
* 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
|
|
541
|
+
* identity mappings in the redirect table are deleted as well.
|
|
510
542
|
*
|
|
511
543
|
* Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
|
|
512
|
-
* will
|
|
544
|
+
* will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
|
|
545
|
+
* some time in the future.
|
|
513
546
|
*/
|
|
514
547
|
deleteBlobsFromRedirectTable(blobRoutes) {
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
}
|
|
518
|
-
// This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
|
|
519
|
-
// these storage ids are unused, they will be deleted as well.
|
|
548
|
+
// maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
|
|
549
|
+
// local ID is being deleted.
|
|
520
550
|
const maybeUnusedStorageIds = new Set();
|
|
521
551
|
for (const route of blobRoutes) {
|
|
522
|
-
const
|
|
552
|
+
const localId = getLocalIdFromGCNodePath(route);
|
|
523
553
|
// If the blob hasn't already been deleted, log an error because this should never happen.
|
|
524
554
|
// If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
|
|
525
555
|
// sweep ops can contain the same data store. It would be interesting to track how often this happens.
|
|
526
556
|
const alreadyDeleted = this.isBlobDeleted(route);
|
|
527
|
-
|
|
557
|
+
const storageId = this.redirectTable.get(localId);
|
|
558
|
+
if (storageId === undefined) {
|
|
528
559
|
this.mc.logger.sendTelemetryEvent({
|
|
529
560
|
eventName: "DeletedAttachmentBlobNotFound",
|
|
530
561
|
category: alreadyDeleted ? "generic" : "error",
|
|
531
|
-
blobId,
|
|
562
|
+
blobId: localId,
|
|
532
563
|
details: { alreadyDeleted },
|
|
533
564
|
});
|
|
534
565
|
continue;
|
|
535
566
|
}
|
|
536
|
-
const storageId = this.redirectTable.get(blobId);
|
|
537
|
-
(0, internal_2.assert)(!!storageId, 0x5bb /* Must be attached to run GC */);
|
|
538
567
|
maybeUnusedStorageIds.add(storageId);
|
|
539
|
-
this.redirectTable.delete(
|
|
568
|
+
this.redirectTable.delete(localId);
|
|
540
569
|
}
|
|
541
|
-
//
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
(0, internal_2.assert)(!!storageId, 0x5bc /* Must be attached to run GC */);
|
|
545
|
-
// For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
|
|
546
|
-
if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
|
|
570
|
+
// Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
|
|
571
|
+
for (const [localId, storageId] of this.redirectTable) {
|
|
572
|
+
if (localId !== storageId) {
|
|
547
573
|
maybeUnusedStorageIds.delete(storageId);
|
|
548
574
|
}
|
|
549
575
|
}
|
|
550
|
-
//
|
|
551
|
-
// This way they'll be absent from the next summary, and the service
|
|
576
|
+
// Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
|
|
577
|
+
// referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
|
|
578
|
+
// is free to delete them from storage.
|
|
579
|
+
// WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
|
|
580
|
+
// has let us add a local ID -> storage ID mapping that is later deleted. AB#47337 tracks this issue
|
|
581
|
+
// and possible solutions.
|
|
552
582
|
for (const storageId of maybeUnusedStorageIds) {
|
|
553
583
|
this.redirectTable.delete(storageId);
|
|
554
584
|
}
|
|
@@ -557,11 +587,11 @@ class BlobManager {
|
|
|
557
587
|
* Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
|
|
558
588
|
* log an error and throw if necessary.
|
|
559
589
|
*/
|
|
560
|
-
verifyBlobNotDeleted(
|
|
561
|
-
if (!this.isBlobDeleted(
|
|
590
|
+
verifyBlobNotDeleted(localId) {
|
|
591
|
+
if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
|
|
562
592
|
return;
|
|
563
593
|
}
|
|
564
|
-
const request = { url:
|
|
594
|
+
const request = { url: localId };
|
|
565
595
|
const error = (0, internal_3.responseToException)((0, internal_3.createResponseError)(404, `Blob was deleted`, request), request);
|
|
566
596
|
// Only log deleted events. Tombstone events are logged by garbage collector.
|
|
567
597
|
this.mc.logger.sendErrorEvent({
|
|
@@ -570,16 +600,6 @@ class BlobManager {
|
|
|
570
600
|
}, error);
|
|
571
601
|
throw error;
|
|
572
602
|
}
|
|
573
|
-
setRedirectTable(table) {
|
|
574
|
-
(0, internal_2.assert)(this.runtime.attachState === internal_1.AttachState.Detached, 0x252 /* "redirect table can only be set in detached container" */);
|
|
575
|
-
(0, internal_2.assert)(this.redirectTable.size === table.size, 0x391 /* Redirect table size must match BlobManager's local ID count */);
|
|
576
|
-
for (const [localId, storageId] of table) {
|
|
577
|
-
(0, internal_2.assert)(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
|
|
578
|
-
this.setRedirection(localId, storageId);
|
|
579
|
-
// set identity (id -> id) entry
|
|
580
|
-
this.setRedirection(storageId, storageId);
|
|
581
|
-
}
|
|
582
|
-
}
|
|
583
603
|
/**
|
|
584
604
|
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
585
605
|
* pending upload and/or their BlobAttach op, which can be given to a new BlobManager to
|
|
@@ -610,15 +630,15 @@ class BlobManager {
|
|
|
610
630
|
}
|
|
611
631
|
exports.BlobManager = BlobManager;
|
|
612
632
|
/**
|
|
613
|
-
* For a
|
|
633
|
+
* For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
|
|
614
634
|
* This path must match the path of the blob handle returned by the createBlob API because blobs are marked
|
|
615
635
|
* referenced by storing these handles in a referenced DDS.
|
|
616
636
|
*/
|
|
617
|
-
const
|
|
637
|
+
const getGCNodePathFromLocalId = (localId) => `/${exports.blobManagerBasePath}/${localId}`;
|
|
618
638
|
/**
|
|
619
|
-
* For a given GC node path, return the
|
|
639
|
+
* For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
|
|
620
640
|
*/
|
|
621
|
-
const
|
|
641
|
+
const getLocalIdFromGCNodePath = (nodePath) => {
|
|
622
642
|
const pathParts = nodePath.split("/");
|
|
623
643
|
(0, internal_2.assert)(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
|
|
624
644
|
return pathParts[2];
|