@fluidframework/container-runtime 2.60.0 → 2.61.0-355516
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mocharc.cjs +1 -2
- package/api-report/container-runtime.legacy.beta.api.md +2 -1
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +33 -16
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +126 -106
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.js +30 -33
- package/dist/blobManager/blobManagerSnapSum.js.map +1 -1
- package/dist/channelCollection.d.ts +6 -2
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +1 -0
- package/dist/channelCollection.js.map +1 -1
- package/dist/containerCompatibility.d.ts +18 -0
- package/dist/containerCompatibility.d.ts.map +1 -1
- package/dist/containerCompatibility.js +23 -1
- package/dist/containerCompatibility.js.map +1 -1
- package/dist/containerRuntime.d.ts +15 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +75 -52
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +5 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +1 -0
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/legacy.d.ts +2 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/lib/blobManager/blobManager.d.ts +33 -16
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +126 -106
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.js +26 -29
- package/lib/blobManager/blobManagerSnapSum.js.map +1 -1
- package/lib/channelCollection.d.ts +6 -2
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +1 -0
- package/lib/channelCollection.js.map +1 -1
- package/lib/containerCompatibility.d.ts +18 -0
- package/lib/containerCompatibility.d.ts.map +1 -1
- package/lib/containerCompatibility.js +22 -0
- package/lib/containerCompatibility.js.map +1 -1
- package/lib/containerRuntime.d.ts +15 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +26 -3
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +5 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +1 -0
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/legacy.d.ts +2 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/tsdoc-metadata.json +1 -1
- package/package.json +27 -27
- package/src/blobManager/blobManager.ts +138 -123
- package/src/blobManager/blobManagerSnapSum.ts +31 -53
- package/src/channelCollection.ts +9 -1
- package/src/containerCompatibility.ts +56 -0
- package/src/containerRuntime.ts +35 -4
- package/src/dataStoreContext.ts +7 -0
- package/src/packageVersion.ts +1 -1
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluidframework/container-runtime",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.61.0-355516",
|
|
4
4
|
"description": "Fluid container runtime",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": {
|
|
@@ -119,18 +119,18 @@
|
|
|
119
119
|
"temp-directory": "nyc/.nyc_output"
|
|
120
120
|
},
|
|
121
121
|
"dependencies": {
|
|
122
|
-
"@fluid-internal/client-utils": "
|
|
123
|
-
"@fluidframework/container-definitions": "
|
|
124
|
-
"@fluidframework/container-runtime-definitions": "
|
|
125
|
-
"@fluidframework/core-interfaces": "
|
|
126
|
-
"@fluidframework/core-utils": "
|
|
127
|
-
"@fluidframework/datastore": "
|
|
128
|
-
"@fluidframework/driver-definitions": "
|
|
129
|
-
"@fluidframework/driver-utils": "
|
|
130
|
-
"@fluidframework/id-compressor": "
|
|
131
|
-
"@fluidframework/runtime-definitions": "
|
|
132
|
-
"@fluidframework/runtime-utils": "
|
|
133
|
-
"@fluidframework/telemetry-utils": "
|
|
122
|
+
"@fluid-internal/client-utils": "2.61.0-355516",
|
|
123
|
+
"@fluidframework/container-definitions": "2.61.0-355516",
|
|
124
|
+
"@fluidframework/container-runtime-definitions": "2.61.0-355516",
|
|
125
|
+
"@fluidframework/core-interfaces": "2.61.0-355516",
|
|
126
|
+
"@fluidframework/core-utils": "2.61.0-355516",
|
|
127
|
+
"@fluidframework/datastore": "2.61.0-355516",
|
|
128
|
+
"@fluidframework/driver-definitions": "2.61.0-355516",
|
|
129
|
+
"@fluidframework/driver-utils": "2.61.0-355516",
|
|
130
|
+
"@fluidframework/id-compressor": "2.61.0-355516",
|
|
131
|
+
"@fluidframework/runtime-definitions": "2.61.0-355516",
|
|
132
|
+
"@fluidframework/runtime-utils": "2.61.0-355516",
|
|
133
|
+
"@fluidframework/telemetry-utils": "2.61.0-355516",
|
|
134
134
|
"@tylerbu/sorted-btree-es6": "^1.8.0",
|
|
135
135
|
"double-ended-queue": "^2.1.0-0",
|
|
136
136
|
"lz4js": "^0.2.0",
|
|
@@ -140,23 +140,23 @@
|
|
|
140
140
|
"devDependencies": {
|
|
141
141
|
"@arethetypeswrong/cli": "^0.17.1",
|
|
142
142
|
"@biomejs/biome": "~1.9.3",
|
|
143
|
-
"@fluid-internal/mocha-test-setup": "
|
|
144
|
-
"@fluid-private/stochastic-test-utils": "
|
|
145
|
-
"@fluid-private/test-pairwise-generator": "
|
|
143
|
+
"@fluid-internal/mocha-test-setup": "2.61.0-355516",
|
|
144
|
+
"@fluid-private/stochastic-test-utils": "2.61.0-355516",
|
|
145
|
+
"@fluid-private/test-pairwise-generator": "2.61.0-355516",
|
|
146
146
|
"@fluid-tools/benchmark": "^0.51.0",
|
|
147
|
-
"@fluid-tools/build-cli": "^0.
|
|
147
|
+
"@fluid-tools/build-cli": "^0.58.1",
|
|
148
148
|
"@fluidframework/build-common": "^2.0.3",
|
|
149
|
-
"@fluidframework/build-tools": "^0.
|
|
150
|
-
"@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.
|
|
149
|
+
"@fluidframework/build-tools": "^0.58.1",
|
|
150
|
+
"@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.60.0",
|
|
151
151
|
"@fluidframework/eslint-config-fluid": "^6.0.0",
|
|
152
|
-
"@fluidframework/test-runtime-utils": "
|
|
153
|
-
"@microsoft/api-extractor": "7.52.
|
|
152
|
+
"@fluidframework/test-runtime-utils": "2.61.0-355516",
|
|
153
|
+
"@microsoft/api-extractor": "7.52.11",
|
|
154
154
|
"@types/double-ended-queue": "^2.1.0",
|
|
155
155
|
"@types/lz4js": "^0.2.0",
|
|
156
156
|
"@types/mocha": "^10.0.10",
|
|
157
157
|
"@types/node": "^18.19.0",
|
|
158
158
|
"@types/sinon": "^17.0.3",
|
|
159
|
-
"c8": "^
|
|
159
|
+
"c8": "^10.1.3",
|
|
160
160
|
"concurrently": "^8.2.1",
|
|
161
161
|
"copyfiles": "^2.4.1",
|
|
162
162
|
"cross-env": "^7.0.3",
|
|
@@ -173,8 +173,8 @@
|
|
|
173
173
|
},
|
|
174
174
|
"scripts": {
|
|
175
175
|
"api": "fluid-build . --task api",
|
|
176
|
-
"api-extractor:commonjs": "flub generate entrypoints --outDir ./dist",
|
|
177
|
-
"api-extractor:esnext": "flub generate entrypoints --outDir ./lib --node10TypeCompat",
|
|
176
|
+
"api-extractor:commonjs": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./dist",
|
|
177
|
+
"api-extractor:esnext": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./lib --node10TypeCompat",
|
|
178
178
|
"build": "fluid-build . --task build",
|
|
179
179
|
"build:api-reports": "concurrently \"npm:build:api-reports:*\"",
|
|
180
180
|
"build:api-reports:current": "api-extractor run --local --config api-extractor/api-extractor.current.json",
|
|
@@ -210,11 +210,11 @@
|
|
|
210
210
|
"pack:tests": "tar -cf ./container-runtime.test-files.tar ./src/test ./dist/test ./lib/test",
|
|
211
211
|
"place:cjs:package-stub": "copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist",
|
|
212
212
|
"test": "npm run test:mocha",
|
|
213
|
-
"test:benchmark:report": "mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js
|
|
213
|
+
"test:benchmark:report": "cross-env \"MOCHA_SPEC=dist/**/*.perf.spec.*js\" mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js",
|
|
214
214
|
"test:coverage": "c8 npm test",
|
|
215
215
|
"test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",
|
|
216
|
-
"test:mocha:cjs": "
|
|
217
|
-
"test:mocha:esm": "mocha
|
|
216
|
+
"test:mocha:cjs": "cross-env MOCHA_SPEC=dist/test mocha",
|
|
217
|
+
"test:mocha:esm": "mocha",
|
|
218
218
|
"test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
|
|
219
219
|
"tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && npm run place:cjs:package-stub",
|
|
220
220
|
"tsc:watch": "npm run place:cjs:package-stub && fluid-tsc commonjs --project ./tsconfig.cjs.json --watch",
|
|
@@ -129,7 +129,7 @@ export class BlobHandle
|
|
|
129
129
|
// the contract explicit and reduces the amount of mocking required for tests.
|
|
130
130
|
export type IBlobManagerRuntime = Pick<
|
|
131
131
|
IContainerRuntime,
|
|
132
|
-
"attachState" | "
|
|
132
|
+
"attachState" | "baseLogger" | "disposed"
|
|
133
133
|
> &
|
|
134
134
|
IEventProvider<IContainerRuntimeEvents>;
|
|
135
135
|
|
|
@@ -181,13 +181,12 @@ export class BlobManager {
|
|
|
181
181
|
private readonly internalEvents = createEmitter<IBlobManagerInternalEvents>();
|
|
182
182
|
|
|
183
183
|
/**
|
|
184
|
-
* Map of local IDs to storage IDs.
|
|
185
|
-
* be a key in this map. Blobs created while the container is detached are
|
|
186
|
-
* gives
|
|
187
|
-
*
|
|
188
|
-
* that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
|
|
184
|
+
* Map of local IDs to storage IDs. Also includes identity mappings of storage ID to storage ID for all known
|
|
185
|
+
* storage IDs. All requested IDs must be a key in this map. Blobs created while the container is detached are
|
|
186
|
+
* stored in IDetachedBlobStorage which gives pseudo storage IDs; the real storage IDs are filled in at attach
|
|
187
|
+
* time via setRedirectTable().
|
|
189
188
|
*/
|
|
190
|
-
private readonly redirectTable: Map<string, string
|
|
189
|
+
private readonly redirectTable: Map<string, string>;
|
|
191
190
|
|
|
192
191
|
/**
|
|
193
192
|
* Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
|
|
@@ -206,13 +205,13 @@ export class BlobManager {
|
|
|
206
205
|
private readonly routeContext: IFluidHandleContext;
|
|
207
206
|
private readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
|
|
208
207
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
209
|
-
// blobPath's format - `/<basePath>/<
|
|
208
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
210
209
|
private readonly blobRequested: (blobPath: string) => void;
|
|
211
210
|
// Called to check if a blob has been deleted by GC.
|
|
212
|
-
// blobPath's format - `/<basePath>/<
|
|
211
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
213
212
|
private readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
214
213
|
private readonly runtime: IBlobManagerRuntime;
|
|
215
|
-
private readonly
|
|
214
|
+
private readonly localIdGenerator: () => string;
|
|
216
215
|
|
|
217
216
|
private readonly createBlobPayloadPending: boolean;
|
|
218
217
|
|
|
@@ -233,14 +232,14 @@ export class BlobManager {
|
|
|
233
232
|
*/
|
|
234
233
|
sendBlobAttachOp: (localId: string, storageId: string) => void;
|
|
235
234
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
236
|
-
// blobPath's format - `/<basePath>/<
|
|
235
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
237
236
|
readonly blobRequested: (blobPath: string) => void;
|
|
238
237
|
// Called to check if a blob has been deleted by GC.
|
|
239
|
-
// blobPath's format - `/<basePath>/<
|
|
238
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
240
239
|
readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
241
240
|
readonly runtime: IBlobManagerRuntime;
|
|
242
241
|
stashedBlobs: IPendingBlobs | undefined;
|
|
243
|
-
readonly
|
|
242
|
+
readonly localIdGenerator?: (() => string) | undefined;
|
|
244
243
|
readonly createBlobPayloadPending: boolean;
|
|
245
244
|
}) {
|
|
246
245
|
const {
|
|
@@ -251,7 +250,7 @@ export class BlobManager {
|
|
|
251
250
|
blobRequested,
|
|
252
251
|
isBlobDeleted,
|
|
253
252
|
runtime,
|
|
254
|
-
|
|
253
|
+
localIdGenerator,
|
|
255
254
|
createBlobPayloadPending,
|
|
256
255
|
} = props;
|
|
257
256
|
this.routeContext = routeContext;
|
|
@@ -259,7 +258,7 @@ export class BlobManager {
|
|
|
259
258
|
this.blobRequested = blobRequested;
|
|
260
259
|
this.isBlobDeleted = isBlobDeleted;
|
|
261
260
|
this.runtime = runtime;
|
|
262
|
-
this.
|
|
261
|
+
this.localIdGenerator = localIdGenerator ?? uuid;
|
|
263
262
|
this.createBlobPayloadPending = createBlobPayloadPending;
|
|
264
263
|
|
|
265
264
|
this.mc = createChildMonitoringContext({
|
|
@@ -267,13 +266,9 @@ export class BlobManager {
|
|
|
267
266
|
namespace: "BlobManager",
|
|
268
267
|
});
|
|
269
268
|
|
|
270
|
-
this.redirectTable = toRedirectTable(
|
|
271
|
-
blobManagerLoadInfo,
|
|
272
|
-
this.mc.logger,
|
|
273
|
-
this.runtime.attachState,
|
|
274
|
-
);
|
|
269
|
+
this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger);
|
|
275
270
|
|
|
276
|
-
this.sendBlobAttachOp = (localId: string,
|
|
271
|
+
this.sendBlobAttachOp = (localId: string, storageId: string) => {
|
|
277
272
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
278
273
|
assert(
|
|
279
274
|
pendingEntry !== undefined,
|
|
@@ -302,7 +297,7 @@ export class BlobManager {
|
|
|
302
297
|
}
|
|
303
298
|
}
|
|
304
299
|
pendingEntry.opsent = true;
|
|
305
|
-
sendBlobAttachOp(localId,
|
|
300
|
+
sendBlobAttachOp(localId, storageId);
|
|
306
301
|
};
|
|
307
302
|
}
|
|
308
303
|
|
|
@@ -329,59 +324,67 @@ export class BlobManager {
|
|
|
329
324
|
});
|
|
330
325
|
}
|
|
331
326
|
|
|
332
|
-
public hasBlob(
|
|
333
|
-
return this.redirectTable.get(
|
|
327
|
+
public hasBlob(localId: string): boolean {
|
|
328
|
+
return this.redirectTable.get(localId) !== undefined;
|
|
329
|
+
}
|
|
330
|
+
|
|
331
|
+
/**
|
|
332
|
+
* Lookup the blob storage ID for a given local blob id.
|
|
333
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
334
|
+
* @returns The storage ID if found and the blob is not pending, undefined otherwise.
|
|
335
|
+
* @remarks
|
|
336
|
+
* For blobs with pending payloads (localId exists but upload hasn't finished), this is expected to return undefined.
|
|
337
|
+
* Consumers should use the observability APIs on the handle (handle.payloadState, payloadShared event)
|
|
338
|
+
* to understand/wait for storage ID availability.
|
|
339
|
+
* Similarly, when the runtime is detached, this will return undefined as no blobs have been uploaded to storage.
|
|
340
|
+
*/
|
|
341
|
+
public lookupTemporaryBlobStorageId(localId: string): string | undefined {
|
|
342
|
+
if (this.runtime.attachState === AttachState.Detached) {
|
|
343
|
+
return undefined;
|
|
344
|
+
}
|
|
345
|
+
// Get the storage ID from the redirect table
|
|
346
|
+
return this.redirectTable.get(localId);
|
|
334
347
|
}
|
|
335
348
|
|
|
336
349
|
/**
|
|
337
350
|
* Retrieve the blob with the given local blob id.
|
|
338
|
-
* @param
|
|
351
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
339
352
|
* @param payloadPending - Whether we suspect the payload may be pending and not available yet.
|
|
340
353
|
* @returns A promise which resolves to the blob contents
|
|
341
354
|
*/
|
|
342
|
-
public async getBlob(
|
|
355
|
+
public async getBlob(localId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
|
|
343
356
|
// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
|
|
344
357
|
// an error, failing the call.
|
|
345
|
-
this.verifyBlobNotDeleted(
|
|
358
|
+
this.verifyBlobNotDeleted(localId);
|
|
346
359
|
// Let runtime know that the corresponding GC node was requested.
|
|
347
360
|
// Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
|
|
348
361
|
// is configured.
|
|
349
|
-
this.blobRequested(
|
|
362
|
+
this.blobRequested(getGCNodePathFromLocalId(localId));
|
|
350
363
|
|
|
351
|
-
const pending = this.pendingBlobs.get(
|
|
364
|
+
const pending = this.pendingBlobs.get(localId);
|
|
352
365
|
if (pending) {
|
|
353
366
|
return pending.blob;
|
|
354
367
|
}
|
|
355
368
|
|
|
356
|
-
let storageId
|
|
357
|
-
if (
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
//
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
attachedStorageId ??
|
|
376
|
-
(await new Promise<string>((resolve) => {
|
|
377
|
-
const onProcessBlobAttach = (localId: string, _storageId: string): void => {
|
|
378
|
-
if (localId === blobId) {
|
|
379
|
-
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
380
|
-
resolve(_storageId);
|
|
381
|
-
}
|
|
382
|
-
};
|
|
383
|
-
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
384
|
-
}));
|
|
369
|
+
let storageId = this.redirectTable.get(localId);
|
|
370
|
+
if (storageId === undefined) {
|
|
371
|
+
// Only blob handles explicitly marked with pending payload are permitted to exist without
|
|
372
|
+
// yet knowing their storage id. Otherwise they must already be associated with a storage id.
|
|
373
|
+
// Handles for detached blobs are not payload pending.
|
|
374
|
+
assert(payloadPending, 0x11f /* "requesting unknown blobs" */);
|
|
375
|
+
// If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
|
|
376
|
+
// eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
|
|
377
|
+
// enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
|
|
378
|
+
// client failed the upload and doesn't exist anymore.
|
|
379
|
+
storageId = await new Promise<string>((resolve) => {
|
|
380
|
+
const onProcessBlobAttach = (_localId: string, _storageId: string): void => {
|
|
381
|
+
if (_localId === localId) {
|
|
382
|
+
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
383
|
+
resolve(_storageId);
|
|
384
|
+
}
|
|
385
|
+
};
|
|
386
|
+
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
387
|
+
});
|
|
385
388
|
}
|
|
386
389
|
|
|
387
390
|
return PerformanceEvent.timedExecAsync(
|
|
@@ -417,7 +420,7 @@ export class BlobManager {
|
|
|
417
420
|
}
|
|
418
421
|
: undefined;
|
|
419
422
|
return new BlobHandle(
|
|
420
|
-
|
|
423
|
+
getGCNodePathFromLocalId(localId),
|
|
421
424
|
this.routeContext,
|
|
422
425
|
async () => this.getBlob(localId, false),
|
|
423
426
|
false, // payloadPending
|
|
@@ -428,11 +431,13 @@ export class BlobManager {
|
|
|
428
431
|
private async createBlobDetached(
|
|
429
432
|
blob: ArrayBufferLike,
|
|
430
433
|
): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
|
|
434
|
+
const localId = this.localIdGenerator();
|
|
431
435
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
432
|
-
// The 'IContainerStorageService.createBlob()' call below will respond with a
|
|
433
|
-
|
|
434
|
-
this.
|
|
435
|
-
|
|
436
|
+
// The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
|
|
437
|
+
// That pseudo storage ID will be replaced with the real storage ID at attach time.
|
|
438
|
+
const { id: detachedStorageId } = await this.storage.createBlob(blob);
|
|
439
|
+
this.setRedirection(localId, detachedStorageId);
|
|
440
|
+
return this.getBlobHandle(localId);
|
|
436
441
|
}
|
|
437
442
|
|
|
438
443
|
public async createBlob(
|
|
@@ -467,7 +472,7 @@ export class BlobManager {
|
|
|
467
472
|
|
|
468
473
|
// Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
|
|
469
474
|
// storage ID mapping is created.
|
|
470
|
-
const localId = this.
|
|
475
|
+
const localId = this.localIdGenerator();
|
|
471
476
|
const pendingEntry: PendingBlob = {
|
|
472
477
|
blob,
|
|
473
478
|
handleP: new Deferred(),
|
|
@@ -494,10 +499,10 @@ export class BlobManager {
|
|
|
494
499
|
private createBlobWithPayloadPending(
|
|
495
500
|
blob: ArrayBufferLike,
|
|
496
501
|
): IFluidHandleInternalPayloadPending<ArrayBufferLike> {
|
|
497
|
-
const localId = this.
|
|
502
|
+
const localId = this.localIdGenerator();
|
|
498
503
|
|
|
499
504
|
const blobHandle = new BlobHandle(
|
|
500
|
-
|
|
505
|
+
getGCNodePathFromLocalId(localId),
|
|
501
506
|
this.routeContext,
|
|
502
507
|
async () => blob,
|
|
503
508
|
true, // payloadPending
|
|
@@ -581,7 +586,7 @@ export class BlobManager {
|
|
|
581
586
|
* Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
|
|
582
587
|
* which is required for GC.
|
|
583
588
|
*/
|
|
584
|
-
private setRedirection(fromId: string, toId: string
|
|
589
|
+
private setRedirection(fromId: string, toId: string): void {
|
|
585
590
|
this.redirectTable.set(fromId, toId);
|
|
586
591
|
}
|
|
587
592
|
|
|
@@ -630,7 +635,7 @@ export class BlobManager {
|
|
|
630
635
|
if (!entry.opsent) {
|
|
631
636
|
this.sendBlobAttachOp(localId, response.id);
|
|
632
637
|
}
|
|
633
|
-
const storageIds = getStorageIds(this.redirectTable
|
|
638
|
+
const storageIds = getStorageIds(this.redirectTable);
|
|
634
639
|
if (storageIds.has(response.id)) {
|
|
635
640
|
// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
|
|
636
641
|
// an existing blob, we don't have to wait for the op to be ack'd since this step has already
|
|
@@ -663,7 +668,7 @@ export class BlobManager {
|
|
|
663
668
|
*/
|
|
664
669
|
public reSubmit(metadata: Record<string, unknown> | undefined): void {
|
|
665
670
|
assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
|
|
666
|
-
const { localId, blobId } = metadata;
|
|
671
|
+
const { localId, blobId: storageId } = metadata;
|
|
667
672
|
// Any blob that we're actively trying to advance to attached state must have a
|
|
668
673
|
// pendingBlobs entry. Decline to resubmit for anything else.
|
|
669
674
|
// For example, we might be asked to resubmit stashed ops for blobs that never had
|
|
@@ -671,7 +676,7 @@ export class BlobManager {
|
|
|
671
676
|
// try to attach them since they won't be accessible to the customer and would just
|
|
672
677
|
// be considered garbage immediately.
|
|
673
678
|
if (this.pendingBlobs.has(localId)) {
|
|
674
|
-
this.sendBlobAttachOp(localId,
|
|
679
|
+
this.sendBlobAttachOp(localId, storageId);
|
|
675
680
|
}
|
|
676
681
|
}
|
|
677
682
|
|
|
@@ -680,19 +685,19 @@ export class BlobManager {
|
|
|
680
685
|
isBlobMetadata(message.metadata),
|
|
681
686
|
0xc02 /* Expected blob metadata for a BlobAttach op */,
|
|
682
687
|
);
|
|
683
|
-
const { localId, blobId } = message.metadata;
|
|
688
|
+
const { localId, blobId: storageId } = message.metadata;
|
|
684
689
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
685
690
|
if (pendingEntry?.abortSignal?.aborted) {
|
|
686
691
|
this.deletePendingBlob(localId);
|
|
687
692
|
return;
|
|
688
693
|
}
|
|
689
694
|
|
|
690
|
-
this.setRedirection(localId,
|
|
695
|
+
this.setRedirection(localId, storageId);
|
|
691
696
|
// set identity (id -> id) entry
|
|
692
|
-
this.setRedirection(
|
|
697
|
+
this.setRedirection(storageId, storageId);
|
|
693
698
|
|
|
694
699
|
if (local) {
|
|
695
|
-
const waitingBlobs = this.opsInFlight.get(
|
|
700
|
+
const waitingBlobs = this.opsInFlight.get(storageId);
|
|
696
701
|
if (waitingBlobs !== undefined) {
|
|
697
702
|
// For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
|
|
698
703
|
// This is safe because the server will keep the blob alive and the op containing the local ID to
|
|
@@ -703,14 +708,14 @@ export class BlobManager {
|
|
|
703
708
|
entry !== undefined,
|
|
704
709
|
0x38f /* local online BlobAttach op with no pending blob entry */,
|
|
705
710
|
);
|
|
706
|
-
this.setRedirection(pendingLocalId,
|
|
711
|
+
this.setRedirection(pendingLocalId, storageId);
|
|
707
712
|
entry.acked = true;
|
|
708
713
|
const blobHandle = this.getBlobHandle(pendingLocalId);
|
|
709
714
|
blobHandle.notifyShared();
|
|
710
715
|
entry.handleP.resolve(blobHandle);
|
|
711
716
|
this.deletePendingBlobMaybe(pendingLocalId);
|
|
712
717
|
}
|
|
713
|
-
this.opsInFlight.delete(
|
|
718
|
+
this.opsInFlight.delete(storageId);
|
|
714
719
|
}
|
|
715
720
|
const localEntry = this.pendingBlobs.get(localId);
|
|
716
721
|
if (localEntry) {
|
|
@@ -721,11 +726,11 @@ export class BlobManager {
|
|
|
721
726
|
this.deletePendingBlobMaybe(localId);
|
|
722
727
|
}
|
|
723
728
|
}
|
|
724
|
-
this.internalEvents.emit("processedBlobAttach", localId,
|
|
729
|
+
this.internalEvents.emit("processedBlobAttach", localId, storageId);
|
|
725
730
|
}
|
|
726
731
|
|
|
727
732
|
public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
|
|
728
|
-
return summarizeBlobManagerState(this.redirectTable
|
|
733
|
+
return summarizeBlobManagerState(this.redirectTable);
|
|
729
734
|
}
|
|
730
735
|
|
|
731
736
|
/**
|
|
@@ -737,13 +742,12 @@ export class BlobManager {
|
|
|
737
742
|
public getGCData(fullGC: boolean = false): IGarbageCollectionData {
|
|
738
743
|
const gcData: IGarbageCollectionData = { gcNodes: {} };
|
|
739
744
|
for (const [localId, storageId] of this.redirectTable) {
|
|
740
|
-
|
|
741
|
-
//
|
|
742
|
-
// id entries have the same key and value, ignore them.
|
|
743
|
-
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
744
|
-
// by adding its handle to a referenced DDS.
|
|
745
|
+
// Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
|
|
746
|
+
// IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
|
|
745
747
|
if (localId !== storageId) {
|
|
746
|
-
|
|
748
|
+
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
749
|
+
// by adding its handle to a referenced DDS.
|
|
750
|
+
gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
|
|
747
751
|
}
|
|
748
752
|
}
|
|
749
753
|
return gcData;
|
|
@@ -764,58 +768,55 @@ export class BlobManager {
|
|
|
764
768
|
* Delete blobs with the given routes from the redirect table.
|
|
765
769
|
*
|
|
766
770
|
* @remarks
|
|
767
|
-
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<
|
|
771
|
+
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
|
|
768
772
|
* Deleting the blobs involves 2 steps:
|
|
769
773
|
*
|
|
770
774
|
* 1. The redirect table entry for the local ids are deleted.
|
|
771
775
|
*
|
|
772
|
-
* 2. If the storage ids corresponding to the deleted local ids are not
|
|
773
|
-
*
|
|
776
|
+
* 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
|
|
777
|
+
* identity mappings in the redirect table are deleted as well.
|
|
774
778
|
*
|
|
775
779
|
* Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
|
|
776
|
-
* will
|
|
780
|
+
* will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
|
|
781
|
+
* some time in the future.
|
|
777
782
|
*/
|
|
778
783
|
private deleteBlobsFromRedirectTable(blobRoutes: readonly string[]): void {
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
// This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
|
|
784
|
-
// these storage ids are unused, they will be deleted as well.
|
|
784
|
+
// maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
|
|
785
|
+
// local ID is being deleted.
|
|
785
786
|
const maybeUnusedStorageIds: Set<string> = new Set();
|
|
786
787
|
for (const route of blobRoutes) {
|
|
787
|
-
const
|
|
788
|
+
const localId = getLocalIdFromGCNodePath(route);
|
|
788
789
|
// If the blob hasn't already been deleted, log an error because this should never happen.
|
|
789
790
|
// If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
|
|
790
791
|
// sweep ops can contain the same data store. It would be interesting to track how often this happens.
|
|
791
792
|
const alreadyDeleted = this.isBlobDeleted(route);
|
|
792
|
-
|
|
793
|
+
const storageId = this.redirectTable.get(localId);
|
|
794
|
+
if (storageId === undefined) {
|
|
793
795
|
this.mc.logger.sendTelemetryEvent({
|
|
794
796
|
eventName: "DeletedAttachmentBlobNotFound",
|
|
795
797
|
category: alreadyDeleted ? "generic" : "error",
|
|
796
|
-
blobId,
|
|
798
|
+
blobId: localId,
|
|
797
799
|
details: { alreadyDeleted },
|
|
798
800
|
});
|
|
799
801
|
continue;
|
|
800
802
|
}
|
|
801
|
-
const storageId = this.redirectTable.get(blobId);
|
|
802
|
-
assert(!!storageId, 0x5bb /* Must be attached to run GC */);
|
|
803
803
|
maybeUnusedStorageIds.add(storageId);
|
|
804
|
-
this.redirectTable.delete(
|
|
804
|
+
this.redirectTable.delete(localId);
|
|
805
805
|
}
|
|
806
806
|
|
|
807
|
-
//
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
assert(!!storageId, 0x5bc /* Must be attached to run GC */);
|
|
811
|
-
// For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
|
|
812
|
-
if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
|
|
807
|
+
// Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
|
|
808
|
+
for (const [localId, storageId] of this.redirectTable) {
|
|
809
|
+
if (localId !== storageId) {
|
|
813
810
|
maybeUnusedStorageIds.delete(storageId);
|
|
814
811
|
}
|
|
815
812
|
}
|
|
816
813
|
|
|
817
|
-
//
|
|
818
|
-
// This way they'll be absent from the next summary, and the service
|
|
814
|
+
// Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
|
|
815
|
+
// referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
|
|
816
|
+
// is free to delete them from storage.
|
|
817
|
+
// WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
|
|
818
|
+
// has let us add a local ID -> storage ID mapping that is later deleted. AB#47337 tracks this issue
|
|
819
|
+
// and possible solutions.
|
|
819
820
|
for (const storageId of maybeUnusedStorageIds) {
|
|
820
821
|
this.redirectTable.delete(storageId);
|
|
821
822
|
}
|
|
@@ -825,12 +826,12 @@ export class BlobManager {
|
|
|
825
826
|
* Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
|
|
826
827
|
* log an error and throw if necessary.
|
|
827
828
|
*/
|
|
828
|
-
private verifyBlobNotDeleted(
|
|
829
|
-
if (!this.isBlobDeleted(
|
|
829
|
+
private verifyBlobNotDeleted(localId: string): void {
|
|
830
|
+
if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
|
|
830
831
|
return;
|
|
831
832
|
}
|
|
832
833
|
|
|
833
|
-
const request = { url:
|
|
834
|
+
const request = { url: localId };
|
|
834
835
|
const error = responseToException(
|
|
835
836
|
createResponseError(404, `Blob was deleted`, request),
|
|
836
837
|
request,
|
|
@@ -846,22 +847,36 @@ export class BlobManager {
|
|
|
846
847
|
throw error;
|
|
847
848
|
}
|
|
848
849
|
|
|
849
|
-
|
|
850
|
+
/**
|
|
851
|
+
* Called in detached state just prior to attaching, this will update the redirect table by
|
|
852
|
+
* converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
|
|
853
|
+
* The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
|
|
854
|
+
* @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
|
|
855
|
+
*/
|
|
856
|
+
public readonly patchRedirectTable = (detachedStorageTable: Map<string, string>): void => {
|
|
850
857
|
assert(
|
|
851
858
|
this.runtime.attachState === AttachState.Detached,
|
|
852
859
|
0x252 /* "redirect table can only be set in detached container" */,
|
|
853
860
|
);
|
|
861
|
+
// The values of the redirect table are the pseudo storage IDs, which are the keys of the
|
|
862
|
+
// detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
|
|
863
|
+
// storage IDs (many in the case that the storage dedupes the blob).
|
|
854
864
|
assert(
|
|
855
|
-
this.redirectTable.size ===
|
|
865
|
+
new Set(this.redirectTable.values()).size === detachedStorageTable.size,
|
|
856
866
|
0x391 /* Redirect table size must match BlobManager's local ID count */,
|
|
857
867
|
);
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
868
|
+
// Taking a snapshot of the redirect table entries before iterating, because
|
|
869
|
+
// we will be adding identity mappings to the the redirect table as we iterate
|
|
870
|
+
// and we don't want to include those in the iteration.
|
|
871
|
+
const redirectTableEntries = [...this.redirectTable.entries()];
|
|
872
|
+
for (const [localId, detachedStorageId] of redirectTableEntries) {
|
|
873
|
+
const newStorageId = detachedStorageTable.get(detachedStorageId);
|
|
874
|
+
assert(newStorageId !== undefined, "Couldn't find a matching storage ID");
|
|
875
|
+
this.setRedirection(localId, newStorageId);
|
|
861
876
|
// set identity (id -> id) entry
|
|
862
|
-
this.setRedirection(
|
|
877
|
+
this.setRedirection(newStorageId, newStorageId);
|
|
863
878
|
}
|
|
864
|
-
}
|
|
879
|
+
};
|
|
865
880
|
|
|
866
881
|
/**
|
|
867
882
|
* To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
|
|
@@ -896,17 +911,17 @@ export class BlobManager {
|
|
|
896
911
|
}
|
|
897
912
|
|
|
898
913
|
/**
|
|
899
|
-
* For a
|
|
914
|
+
* For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
|
|
900
915
|
* This path must match the path of the blob handle returned by the createBlob API because blobs are marked
|
|
901
916
|
* referenced by storing these handles in a referenced DDS.
|
|
902
917
|
*/
|
|
903
|
-
const
|
|
904
|
-
`/${blobManagerBasePath}/${
|
|
918
|
+
const getGCNodePathFromLocalId = (localId: string): string =>
|
|
919
|
+
`/${blobManagerBasePath}/${localId}`;
|
|
905
920
|
|
|
906
921
|
/**
|
|
907
|
-
* For a given GC node path, return the
|
|
922
|
+
* For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
|
|
908
923
|
*/
|
|
909
|
-
const
|
|
924
|
+
const getLocalIdFromGCNodePath = (nodePath: string): string => {
|
|
910
925
|
const pathParts = nodePath.split("/");
|
|
911
926
|
assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
|
|
912
927
|
return pathParts[2];
|