@fluidframework/container-runtime 2.60.0 → 2.61.0-355054
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.mocharc.cjs +1 -2
- package/container-runtime.test-files.tar +0 -0
- package/dist/blobManager/blobManager.d.ts +21 -15
- package/dist/blobManager/blobManager.d.ts.map +1 -1
- package/dist/blobManager/blobManager.js +105 -102
- package/dist/blobManager/blobManager.js.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/dist/blobManager/blobManagerSnapSum.js +30 -33
- package/dist/blobManager/blobManagerSnapSum.js.map +1 -1
- package/dist/containerRuntime.js +1 -1
- package/dist/containerRuntime.js.map +1 -1
- package/dist/legacy.d.ts +2 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/internal.d.ts +1 -1
- package/legacy.d.ts +1 -1
- package/lib/blobManager/blobManager.d.ts +21 -15
- package/lib/blobManager/blobManager.d.ts.map +1 -1
- package/lib/blobManager/blobManager.js +105 -102
- package/lib/blobManager/blobManager.js.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.d.ts +4 -4
- package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -1
- package/lib/blobManager/blobManagerSnapSum.js +26 -29
- package/lib/blobManager/blobManagerSnapSum.js.map +1 -1
- package/lib/containerRuntime.js +1 -1
- package/lib/containerRuntime.js.map +1 -1
- package/lib/legacy.d.ts +2 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/tsdoc-metadata.json +1 -1
- package/package.json +26 -26
- package/src/blobManager/blobManager.ts +118 -121
- package/src/blobManager/blobManagerSnapSum.ts +31 -53
- package/src/containerRuntime.ts +1 -1
- package/src/packageVersion.ts +1 -1
package/lib/legacy.d.ts
CHANGED
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
*/
|
|
10
10
|
|
|
11
11
|
export {
|
|
12
|
-
// @
|
|
12
|
+
// #region @legacyBeta APIs
|
|
13
13
|
AllowTombstoneRequestHeaderKey,
|
|
14
14
|
CompressionAlgorithms,
|
|
15
15
|
ContainerMessageType,
|
|
@@ -65,4 +65,5 @@ export {
|
|
|
65
65
|
TombstoneResponseHeaderKey,
|
|
66
66
|
disabledCompressionConfig,
|
|
67
67
|
loadContainerRuntime
|
|
68
|
+
// #endregion
|
|
68
69
|
} from "./index.js";
|
package/lib/packageVersion.d.ts
CHANGED
|
@@ -5,5 +5,5 @@
|
|
|
5
5
|
* THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
|
|
6
6
|
*/
|
|
7
7
|
export declare const pkgName = "@fluidframework/container-runtime";
|
|
8
|
-
export declare const pkgVersion = "2.
|
|
8
|
+
export declare const pkgVersion = "2.61.0-355054";
|
|
9
9
|
//# sourceMappingURL=packageVersion.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,
|
|
1
|
+
{"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,kBAAkB,CAAC"}
|
package/lib/packageVersion.js
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,
|
|
1
|
+
{"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,eAAe,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.61.0-355054\";\n"]}
|
package/lib/tsdoc-metadata.json
CHANGED
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluidframework/container-runtime",
|
|
3
|
-
"version": "2.
|
|
3
|
+
"version": "2.61.0-355054",
|
|
4
4
|
"description": "Fluid container runtime",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": {
|
|
@@ -119,18 +119,18 @@
|
|
|
119
119
|
"temp-directory": "nyc/.nyc_output"
|
|
120
120
|
},
|
|
121
121
|
"dependencies": {
|
|
122
|
-
"@fluid-internal/client-utils": "
|
|
123
|
-
"@fluidframework/container-definitions": "
|
|
124
|
-
"@fluidframework/container-runtime-definitions": "
|
|
125
|
-
"@fluidframework/core-interfaces": "
|
|
126
|
-
"@fluidframework/core-utils": "
|
|
127
|
-
"@fluidframework/datastore": "
|
|
128
|
-
"@fluidframework/driver-definitions": "
|
|
129
|
-
"@fluidframework/driver-utils": "
|
|
130
|
-
"@fluidframework/id-compressor": "
|
|
131
|
-
"@fluidframework/runtime-definitions": "
|
|
132
|
-
"@fluidframework/runtime-utils": "
|
|
133
|
-
"@fluidframework/telemetry-utils": "
|
|
122
|
+
"@fluid-internal/client-utils": "2.61.0-355054",
|
|
123
|
+
"@fluidframework/container-definitions": "2.61.0-355054",
|
|
124
|
+
"@fluidframework/container-runtime-definitions": "2.61.0-355054",
|
|
125
|
+
"@fluidframework/core-interfaces": "2.61.0-355054",
|
|
126
|
+
"@fluidframework/core-utils": "2.61.0-355054",
|
|
127
|
+
"@fluidframework/datastore": "2.61.0-355054",
|
|
128
|
+
"@fluidframework/driver-definitions": "2.61.0-355054",
|
|
129
|
+
"@fluidframework/driver-utils": "2.61.0-355054",
|
|
130
|
+
"@fluidframework/id-compressor": "2.61.0-355054",
|
|
131
|
+
"@fluidframework/runtime-definitions": "2.61.0-355054",
|
|
132
|
+
"@fluidframework/runtime-utils": "2.61.0-355054",
|
|
133
|
+
"@fluidframework/telemetry-utils": "2.61.0-355054",
|
|
134
134
|
"@tylerbu/sorted-btree-es6": "^1.8.0",
|
|
135
135
|
"double-ended-queue": "^2.1.0-0",
|
|
136
136
|
"lz4js": "^0.2.0",
|
|
@@ -140,23 +140,23 @@
|
|
|
140
140
|
"devDependencies": {
|
|
141
141
|
"@arethetypeswrong/cli": "^0.17.1",
|
|
142
142
|
"@biomejs/biome": "~1.9.3",
|
|
143
|
-
"@fluid-internal/mocha-test-setup": "
|
|
144
|
-
"@fluid-private/stochastic-test-utils": "
|
|
145
|
-
"@fluid-private/test-pairwise-generator": "
|
|
143
|
+
"@fluid-internal/mocha-test-setup": "2.61.0-355054",
|
|
144
|
+
"@fluid-private/stochastic-test-utils": "2.61.0-355054",
|
|
145
|
+
"@fluid-private/test-pairwise-generator": "2.61.0-355054",
|
|
146
146
|
"@fluid-tools/benchmark": "^0.51.0",
|
|
147
|
-
"@fluid-tools/build-cli": "^0.
|
|
147
|
+
"@fluid-tools/build-cli": "^0.58.2",
|
|
148
148
|
"@fluidframework/build-common": "^2.0.3",
|
|
149
|
-
"@fluidframework/build-tools": "^0.
|
|
150
|
-
"@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.
|
|
149
|
+
"@fluidframework/build-tools": "^0.58.2",
|
|
150
|
+
"@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.60.0",
|
|
151
151
|
"@fluidframework/eslint-config-fluid": "^6.0.0",
|
|
152
|
-
"@fluidframework/test-runtime-utils": "
|
|
153
|
-
"@microsoft/api-extractor": "7.52.
|
|
152
|
+
"@fluidframework/test-runtime-utils": "2.61.0-355054",
|
|
153
|
+
"@microsoft/api-extractor": "7.52.11",
|
|
154
154
|
"@types/double-ended-queue": "^2.1.0",
|
|
155
155
|
"@types/lz4js": "^0.2.0",
|
|
156
156
|
"@types/mocha": "^10.0.10",
|
|
157
157
|
"@types/node": "^18.19.0",
|
|
158
158
|
"@types/sinon": "^17.0.3",
|
|
159
|
-
"c8": "^
|
|
159
|
+
"c8": "^10.1.3",
|
|
160
160
|
"concurrently": "^8.2.1",
|
|
161
161
|
"copyfiles": "^2.4.1",
|
|
162
162
|
"cross-env": "^7.0.3",
|
|
@@ -173,8 +173,8 @@
|
|
|
173
173
|
},
|
|
174
174
|
"scripts": {
|
|
175
175
|
"api": "fluid-build . --task api",
|
|
176
|
-
"api-extractor:commonjs": "flub generate entrypoints --outDir ./dist",
|
|
177
|
-
"api-extractor:esnext": "flub generate entrypoints --outDir ./lib --node10TypeCompat",
|
|
176
|
+
"api-extractor:commonjs": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./dist",
|
|
177
|
+
"api-extractor:esnext": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./lib --node10TypeCompat",
|
|
178
178
|
"build": "fluid-build . --task build",
|
|
179
179
|
"build:api-reports": "concurrently \"npm:build:api-reports:*\"",
|
|
180
180
|
"build:api-reports:current": "api-extractor run --local --config api-extractor/api-extractor.current.json",
|
|
@@ -213,8 +213,8 @@
|
|
|
213
213
|
"test:benchmark:report": "mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js \"./dist/**/*.perf.spec.*js\"",
|
|
214
214
|
"test:coverage": "c8 npm test",
|
|
215
215
|
"test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",
|
|
216
|
-
"test:mocha:cjs": "
|
|
217
|
-
"test:mocha:esm": "mocha
|
|
216
|
+
"test:mocha:cjs": "cross-env MOCHA_SPEC=dist/test mocha",
|
|
217
|
+
"test:mocha:esm": "mocha",
|
|
218
218
|
"test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
|
|
219
219
|
"tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && npm run place:cjs:package-stub",
|
|
220
220
|
"tsc:watch": "npm run place:cjs:package-stub && fluid-tsc commonjs --project ./tsconfig.cjs.json --watch",
|
|
@@ -181,13 +181,12 @@ export class BlobManager {
|
|
|
181
181
|
private readonly internalEvents = createEmitter<IBlobManagerInternalEvents>();
|
|
182
182
|
|
|
183
183
|
/**
|
|
184
|
-
* Map of local IDs to storage IDs.
|
|
185
|
-
* be a key in this map. Blobs created while the container is detached are
|
|
186
|
-
* gives
|
|
187
|
-
*
|
|
188
|
-
* that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
|
|
184
|
+
* Map of local IDs to storage IDs. Also includes identity mappings of storage ID to storage ID for all known
|
|
185
|
+
* storage IDs. All requested IDs must be a key in this map. Blobs created while the container is detached are
|
|
186
|
+
* stored in IDetachedBlobStorage which gives pseudo storage IDs; the real storage IDs are filled in at attach
|
|
187
|
+
* time via setRedirectTable().
|
|
189
188
|
*/
|
|
190
|
-
private readonly redirectTable: Map<string, string
|
|
189
|
+
private readonly redirectTable: Map<string, string>;
|
|
191
190
|
|
|
192
191
|
/**
|
|
193
192
|
* Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
|
|
@@ -206,13 +205,13 @@ export class BlobManager {
|
|
|
206
205
|
private readonly routeContext: IFluidHandleContext;
|
|
207
206
|
private readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
|
|
208
207
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
209
|
-
// blobPath's format - `/<basePath>/<
|
|
208
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
210
209
|
private readonly blobRequested: (blobPath: string) => void;
|
|
211
210
|
// Called to check if a blob has been deleted by GC.
|
|
212
|
-
// blobPath's format - `/<basePath>/<
|
|
211
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
213
212
|
private readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
214
213
|
private readonly runtime: IBlobManagerRuntime;
|
|
215
|
-
private readonly
|
|
214
|
+
private readonly localIdGenerator: () => string;
|
|
216
215
|
|
|
217
216
|
private readonly createBlobPayloadPending: boolean;
|
|
218
217
|
|
|
@@ -233,14 +232,14 @@ export class BlobManager {
|
|
|
233
232
|
*/
|
|
234
233
|
sendBlobAttachOp: (localId: string, storageId: string) => void;
|
|
235
234
|
// Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
|
|
236
|
-
// blobPath's format - `/<basePath>/<
|
|
235
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
237
236
|
readonly blobRequested: (blobPath: string) => void;
|
|
238
237
|
// Called to check if a blob has been deleted by GC.
|
|
239
|
-
// blobPath's format - `/<basePath>/<
|
|
238
|
+
// blobPath's format - `/<basePath>/<localId>`.
|
|
240
239
|
readonly isBlobDeleted: (blobPath: string) => boolean;
|
|
241
240
|
readonly runtime: IBlobManagerRuntime;
|
|
242
241
|
stashedBlobs: IPendingBlobs | undefined;
|
|
243
|
-
readonly
|
|
242
|
+
readonly localIdGenerator?: (() => string) | undefined;
|
|
244
243
|
readonly createBlobPayloadPending: boolean;
|
|
245
244
|
}) {
|
|
246
245
|
const {
|
|
@@ -251,7 +250,7 @@ export class BlobManager {
|
|
|
251
250
|
blobRequested,
|
|
252
251
|
isBlobDeleted,
|
|
253
252
|
runtime,
|
|
254
|
-
|
|
253
|
+
localIdGenerator,
|
|
255
254
|
createBlobPayloadPending,
|
|
256
255
|
} = props;
|
|
257
256
|
this.routeContext = routeContext;
|
|
@@ -259,7 +258,7 @@ export class BlobManager {
|
|
|
259
258
|
this.blobRequested = blobRequested;
|
|
260
259
|
this.isBlobDeleted = isBlobDeleted;
|
|
261
260
|
this.runtime = runtime;
|
|
262
|
-
this.
|
|
261
|
+
this.localIdGenerator = localIdGenerator ?? uuid;
|
|
263
262
|
this.createBlobPayloadPending = createBlobPayloadPending;
|
|
264
263
|
|
|
265
264
|
this.mc = createChildMonitoringContext({
|
|
@@ -267,13 +266,9 @@ export class BlobManager {
|
|
|
267
266
|
namespace: "BlobManager",
|
|
268
267
|
});
|
|
269
268
|
|
|
270
|
-
this.redirectTable = toRedirectTable(
|
|
271
|
-
blobManagerLoadInfo,
|
|
272
|
-
this.mc.logger,
|
|
273
|
-
this.runtime.attachState,
|
|
274
|
-
);
|
|
269
|
+
this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger);
|
|
275
270
|
|
|
276
|
-
this.sendBlobAttachOp = (localId: string,
|
|
271
|
+
this.sendBlobAttachOp = (localId: string, storageId: string) => {
|
|
277
272
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
278
273
|
assert(
|
|
279
274
|
pendingEntry !== undefined,
|
|
@@ -302,7 +297,7 @@ export class BlobManager {
|
|
|
302
297
|
}
|
|
303
298
|
}
|
|
304
299
|
pendingEntry.opsent = true;
|
|
305
|
-
sendBlobAttachOp(localId,
|
|
300
|
+
sendBlobAttachOp(localId, storageId);
|
|
306
301
|
};
|
|
307
302
|
}
|
|
308
303
|
|
|
@@ -329,59 +324,49 @@ export class BlobManager {
|
|
|
329
324
|
});
|
|
330
325
|
}
|
|
331
326
|
|
|
332
|
-
public hasBlob(
|
|
333
|
-
return this.redirectTable.get(
|
|
327
|
+
public hasBlob(localId: string): boolean {
|
|
328
|
+
return this.redirectTable.get(localId) !== undefined;
|
|
334
329
|
}
|
|
335
330
|
|
|
336
331
|
/**
|
|
337
332
|
* Retrieve the blob with the given local blob id.
|
|
338
|
-
* @param
|
|
333
|
+
* @param localId - The local blob id. Likely coming from a handle.
|
|
339
334
|
* @param payloadPending - Whether we suspect the payload may be pending and not available yet.
|
|
340
335
|
* @returns A promise which resolves to the blob contents
|
|
341
336
|
*/
|
|
342
|
-
public async getBlob(
|
|
337
|
+
public async getBlob(localId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
|
|
343
338
|
// Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
|
|
344
339
|
// an error, failing the call.
|
|
345
|
-
this.verifyBlobNotDeleted(
|
|
340
|
+
this.verifyBlobNotDeleted(localId);
|
|
346
341
|
// Let runtime know that the corresponding GC node was requested.
|
|
347
342
|
// Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
|
|
348
343
|
// is configured.
|
|
349
|
-
this.blobRequested(
|
|
344
|
+
this.blobRequested(getGCNodePathFromLocalId(localId));
|
|
350
345
|
|
|
351
|
-
const pending = this.pendingBlobs.get(
|
|
346
|
+
const pending = this.pendingBlobs.get(localId);
|
|
352
347
|
if (pending) {
|
|
353
348
|
return pending.blob;
|
|
354
349
|
}
|
|
355
350
|
|
|
356
|
-
let storageId
|
|
357
|
-
if (
|
|
358
|
-
|
|
359
|
-
|
|
360
|
-
//
|
|
361
|
-
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
|
|
365
|
-
|
|
366
|
-
|
|
367
|
-
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
attachedStorageId ??
|
|
376
|
-
(await new Promise<string>((resolve) => {
|
|
377
|
-
const onProcessBlobAttach = (localId: string, _storageId: string): void => {
|
|
378
|
-
if (localId === blobId) {
|
|
379
|
-
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
380
|
-
resolve(_storageId);
|
|
381
|
-
}
|
|
382
|
-
};
|
|
383
|
-
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
384
|
-
}));
|
|
351
|
+
let storageId = this.redirectTable.get(localId);
|
|
352
|
+
if (storageId === undefined) {
|
|
353
|
+
// Only blob handles explicitly marked with pending payload are permitted to exist without
|
|
354
|
+
// yet knowing their storage id. Otherwise they must already be associated with a storage id.
|
|
355
|
+
// Handles for detached blobs are not payload pending.
|
|
356
|
+
assert(payloadPending, 0x11f /* "requesting unknown blobs" */);
|
|
357
|
+
// If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
|
|
358
|
+
// eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
|
|
359
|
+
// enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
|
|
360
|
+
// client failed the upload and doesn't exist anymore.
|
|
361
|
+
storageId = await new Promise<string>((resolve) => {
|
|
362
|
+
const onProcessBlobAttach = (_localId: string, _storageId: string): void => {
|
|
363
|
+
if (_localId === localId) {
|
|
364
|
+
this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
|
|
365
|
+
resolve(_storageId);
|
|
366
|
+
}
|
|
367
|
+
};
|
|
368
|
+
this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
|
|
369
|
+
});
|
|
385
370
|
}
|
|
386
371
|
|
|
387
372
|
return PerformanceEvent.timedExecAsync(
|
|
@@ -417,7 +402,7 @@ export class BlobManager {
|
|
|
417
402
|
}
|
|
418
403
|
: undefined;
|
|
419
404
|
return new BlobHandle(
|
|
420
|
-
|
|
405
|
+
getGCNodePathFromLocalId(localId),
|
|
421
406
|
this.routeContext,
|
|
422
407
|
async () => this.getBlob(localId, false),
|
|
423
408
|
false, // payloadPending
|
|
@@ -428,11 +413,13 @@ export class BlobManager {
|
|
|
428
413
|
private async createBlobDetached(
|
|
429
414
|
blob: ArrayBufferLike,
|
|
430
415
|
): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
|
|
416
|
+
const localId = this.localIdGenerator();
|
|
431
417
|
// Blobs created while the container is detached are stored in IDetachedBlobStorage.
|
|
432
|
-
// The 'IContainerStorageService.createBlob()' call below will respond with a
|
|
433
|
-
|
|
434
|
-
this.
|
|
435
|
-
|
|
418
|
+
// The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
|
|
419
|
+
// That pseudo storage ID will be replaced with the real storage ID at attach time.
|
|
420
|
+
const { id: detachedStorageId } = await this.storage.createBlob(blob);
|
|
421
|
+
this.setRedirection(localId, detachedStorageId);
|
|
422
|
+
return this.getBlobHandle(localId);
|
|
436
423
|
}
|
|
437
424
|
|
|
438
425
|
public async createBlob(
|
|
@@ -467,7 +454,7 @@ export class BlobManager {
|
|
|
467
454
|
|
|
468
455
|
// Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
|
|
469
456
|
// storage ID mapping is created.
|
|
470
|
-
const localId = this.
|
|
457
|
+
const localId = this.localIdGenerator();
|
|
471
458
|
const pendingEntry: PendingBlob = {
|
|
472
459
|
blob,
|
|
473
460
|
handleP: new Deferred(),
|
|
@@ -494,10 +481,10 @@ export class BlobManager {
|
|
|
494
481
|
private createBlobWithPayloadPending(
|
|
495
482
|
blob: ArrayBufferLike,
|
|
496
483
|
): IFluidHandleInternalPayloadPending<ArrayBufferLike> {
|
|
497
|
-
const localId = this.
|
|
484
|
+
const localId = this.localIdGenerator();
|
|
498
485
|
|
|
499
486
|
const blobHandle = new BlobHandle(
|
|
500
|
-
|
|
487
|
+
getGCNodePathFromLocalId(localId),
|
|
501
488
|
this.routeContext,
|
|
502
489
|
async () => blob,
|
|
503
490
|
true, // payloadPending
|
|
@@ -581,7 +568,7 @@ export class BlobManager {
|
|
|
581
568
|
* Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
|
|
582
569
|
* which is required for GC.
|
|
583
570
|
*/
|
|
584
|
-
private setRedirection(fromId: string, toId: string
|
|
571
|
+
private setRedirection(fromId: string, toId: string): void {
|
|
585
572
|
this.redirectTable.set(fromId, toId);
|
|
586
573
|
}
|
|
587
574
|
|
|
@@ -630,7 +617,7 @@ export class BlobManager {
|
|
|
630
617
|
if (!entry.opsent) {
|
|
631
618
|
this.sendBlobAttachOp(localId, response.id);
|
|
632
619
|
}
|
|
633
|
-
const storageIds = getStorageIds(this.redirectTable
|
|
620
|
+
const storageIds = getStorageIds(this.redirectTable);
|
|
634
621
|
if (storageIds.has(response.id)) {
|
|
635
622
|
// The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
|
|
636
623
|
// an existing blob, we don't have to wait for the op to be ack'd since this step has already
|
|
@@ -663,7 +650,7 @@ export class BlobManager {
|
|
|
663
650
|
*/
|
|
664
651
|
public reSubmit(metadata: Record<string, unknown> | undefined): void {
|
|
665
652
|
assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
|
|
666
|
-
const { localId, blobId } = metadata;
|
|
653
|
+
const { localId, blobId: storageId } = metadata;
|
|
667
654
|
// Any blob that we're actively trying to advance to attached state must have a
|
|
668
655
|
// pendingBlobs entry. Decline to resubmit for anything else.
|
|
669
656
|
// For example, we might be asked to resubmit stashed ops for blobs that never had
|
|
@@ -671,7 +658,7 @@ export class BlobManager {
|
|
|
671
658
|
// try to attach them since they won't be accessible to the customer and would just
|
|
672
659
|
// be considered garbage immediately.
|
|
673
660
|
if (this.pendingBlobs.has(localId)) {
|
|
674
|
-
this.sendBlobAttachOp(localId,
|
|
661
|
+
this.sendBlobAttachOp(localId, storageId);
|
|
675
662
|
}
|
|
676
663
|
}
|
|
677
664
|
|
|
@@ -680,19 +667,19 @@ export class BlobManager {
|
|
|
680
667
|
isBlobMetadata(message.metadata),
|
|
681
668
|
0xc02 /* Expected blob metadata for a BlobAttach op */,
|
|
682
669
|
);
|
|
683
|
-
const { localId, blobId } = message.metadata;
|
|
670
|
+
const { localId, blobId: storageId } = message.metadata;
|
|
684
671
|
const pendingEntry = this.pendingBlobs.get(localId);
|
|
685
672
|
if (pendingEntry?.abortSignal?.aborted) {
|
|
686
673
|
this.deletePendingBlob(localId);
|
|
687
674
|
return;
|
|
688
675
|
}
|
|
689
676
|
|
|
690
|
-
this.setRedirection(localId,
|
|
677
|
+
this.setRedirection(localId, storageId);
|
|
691
678
|
// set identity (id -> id) entry
|
|
692
|
-
this.setRedirection(
|
|
679
|
+
this.setRedirection(storageId, storageId);
|
|
693
680
|
|
|
694
681
|
if (local) {
|
|
695
|
-
const waitingBlobs = this.opsInFlight.get(
|
|
682
|
+
const waitingBlobs = this.opsInFlight.get(storageId);
|
|
696
683
|
if (waitingBlobs !== undefined) {
|
|
697
684
|
// For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
|
|
698
685
|
// This is safe because the server will keep the blob alive and the op containing the local ID to
|
|
@@ -703,14 +690,14 @@ export class BlobManager {
|
|
|
703
690
|
entry !== undefined,
|
|
704
691
|
0x38f /* local online BlobAttach op with no pending blob entry */,
|
|
705
692
|
);
|
|
706
|
-
this.setRedirection(pendingLocalId,
|
|
693
|
+
this.setRedirection(pendingLocalId, storageId);
|
|
707
694
|
entry.acked = true;
|
|
708
695
|
const blobHandle = this.getBlobHandle(pendingLocalId);
|
|
709
696
|
blobHandle.notifyShared();
|
|
710
697
|
entry.handleP.resolve(blobHandle);
|
|
711
698
|
this.deletePendingBlobMaybe(pendingLocalId);
|
|
712
699
|
}
|
|
713
|
-
this.opsInFlight.delete(
|
|
700
|
+
this.opsInFlight.delete(storageId);
|
|
714
701
|
}
|
|
715
702
|
const localEntry = this.pendingBlobs.get(localId);
|
|
716
703
|
if (localEntry) {
|
|
@@ -721,11 +708,11 @@ export class BlobManager {
|
|
|
721
708
|
this.deletePendingBlobMaybe(localId);
|
|
722
709
|
}
|
|
723
710
|
}
|
|
724
|
-
this.internalEvents.emit("processedBlobAttach", localId,
|
|
711
|
+
this.internalEvents.emit("processedBlobAttach", localId, storageId);
|
|
725
712
|
}
|
|
726
713
|
|
|
727
714
|
public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
|
|
728
|
-
return summarizeBlobManagerState(this.redirectTable
|
|
715
|
+
return summarizeBlobManagerState(this.redirectTable);
|
|
729
716
|
}
|
|
730
717
|
|
|
731
718
|
/**
|
|
@@ -737,13 +724,12 @@ export class BlobManager {
|
|
|
737
724
|
public getGCData(fullGC: boolean = false): IGarbageCollectionData {
|
|
738
725
|
const gcData: IGarbageCollectionData = { gcNodes: {} };
|
|
739
726
|
for (const [localId, storageId] of this.redirectTable) {
|
|
740
|
-
|
|
741
|
-
//
|
|
742
|
-
// id entries have the same key and value, ignore them.
|
|
743
|
-
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
744
|
-
// by adding its handle to a referenced DDS.
|
|
727
|
+
// Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
|
|
728
|
+
// IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
|
|
745
729
|
if (localId !== storageId) {
|
|
746
|
-
|
|
730
|
+
// The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
|
|
731
|
+
// by adding its handle to a referenced DDS.
|
|
732
|
+
gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
|
|
747
733
|
}
|
|
748
734
|
}
|
|
749
735
|
return gcData;
|
|
@@ -764,58 +750,55 @@ export class BlobManager {
|
|
|
764
750
|
* Delete blobs with the given routes from the redirect table.
|
|
765
751
|
*
|
|
766
752
|
* @remarks
|
|
767
|
-
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<
|
|
753
|
+
* The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
|
|
768
754
|
* Deleting the blobs involves 2 steps:
|
|
769
755
|
*
|
|
770
756
|
* 1. The redirect table entry for the local ids are deleted.
|
|
771
757
|
*
|
|
772
|
-
* 2. If the storage ids corresponding to the deleted local ids are not
|
|
773
|
-
*
|
|
758
|
+
* 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
|
|
759
|
+
* identity mappings in the redirect table are deleted as well.
|
|
774
760
|
*
|
|
775
761
|
* Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
|
|
776
|
-
* will
|
|
762
|
+
* will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
|
|
763
|
+
* some time in the future.
|
|
777
764
|
*/
|
|
778
765
|
private deleteBlobsFromRedirectTable(blobRoutes: readonly string[]): void {
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
}
|
|
782
|
-
|
|
783
|
-
// This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
|
|
784
|
-
// these storage ids are unused, they will be deleted as well.
|
|
766
|
+
// maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
|
|
767
|
+
// local ID is being deleted.
|
|
785
768
|
const maybeUnusedStorageIds: Set<string> = new Set();
|
|
786
769
|
for (const route of blobRoutes) {
|
|
787
|
-
const
|
|
770
|
+
const localId = getLocalIdFromGCNodePath(route);
|
|
788
771
|
// If the blob hasn't already been deleted, log an error because this should never happen.
|
|
789
772
|
// If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
|
|
790
773
|
// sweep ops can contain the same data store. It would be interesting to track how often this happens.
|
|
791
774
|
const alreadyDeleted = this.isBlobDeleted(route);
|
|
792
|
-
|
|
775
|
+
const storageId = this.redirectTable.get(localId);
|
|
776
|
+
if (storageId === undefined) {
|
|
793
777
|
this.mc.logger.sendTelemetryEvent({
|
|
794
778
|
eventName: "DeletedAttachmentBlobNotFound",
|
|
795
779
|
category: alreadyDeleted ? "generic" : "error",
|
|
796
|
-
blobId,
|
|
780
|
+
blobId: localId,
|
|
797
781
|
details: { alreadyDeleted },
|
|
798
782
|
});
|
|
799
783
|
continue;
|
|
800
784
|
}
|
|
801
|
-
const storageId = this.redirectTable.get(blobId);
|
|
802
|
-
assert(!!storageId, 0x5bb /* Must be attached to run GC */);
|
|
803
785
|
maybeUnusedStorageIds.add(storageId);
|
|
804
|
-
this.redirectTable.delete(
|
|
786
|
+
this.redirectTable.delete(localId);
|
|
805
787
|
}
|
|
806
788
|
|
|
807
|
-
//
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
assert(!!storageId, 0x5bc /* Must be attached to run GC */);
|
|
811
|
-
// For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
|
|
812
|
-
if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
|
|
789
|
+
// Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
|
|
790
|
+
for (const [localId, storageId] of this.redirectTable) {
|
|
791
|
+
if (localId !== storageId) {
|
|
813
792
|
maybeUnusedStorageIds.delete(storageId);
|
|
814
793
|
}
|
|
815
794
|
}
|
|
816
795
|
|
|
817
|
-
//
|
|
818
|
-
// This way they'll be absent from the next summary, and the service
|
|
796
|
+
// Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
|
|
797
|
+
// referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
|
|
798
|
+
// is free to delete them from storage.
|
|
799
|
+
// WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
|
|
800
|
+
// has let us add a local ID -> storage ID mapping that is later deleted. AB#47337 tracks this issue
|
|
801
|
+
// and possible solutions.
|
|
819
802
|
for (const storageId of maybeUnusedStorageIds) {
|
|
820
803
|
this.redirectTable.delete(storageId);
|
|
821
804
|
}
|
|
@@ -825,12 +808,12 @@ export class BlobManager {
|
|
|
825
808
|
* Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
|
|
826
809
|
* log an error and throw if necessary.
|
|
827
810
|
*/
|
|
828
|
-
private verifyBlobNotDeleted(
|
|
829
|
-
if (!this.isBlobDeleted(
|
|
811
|
+
private verifyBlobNotDeleted(localId: string): void {
|
|
812
|
+
if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
|
|
830
813
|
return;
|
|
831
814
|
}
|
|
832
815
|
|
|
833
|
-
const request = { url:
|
|
816
|
+
const request = { url: localId };
|
|
834
817
|
const error = responseToException(
|
|
835
818
|
createResponseError(404, `Blob was deleted`, request),
|
|
836
819
|
request,
|
|
@@ -846,20 +829,34 @@ export class BlobManager {
|
|
|
846
829
|
throw error;
|
|
847
830
|
}
|
|
848
831
|
|
|
849
|
-
|
|
832
|
+
/**
|
|
833
|
+
* Called in detached state just prior to attaching, this will update the redirect table by
|
|
834
|
+
* converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
|
|
835
|
+
* The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
|
|
836
|
+
* @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
|
|
837
|
+
*/
|
|
838
|
+
public patchRedirectTable(detachedStorageTable: Map<string, string>): void {
|
|
850
839
|
assert(
|
|
851
840
|
this.runtime.attachState === AttachState.Detached,
|
|
852
841
|
0x252 /* "redirect table can only be set in detached container" */,
|
|
853
842
|
);
|
|
843
|
+
// The values of the redirect table are the pseudo storage IDs, which are the keys of the
|
|
844
|
+
// detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
|
|
845
|
+
// storage IDs (many in the case that the storage dedupes the blob).
|
|
854
846
|
assert(
|
|
855
|
-
this.redirectTable.size ===
|
|
847
|
+
new Set(this.redirectTable.values()).size === detachedStorageTable.size,
|
|
856
848
|
0x391 /* Redirect table size must match BlobManager's local ID count */,
|
|
857
849
|
);
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
|
|
850
|
+
// Taking a snapshot of the redirect table entries before iterating, because
|
|
851
|
+
// we will be adding identity mappings to the the redirect table as we iterate
|
|
852
|
+
// and we don't want to include those in the iteration.
|
|
853
|
+
const redirectTableEntries = [...this.redirectTable.entries()];
|
|
854
|
+
for (const [localId, detachedStorageId] of redirectTableEntries) {
|
|
855
|
+
const newStorageId = detachedStorageTable.get(detachedStorageId);
|
|
856
|
+
assert(newStorageId !== undefined, "Couldn't find a matching storage ID");
|
|
857
|
+
this.setRedirection(localId, newStorageId);
|
|
861
858
|
// set identity (id -> id) entry
|
|
862
|
-
this.setRedirection(
|
|
859
|
+
this.setRedirection(newStorageId, newStorageId);
|
|
863
860
|
}
|
|
864
861
|
}
|
|
865
862
|
|
|
@@ -896,17 +893,17 @@ export class BlobManager {
|
|
|
896
893
|
}
|
|
897
894
|
|
|
898
895
|
/**
|
|
899
|
-
* For a
|
|
896
|
+
* For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
|
|
900
897
|
* This path must match the path of the blob handle returned by the createBlob API because blobs are marked
|
|
901
898
|
* referenced by storing these handles in a referenced DDS.
|
|
902
899
|
*/
|
|
903
|
-
const
|
|
904
|
-
`/${blobManagerBasePath}/${
|
|
900
|
+
const getGCNodePathFromLocalId = (localId: string): string =>
|
|
901
|
+
`/${blobManagerBasePath}/${localId}`;
|
|
905
902
|
|
|
906
903
|
/**
|
|
907
|
-
* For a given GC node path, return the
|
|
904
|
+
* For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
|
|
908
905
|
*/
|
|
909
|
-
const
|
|
906
|
+
const getLocalIdFromGCNodePath = (nodePath: string): string => {
|
|
910
907
|
const pathParts = nodePath.split("/");
|
|
911
908
|
assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
|
|
912
909
|
return pathParts[2];
|