@fluidframework/container-runtime 2.60.0 → 2.61.0-355054

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (40) hide show
  1. package/.mocharc.cjs +1 -2
  2. package/container-runtime.test-files.tar +0 -0
  3. package/dist/blobManager/blobManager.d.ts +21 -15
  4. package/dist/blobManager/blobManager.d.ts.map +1 -1
  5. package/dist/blobManager/blobManager.js +105 -102
  6. package/dist/blobManager/blobManager.js.map +1 -1
  7. package/dist/blobManager/blobManagerSnapSum.d.ts +4 -4
  8. package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -1
  9. package/dist/blobManager/blobManagerSnapSum.js +30 -33
  10. package/dist/blobManager/blobManagerSnapSum.js.map +1 -1
  11. package/dist/containerRuntime.js +1 -1
  12. package/dist/containerRuntime.js.map +1 -1
  13. package/dist/legacy.d.ts +2 -1
  14. package/dist/packageVersion.d.ts +1 -1
  15. package/dist/packageVersion.d.ts.map +1 -1
  16. package/dist/packageVersion.js +1 -1
  17. package/dist/packageVersion.js.map +1 -1
  18. package/internal.d.ts +1 -1
  19. package/legacy.d.ts +1 -1
  20. package/lib/blobManager/blobManager.d.ts +21 -15
  21. package/lib/blobManager/blobManager.d.ts.map +1 -1
  22. package/lib/blobManager/blobManager.js +105 -102
  23. package/lib/blobManager/blobManager.js.map +1 -1
  24. package/lib/blobManager/blobManagerSnapSum.d.ts +4 -4
  25. package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -1
  26. package/lib/blobManager/blobManagerSnapSum.js +26 -29
  27. package/lib/blobManager/blobManagerSnapSum.js.map +1 -1
  28. package/lib/containerRuntime.js +1 -1
  29. package/lib/containerRuntime.js.map +1 -1
  30. package/lib/legacy.d.ts +2 -1
  31. package/lib/packageVersion.d.ts +1 -1
  32. package/lib/packageVersion.d.ts.map +1 -1
  33. package/lib/packageVersion.js +1 -1
  34. package/lib/packageVersion.js.map +1 -1
  35. package/lib/tsdoc-metadata.json +1 -1
  36. package/package.json +26 -26
  37. package/src/blobManager/blobManager.ts +118 -121
  38. package/src/blobManager/blobManagerSnapSum.ts +31 -53
  39. package/src/containerRuntime.ts +1 -1
  40. package/src/packageVersion.ts +1 -1
package/lib/legacy.d.ts CHANGED
@@ -9,7 +9,7 @@
9
9
  */
10
10
 
11
11
  export {
12
- // @legacy APIs
12
+ // #region @legacyBeta APIs
13
13
  AllowTombstoneRequestHeaderKey,
14
14
  CompressionAlgorithms,
15
15
  ContainerMessageType,
@@ -65,4 +65,5 @@ export {
65
65
  TombstoneResponseHeaderKey,
66
66
  disabledCompressionConfig,
67
67
  loadContainerRuntime
68
+ // #endregion
68
69
  } from "./index.js";
@@ -5,5 +5,5 @@
5
5
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
6
6
  */
7
7
  export declare const pkgName = "@fluidframework/container-runtime";
8
- export declare const pkgVersion = "2.60.0";
8
+ export declare const pkgVersion = "2.61.0-355054";
9
9
  //# sourceMappingURL=packageVersion.d.ts.map
@@ -1 +1 @@
1
- {"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,WAAW,CAAC"}
1
+ {"version":3,"file":"packageVersion.d.ts","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,eAAO,MAAM,OAAO,sCAAsC,CAAC;AAC3D,eAAO,MAAM,UAAU,kBAAkB,CAAC"}
@@ -5,5 +5,5 @@
5
5
  * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY
6
6
  */
7
7
  export const pkgName = "@fluidframework/container-runtime";
8
- export const pkgVersion = "2.60.0";
8
+ export const pkgVersion = "2.61.0-355054";
9
9
  //# sourceMappingURL=packageVersion.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,QAAQ,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.60.0\";\n"]}
1
+ {"version":3,"file":"packageVersion.js","sourceRoot":"","sources":["../src/packageVersion.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAEH,MAAM,CAAC,MAAM,OAAO,GAAG,mCAAmC,CAAC;AAC3D,MAAM,CAAC,MAAM,UAAU,GAAG,eAAe,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n *\n * THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY\n */\n\nexport const pkgName = \"@fluidframework/container-runtime\";\nexport const pkgVersion = \"2.61.0-355054\";\n"]}
@@ -5,7 +5,7 @@
5
5
  "toolPackages": [
6
6
  {
7
7
  "packageName": "@microsoft/api-extractor",
8
- "packageVersion": "7.52.8"
8
+ "packageVersion": "7.52.11"
9
9
  }
10
10
  ]
11
11
  }
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fluidframework/container-runtime",
3
- "version": "2.60.0",
3
+ "version": "2.61.0-355054",
4
4
  "description": "Fluid container runtime",
5
5
  "homepage": "https://fluidframework.com",
6
6
  "repository": {
@@ -119,18 +119,18 @@
119
119
  "temp-directory": "nyc/.nyc_output"
120
120
  },
121
121
  "dependencies": {
122
- "@fluid-internal/client-utils": "~2.60.0",
123
- "@fluidframework/container-definitions": "~2.60.0",
124
- "@fluidframework/container-runtime-definitions": "~2.60.0",
125
- "@fluidframework/core-interfaces": "~2.60.0",
126
- "@fluidframework/core-utils": "~2.60.0",
127
- "@fluidframework/datastore": "~2.60.0",
128
- "@fluidframework/driver-definitions": "~2.60.0",
129
- "@fluidframework/driver-utils": "~2.60.0",
130
- "@fluidframework/id-compressor": "~2.60.0",
131
- "@fluidframework/runtime-definitions": "~2.60.0",
132
- "@fluidframework/runtime-utils": "~2.60.0",
133
- "@fluidframework/telemetry-utils": "~2.60.0",
122
+ "@fluid-internal/client-utils": "2.61.0-355054",
123
+ "@fluidframework/container-definitions": "2.61.0-355054",
124
+ "@fluidframework/container-runtime-definitions": "2.61.0-355054",
125
+ "@fluidframework/core-interfaces": "2.61.0-355054",
126
+ "@fluidframework/core-utils": "2.61.0-355054",
127
+ "@fluidframework/datastore": "2.61.0-355054",
128
+ "@fluidframework/driver-definitions": "2.61.0-355054",
129
+ "@fluidframework/driver-utils": "2.61.0-355054",
130
+ "@fluidframework/id-compressor": "2.61.0-355054",
131
+ "@fluidframework/runtime-definitions": "2.61.0-355054",
132
+ "@fluidframework/runtime-utils": "2.61.0-355054",
133
+ "@fluidframework/telemetry-utils": "2.61.0-355054",
134
134
  "@tylerbu/sorted-btree-es6": "^1.8.0",
135
135
  "double-ended-queue": "^2.1.0-0",
136
136
  "lz4js": "^0.2.0",
@@ -140,23 +140,23 @@
140
140
  "devDependencies": {
141
141
  "@arethetypeswrong/cli": "^0.17.1",
142
142
  "@biomejs/biome": "~1.9.3",
143
- "@fluid-internal/mocha-test-setup": "~2.60.0",
144
- "@fluid-private/stochastic-test-utils": "~2.60.0",
145
- "@fluid-private/test-pairwise-generator": "~2.60.0",
143
+ "@fluid-internal/mocha-test-setup": "2.61.0-355054",
144
+ "@fluid-private/stochastic-test-utils": "2.61.0-355054",
145
+ "@fluid-private/test-pairwise-generator": "2.61.0-355054",
146
146
  "@fluid-tools/benchmark": "^0.51.0",
147
- "@fluid-tools/build-cli": "^0.57.0",
147
+ "@fluid-tools/build-cli": "^0.58.2",
148
148
  "@fluidframework/build-common": "^2.0.3",
149
- "@fluidframework/build-tools": "^0.57.0",
150
- "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.53.0",
149
+ "@fluidframework/build-tools": "^0.58.2",
150
+ "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.60.0",
151
151
  "@fluidframework/eslint-config-fluid": "^6.0.0",
152
- "@fluidframework/test-runtime-utils": "~2.60.0",
153
- "@microsoft/api-extractor": "7.52.8",
152
+ "@fluidframework/test-runtime-utils": "2.61.0-355054",
153
+ "@microsoft/api-extractor": "7.52.11",
154
154
  "@types/double-ended-queue": "^2.1.0",
155
155
  "@types/lz4js": "^0.2.0",
156
156
  "@types/mocha": "^10.0.10",
157
157
  "@types/node": "^18.19.0",
158
158
  "@types/sinon": "^17.0.3",
159
- "c8": "^8.0.1",
159
+ "c8": "^10.1.3",
160
160
  "concurrently": "^8.2.1",
161
161
  "copyfiles": "^2.4.1",
162
162
  "cross-env": "^7.0.3",
@@ -173,8 +173,8 @@
173
173
  },
174
174
  "scripts": {
175
175
  "api": "fluid-build . --task api",
176
- "api-extractor:commonjs": "flub generate entrypoints --outDir ./dist",
177
- "api-extractor:esnext": "flub generate entrypoints --outDir ./lib --node10TypeCompat",
176
+ "api-extractor:commonjs": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./dist",
177
+ "api-extractor:esnext": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./lib --node10TypeCompat",
178
178
  "build": "fluid-build . --task build",
179
179
  "build:api-reports": "concurrently \"npm:build:api-reports:*\"",
180
180
  "build:api-reports:current": "api-extractor run --local --config api-extractor/api-extractor.current.json",
@@ -213,8 +213,8 @@
213
213
  "test:benchmark:report": "mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js \"./dist/**/*.perf.spec.*js\"",
214
214
  "test:coverage": "c8 npm test",
215
215
  "test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",
216
- "test:mocha:cjs": "mocha --recursive \"dist/test/**/*.spec.*js\"",
217
- "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.*js\"",
216
+ "test:mocha:cjs": "cross-env MOCHA_SPEC=dist/test mocha",
217
+ "test:mocha:esm": "mocha",
218
218
  "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
219
219
  "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && npm run place:cjs:package-stub",
220
220
  "tsc:watch": "npm run place:cjs:package-stub && fluid-tsc commonjs --project ./tsconfig.cjs.json --watch",
@@ -181,13 +181,12 @@ export class BlobManager {
181
181
  private readonly internalEvents = createEmitter<IBlobManagerInternalEvents>();
182
182
 
183
183
  /**
184
- * Map of local IDs to storage IDs. Contains identity entries (storageId storageId) for storage IDs. All requested IDs should
185
- * be a key in this map. Blobs created while the container is detached are stored in IDetachedBlobStorage which
186
- * gives local IDs; the storage IDs are filled in at attach time.
187
- * Note: It contains mappings from all clients, i.e., from remote clients as well. local ID comes from the client
188
- * that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
184
+ * Map of local IDs to storage IDs. Also includes identity mappings of storage ID to storage ID for all known
185
+ * storage IDs. All requested IDs must be a key in this map. Blobs created while the container is detached are
186
+ * stored in IDetachedBlobStorage which gives pseudo storage IDs; the real storage IDs are filled in at attach
187
+ * time via setRedirectTable().
189
188
  */
190
- private readonly redirectTable: Map<string, string | undefined>;
189
+ private readonly redirectTable: Map<string, string>;
191
190
 
192
191
  /**
193
192
  * Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
@@ -206,13 +205,13 @@ export class BlobManager {
206
205
  private readonly routeContext: IFluidHandleContext;
207
206
  private readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
208
207
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
209
- // blobPath's format - `/<basePath>/<blobId>`.
208
+ // blobPath's format - `/<basePath>/<localId>`.
210
209
  private readonly blobRequested: (blobPath: string) => void;
211
210
  // Called to check if a blob has been deleted by GC.
212
- // blobPath's format - `/<basePath>/<blobId>`.
211
+ // blobPath's format - `/<basePath>/<localId>`.
213
212
  private readonly isBlobDeleted: (blobPath: string) => boolean;
214
213
  private readonly runtime: IBlobManagerRuntime;
215
- private readonly localBlobIdGenerator: () => string;
214
+ private readonly localIdGenerator: () => string;
216
215
 
217
216
  private readonly createBlobPayloadPending: boolean;
218
217
 
@@ -233,14 +232,14 @@ export class BlobManager {
233
232
  */
234
233
  sendBlobAttachOp: (localId: string, storageId: string) => void;
235
234
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
236
- // blobPath's format - `/<basePath>/<blobId>`.
235
+ // blobPath's format - `/<basePath>/<localId>`.
237
236
  readonly blobRequested: (blobPath: string) => void;
238
237
  // Called to check if a blob has been deleted by GC.
239
- // blobPath's format - `/<basePath>/<blobId>`.
238
+ // blobPath's format - `/<basePath>/<localId>`.
240
239
  readonly isBlobDeleted: (blobPath: string) => boolean;
241
240
  readonly runtime: IBlobManagerRuntime;
242
241
  stashedBlobs: IPendingBlobs | undefined;
243
- readonly localBlobIdGenerator?: (() => string) | undefined;
242
+ readonly localIdGenerator?: (() => string) | undefined;
244
243
  readonly createBlobPayloadPending: boolean;
245
244
  }) {
246
245
  const {
@@ -251,7 +250,7 @@ export class BlobManager {
251
250
  blobRequested,
252
251
  isBlobDeleted,
253
252
  runtime,
254
- localBlobIdGenerator,
253
+ localIdGenerator,
255
254
  createBlobPayloadPending,
256
255
  } = props;
257
256
  this.routeContext = routeContext;
@@ -259,7 +258,7 @@ export class BlobManager {
259
258
  this.blobRequested = blobRequested;
260
259
  this.isBlobDeleted = isBlobDeleted;
261
260
  this.runtime = runtime;
262
- this.localBlobIdGenerator = localBlobIdGenerator ?? uuid;
261
+ this.localIdGenerator = localIdGenerator ?? uuid;
263
262
  this.createBlobPayloadPending = createBlobPayloadPending;
264
263
 
265
264
  this.mc = createChildMonitoringContext({
@@ -267,13 +266,9 @@ export class BlobManager {
267
266
  namespace: "BlobManager",
268
267
  });
269
268
 
270
- this.redirectTable = toRedirectTable(
271
- blobManagerLoadInfo,
272
- this.mc.logger,
273
- this.runtime.attachState,
274
- );
269
+ this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger);
275
270
 
276
- this.sendBlobAttachOp = (localId: string, blobId: string) => {
271
+ this.sendBlobAttachOp = (localId: string, storageId: string) => {
277
272
  const pendingEntry = this.pendingBlobs.get(localId);
278
273
  assert(
279
274
  pendingEntry !== undefined,
@@ -302,7 +297,7 @@ export class BlobManager {
302
297
  }
303
298
  }
304
299
  pendingEntry.opsent = true;
305
- sendBlobAttachOp(localId, blobId);
300
+ sendBlobAttachOp(localId, storageId);
306
301
  };
307
302
  }
308
303
 
@@ -329,59 +324,49 @@ export class BlobManager {
329
324
  });
330
325
  }
331
326
 
332
- public hasBlob(blobId: string): boolean {
333
- return this.redirectTable.get(blobId) !== undefined;
327
+ public hasBlob(localId: string): boolean {
328
+ return this.redirectTable.get(localId) !== undefined;
334
329
  }
335
330
 
336
331
  /**
337
332
  * Retrieve the blob with the given local blob id.
338
- * @param blobId - The local blob id. Likely coming from a handle.
333
+ * @param localId - The local blob id. Likely coming from a handle.
339
334
  * @param payloadPending - Whether we suspect the payload may be pending and not available yet.
340
335
  * @returns A promise which resolves to the blob contents
341
336
  */
342
- public async getBlob(blobId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
337
+ public async getBlob(localId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
343
338
  // Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
344
339
  // an error, failing the call.
345
- this.verifyBlobNotDeleted(blobId);
340
+ this.verifyBlobNotDeleted(localId);
346
341
  // Let runtime know that the corresponding GC node was requested.
347
342
  // Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
348
343
  // is configured.
349
- this.blobRequested(getGCNodePathFromBlobId(blobId));
344
+ this.blobRequested(getGCNodePathFromLocalId(localId));
350
345
 
351
- const pending = this.pendingBlobs.get(blobId);
346
+ const pending = this.pendingBlobs.get(localId);
352
347
  if (pending) {
353
348
  return pending.blob;
354
349
  }
355
350
 
356
- let storageId: string;
357
- if (this.runtime.attachState === AttachState.Detached) {
358
- assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
359
-
360
- // Blobs created while the container is detached are stored in IDetachedBlobStorage.
361
- // The 'IContainerStorageService.readBlob()' call below will retrieve these via localId.
362
- storageId = blobId;
363
- } else {
364
- const attachedStorageId = this.redirectTable.get(blobId);
365
- if (!payloadPending) {
366
- // Only blob handles explicitly marked with pending payload are permitted to exist without
367
- // yet knowing their storage id. Otherwise they must already be associated with a storage id.
368
- assert(attachedStorageId !== undefined, 0x11f /* "requesting unknown blobs" */);
369
- }
370
- // If we didn't find it in the redirectTable, assume the attach op is coming eventually and wait.
371
- // We do this even if the local client doesn't have the blob payloadPending flag enabled, in case a
372
- // remote client does have it enabled. This wait may be infinite if the uploading client failed
373
- // the upload and doesn't exist anymore.
374
- storageId =
375
- attachedStorageId ??
376
- (await new Promise<string>((resolve) => {
377
- const onProcessBlobAttach = (localId: string, _storageId: string): void => {
378
- if (localId === blobId) {
379
- this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
380
- resolve(_storageId);
381
- }
382
- };
383
- this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
384
- }));
351
+ let storageId = this.redirectTable.get(localId);
352
+ if (storageId === undefined) {
353
+ // Only blob handles explicitly marked with pending payload are permitted to exist without
354
+ // yet knowing their storage id. Otherwise they must already be associated with a storage id.
355
+ // Handles for detached blobs are not payload pending.
356
+ assert(payloadPending, 0x11f /* "requesting unknown blobs" */);
357
+ // If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
358
+ // eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
359
+ // enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
360
+ // client failed the upload and doesn't exist anymore.
361
+ storageId = await new Promise<string>((resolve) => {
362
+ const onProcessBlobAttach = (_localId: string, _storageId: string): void => {
363
+ if (_localId === localId) {
364
+ this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
365
+ resolve(_storageId);
366
+ }
367
+ };
368
+ this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
369
+ });
385
370
  }
386
371
 
387
372
  return PerformanceEvent.timedExecAsync(
@@ -417,7 +402,7 @@ export class BlobManager {
417
402
  }
418
403
  : undefined;
419
404
  return new BlobHandle(
420
- getGCNodePathFromBlobId(localId),
405
+ getGCNodePathFromLocalId(localId),
421
406
  this.routeContext,
422
407
  async () => this.getBlob(localId, false),
423
408
  false, // payloadPending
@@ -428,11 +413,13 @@ export class BlobManager {
428
413
  private async createBlobDetached(
429
414
  blob: ArrayBufferLike,
430
415
  ): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
416
+ const localId = this.localIdGenerator();
431
417
  // Blobs created while the container is detached are stored in IDetachedBlobStorage.
432
- // The 'IContainerStorageService.createBlob()' call below will respond with a localId.
433
- const response = await this.storage.createBlob(blob);
434
- this.setRedirection(response.id, undefined);
435
- return this.getBlobHandle(response.id);
418
+ // The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
419
+ // That pseudo storage ID will be replaced with the real storage ID at attach time.
420
+ const { id: detachedStorageId } = await this.storage.createBlob(blob);
421
+ this.setRedirection(localId, detachedStorageId);
422
+ return this.getBlobHandle(localId);
436
423
  }
437
424
 
438
425
  public async createBlob(
@@ -467,7 +454,7 @@ export class BlobManager {
467
454
 
468
455
  // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
469
456
  // storage ID mapping is created.
470
- const localId = this.localBlobIdGenerator();
457
+ const localId = this.localIdGenerator();
471
458
  const pendingEntry: PendingBlob = {
472
459
  blob,
473
460
  handleP: new Deferred(),
@@ -494,10 +481,10 @@ export class BlobManager {
494
481
  private createBlobWithPayloadPending(
495
482
  blob: ArrayBufferLike,
496
483
  ): IFluidHandleInternalPayloadPending<ArrayBufferLike> {
497
- const localId = this.localBlobIdGenerator();
484
+ const localId = this.localIdGenerator();
498
485
 
499
486
  const blobHandle = new BlobHandle(
500
- getGCNodePathFromBlobId(localId),
487
+ getGCNodePathFromLocalId(localId),
501
488
  this.routeContext,
502
489
  async () => blob,
503
490
  true, // payloadPending
@@ -581,7 +568,7 @@ export class BlobManager {
581
568
  * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
582
569
  * which is required for GC.
583
570
  */
584
- private setRedirection(fromId: string, toId: string | undefined): void {
571
+ private setRedirection(fromId: string, toId: string): void {
585
572
  this.redirectTable.set(fromId, toId);
586
573
  }
587
574
 
@@ -630,7 +617,7 @@ export class BlobManager {
630
617
  if (!entry.opsent) {
631
618
  this.sendBlobAttachOp(localId, response.id);
632
619
  }
633
- const storageIds = getStorageIds(this.redirectTable, this.runtime.attachState);
620
+ const storageIds = getStorageIds(this.redirectTable);
634
621
  if (storageIds.has(response.id)) {
635
622
  // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
636
623
  // an existing blob, we don't have to wait for the op to be ack'd since this step has already
@@ -663,7 +650,7 @@ export class BlobManager {
663
650
  */
664
651
  public reSubmit(metadata: Record<string, unknown> | undefined): void {
665
652
  assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
666
- const { localId, blobId } = metadata;
653
+ const { localId, blobId: storageId } = metadata;
667
654
  // Any blob that we're actively trying to advance to attached state must have a
668
655
  // pendingBlobs entry. Decline to resubmit for anything else.
669
656
  // For example, we might be asked to resubmit stashed ops for blobs that never had
@@ -671,7 +658,7 @@ export class BlobManager {
671
658
  // try to attach them since they won't be accessible to the customer and would just
672
659
  // be considered garbage immediately.
673
660
  if (this.pendingBlobs.has(localId)) {
674
- this.sendBlobAttachOp(localId, blobId);
661
+ this.sendBlobAttachOp(localId, storageId);
675
662
  }
676
663
  }
677
664
 
@@ -680,19 +667,19 @@ export class BlobManager {
680
667
  isBlobMetadata(message.metadata),
681
668
  0xc02 /* Expected blob metadata for a BlobAttach op */,
682
669
  );
683
- const { localId, blobId } = message.metadata;
670
+ const { localId, blobId: storageId } = message.metadata;
684
671
  const pendingEntry = this.pendingBlobs.get(localId);
685
672
  if (pendingEntry?.abortSignal?.aborted) {
686
673
  this.deletePendingBlob(localId);
687
674
  return;
688
675
  }
689
676
 
690
- this.setRedirection(localId, blobId);
677
+ this.setRedirection(localId, storageId);
691
678
  // set identity (id -> id) entry
692
- this.setRedirection(blobId, blobId);
679
+ this.setRedirection(storageId, storageId);
693
680
 
694
681
  if (local) {
695
- const waitingBlobs = this.opsInFlight.get(blobId);
682
+ const waitingBlobs = this.opsInFlight.get(storageId);
696
683
  if (waitingBlobs !== undefined) {
697
684
  // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
698
685
  // This is safe because the server will keep the blob alive and the op containing the local ID to
@@ -703,14 +690,14 @@ export class BlobManager {
703
690
  entry !== undefined,
704
691
  0x38f /* local online BlobAttach op with no pending blob entry */,
705
692
  );
706
- this.setRedirection(pendingLocalId, blobId);
693
+ this.setRedirection(pendingLocalId, storageId);
707
694
  entry.acked = true;
708
695
  const blobHandle = this.getBlobHandle(pendingLocalId);
709
696
  blobHandle.notifyShared();
710
697
  entry.handleP.resolve(blobHandle);
711
698
  this.deletePendingBlobMaybe(pendingLocalId);
712
699
  }
713
- this.opsInFlight.delete(blobId);
700
+ this.opsInFlight.delete(storageId);
714
701
  }
715
702
  const localEntry = this.pendingBlobs.get(localId);
716
703
  if (localEntry) {
@@ -721,11 +708,11 @@ export class BlobManager {
721
708
  this.deletePendingBlobMaybe(localId);
722
709
  }
723
710
  }
724
- this.internalEvents.emit("processedBlobAttach", localId, blobId);
711
+ this.internalEvents.emit("processedBlobAttach", localId, storageId);
725
712
  }
726
713
 
727
714
  public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
728
- return summarizeBlobManagerState(this.redirectTable, this.runtime.attachState);
715
+ return summarizeBlobManagerState(this.redirectTable);
729
716
  }
730
717
 
731
718
  /**
@@ -737,13 +724,12 @@ export class BlobManager {
737
724
  public getGCData(fullGC: boolean = false): IGarbageCollectionData {
738
725
  const gcData: IGarbageCollectionData = { gcNodes: {} };
739
726
  for (const [localId, storageId] of this.redirectTable) {
740
- assert(!!storageId, 0x390 /* Must be attached to get GC data */);
741
- // Only return local ids as GC nodes because a blob can only be referenced via its local id. The storage
742
- // id entries have the same key and value, ignore them.
743
- // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
744
- // by adding its handle to a referenced DDS.
727
+ // Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
728
+ // IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
745
729
  if (localId !== storageId) {
746
- gcData.gcNodes[getGCNodePathFromBlobId(localId)] = [];
730
+ // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
731
+ // by adding its handle to a referenced DDS.
732
+ gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
747
733
  }
748
734
  }
749
735
  return gcData;
@@ -764,58 +750,55 @@ export class BlobManager {
764
750
  * Delete blobs with the given routes from the redirect table.
765
751
  *
766
752
  * @remarks
767
- * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<blobId>`. The blob ids are all local ids.
753
+ * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
768
754
  * Deleting the blobs involves 2 steps:
769
755
  *
770
756
  * 1. The redirect table entry for the local ids are deleted.
771
757
  *
772
- * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
773
- * for the storage ids are deleted as well.
758
+ * 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
759
+ * identity mappings in the redirect table are deleted as well.
774
760
  *
775
761
  * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
776
- * will remove them the next summary. The service would them delete them some time in the future.
762
+ * will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
763
+ * some time in the future.
777
764
  */
778
765
  private deleteBlobsFromRedirectTable(blobRoutes: readonly string[]): void {
779
- if (blobRoutes.length === 0) {
780
- return;
781
- }
782
-
783
- // This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
784
- // these storage ids are unused, they will be deleted as well.
766
+ // maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
767
+ // local ID is being deleted.
785
768
  const maybeUnusedStorageIds: Set<string> = new Set();
786
769
  for (const route of blobRoutes) {
787
- const blobId = getBlobIdFromGCNodePath(route);
770
+ const localId = getLocalIdFromGCNodePath(route);
788
771
  // If the blob hasn't already been deleted, log an error because this should never happen.
789
772
  // If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
790
773
  // sweep ops can contain the same data store. It would be interesting to track how often this happens.
791
774
  const alreadyDeleted = this.isBlobDeleted(route);
792
- if (!this.redirectTable.has(blobId)) {
775
+ const storageId = this.redirectTable.get(localId);
776
+ if (storageId === undefined) {
793
777
  this.mc.logger.sendTelemetryEvent({
794
778
  eventName: "DeletedAttachmentBlobNotFound",
795
779
  category: alreadyDeleted ? "generic" : "error",
796
- blobId,
780
+ blobId: localId,
797
781
  details: { alreadyDeleted },
798
782
  });
799
783
  continue;
800
784
  }
801
- const storageId = this.redirectTable.get(blobId);
802
- assert(!!storageId, 0x5bb /* Must be attached to run GC */);
803
785
  maybeUnusedStorageIds.add(storageId);
804
- this.redirectTable.delete(blobId);
786
+ this.redirectTable.delete(localId);
805
787
  }
806
788
 
807
- // Find out storage ids that are in-use and remove them from maybeUnusedStorageIds. A storage id is in-use if
808
- // the redirect table has a local id -> storage id entry for it.
809
- for (const [localId, storageId] of this.redirectTable.entries()) {
810
- assert(!!storageId, 0x5bc /* Must be attached to run GC */);
811
- // For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
812
- if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
789
+ // Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
790
+ for (const [localId, storageId] of this.redirectTable) {
791
+ if (localId !== storageId) {
813
792
  maybeUnusedStorageIds.delete(storageId);
814
793
  }
815
794
  }
816
795
 
817
- // For unused storage ids, delete their id -> id entries from the redirect table.
818
- // This way they'll be absent from the next summary, and the service is free to delete them from storage.
796
+ // Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
797
+ // referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
798
+ // is free to delete them from storage.
799
+ // WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
800
+ // has let us add a local ID -> storage ID mapping that is later deleted. AB#47337 tracks this issue
801
+ // and possible solutions.
819
802
  for (const storageId of maybeUnusedStorageIds) {
820
803
  this.redirectTable.delete(storageId);
821
804
  }
@@ -825,12 +808,12 @@ export class BlobManager {
825
808
  * Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
826
809
  * log an error and throw if necessary.
827
810
  */
828
- private verifyBlobNotDeleted(blobId: string): void {
829
- if (!this.isBlobDeleted(getGCNodePathFromBlobId(blobId))) {
811
+ private verifyBlobNotDeleted(localId: string): void {
812
+ if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
830
813
  return;
831
814
  }
832
815
 
833
- const request = { url: blobId };
816
+ const request = { url: localId };
834
817
  const error = responseToException(
835
818
  createResponseError(404, `Blob was deleted`, request),
836
819
  request,
@@ -846,20 +829,34 @@ export class BlobManager {
846
829
  throw error;
847
830
  }
848
831
 
849
- public setRedirectTable(table: Map<string, string>): void {
832
+ /**
833
+ * Called in detached state just prior to attaching, this will update the redirect table by
834
+ * converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
835
+ * The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
836
+ * @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
837
+ */
838
+ public patchRedirectTable(detachedStorageTable: Map<string, string>): void {
850
839
  assert(
851
840
  this.runtime.attachState === AttachState.Detached,
852
841
  0x252 /* "redirect table can only be set in detached container" */,
853
842
  );
843
+ // The values of the redirect table are the pseudo storage IDs, which are the keys of the
844
+ // detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
845
+ // storage IDs (many in the case that the storage dedupes the blob).
854
846
  assert(
855
- this.redirectTable.size === table.size,
847
+ new Set(this.redirectTable.values()).size === detachedStorageTable.size,
856
848
  0x391 /* Redirect table size must match BlobManager's local ID count */,
857
849
  );
858
- for (const [localId, storageId] of table) {
859
- assert(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
860
- this.setRedirection(localId, storageId);
850
+ // Taking a snapshot of the redirect table entries before iterating, because
851
+ // we will be adding identity mappings to the the redirect table as we iterate
852
+ // and we don't want to include those in the iteration.
853
+ const redirectTableEntries = [...this.redirectTable.entries()];
854
+ for (const [localId, detachedStorageId] of redirectTableEntries) {
855
+ const newStorageId = detachedStorageTable.get(detachedStorageId);
856
+ assert(newStorageId !== undefined, "Couldn't find a matching storage ID");
857
+ this.setRedirection(localId, newStorageId);
861
858
  // set identity (id -> id) entry
862
- this.setRedirection(storageId, storageId);
859
+ this.setRedirection(newStorageId, newStorageId);
863
860
  }
864
861
  }
865
862
 
@@ -896,17 +893,17 @@ export class BlobManager {
896
893
  }
897
894
 
898
895
  /**
899
- * For a blobId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<blobId>`.
896
+ * For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
900
897
  * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
901
898
  * referenced by storing these handles in a referenced DDS.
902
899
  */
903
- const getGCNodePathFromBlobId = (blobId: string): string =>
904
- `/${blobManagerBasePath}/${blobId}`;
900
+ const getGCNodePathFromLocalId = (localId: string): string =>
901
+ `/${blobManagerBasePath}/${localId}`;
905
902
 
906
903
  /**
907
- * For a given GC node path, return the blobId. The node path is of the format `/<basePath>/<blobId>`.
904
+ * For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
908
905
  */
909
- const getBlobIdFromGCNodePath = (nodePath: string): string => {
906
+ const getLocalIdFromGCNodePath = (nodePath: string): string => {
910
907
  const pathParts = nodePath.split("/");
911
908
  assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
912
909
  return pathParts[2];