@fluidframework/container-runtime 2.60.0 → 2.61.0-355516

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. package/.mocharc.cjs +1 -2
  2. package/api-report/container-runtime.legacy.beta.api.md +2 -1
  3. package/container-runtime.test-files.tar +0 -0
  4. package/dist/blobManager/blobManager.d.ts +33 -16
  5. package/dist/blobManager/blobManager.d.ts.map +1 -1
  6. package/dist/blobManager/blobManager.js +126 -106
  7. package/dist/blobManager/blobManager.js.map +1 -1
  8. package/dist/blobManager/blobManagerSnapSum.d.ts +4 -4
  9. package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -1
  10. package/dist/blobManager/blobManagerSnapSum.js +30 -33
  11. package/dist/blobManager/blobManagerSnapSum.js.map +1 -1
  12. package/dist/channelCollection.d.ts +6 -2
  13. package/dist/channelCollection.d.ts.map +1 -1
  14. package/dist/channelCollection.js +1 -0
  15. package/dist/channelCollection.js.map +1 -1
  16. package/dist/containerCompatibility.d.ts +18 -0
  17. package/dist/containerCompatibility.d.ts.map +1 -1
  18. package/dist/containerCompatibility.js +23 -1
  19. package/dist/containerCompatibility.js.map +1 -1
  20. package/dist/containerRuntime.d.ts +15 -3
  21. package/dist/containerRuntime.d.ts.map +1 -1
  22. package/dist/containerRuntime.js +75 -52
  23. package/dist/containerRuntime.js.map +1 -1
  24. package/dist/dataStoreContext.d.ts +5 -1
  25. package/dist/dataStoreContext.d.ts.map +1 -1
  26. package/dist/dataStoreContext.js +1 -0
  27. package/dist/dataStoreContext.js.map +1 -1
  28. package/dist/legacy.d.ts +2 -1
  29. package/dist/packageVersion.d.ts +1 -1
  30. package/dist/packageVersion.d.ts.map +1 -1
  31. package/dist/packageVersion.js +1 -1
  32. package/dist/packageVersion.js.map +1 -1
  33. package/lib/blobManager/blobManager.d.ts +33 -16
  34. package/lib/blobManager/blobManager.d.ts.map +1 -1
  35. package/lib/blobManager/blobManager.js +126 -106
  36. package/lib/blobManager/blobManager.js.map +1 -1
  37. package/lib/blobManager/blobManagerSnapSum.d.ts +4 -4
  38. package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -1
  39. package/lib/blobManager/blobManagerSnapSum.js +26 -29
  40. package/lib/blobManager/blobManagerSnapSum.js.map +1 -1
  41. package/lib/channelCollection.d.ts +6 -2
  42. package/lib/channelCollection.d.ts.map +1 -1
  43. package/lib/channelCollection.js +1 -0
  44. package/lib/channelCollection.js.map +1 -1
  45. package/lib/containerCompatibility.d.ts +18 -0
  46. package/lib/containerCompatibility.d.ts.map +1 -1
  47. package/lib/containerCompatibility.js +22 -0
  48. package/lib/containerCompatibility.js.map +1 -1
  49. package/lib/containerRuntime.d.ts +15 -3
  50. package/lib/containerRuntime.d.ts.map +1 -1
  51. package/lib/containerRuntime.js +26 -3
  52. package/lib/containerRuntime.js.map +1 -1
  53. package/lib/dataStoreContext.d.ts +5 -1
  54. package/lib/dataStoreContext.d.ts.map +1 -1
  55. package/lib/dataStoreContext.js +1 -0
  56. package/lib/dataStoreContext.js.map +1 -1
  57. package/lib/legacy.d.ts +2 -1
  58. package/lib/packageVersion.d.ts +1 -1
  59. package/lib/packageVersion.d.ts.map +1 -1
  60. package/lib/packageVersion.js +1 -1
  61. package/lib/packageVersion.js.map +1 -1
  62. package/lib/tsdoc-metadata.json +1 -1
  63. package/package.json +27 -27
  64. package/src/blobManager/blobManager.ts +138 -123
  65. package/src/blobManager/blobManagerSnapSum.ts +31 -53
  66. package/src/channelCollection.ts +9 -1
  67. package/src/containerCompatibility.ts +56 -0
  68. package/src/containerRuntime.ts +35 -4
  69. package/src/dataStoreContext.ts +7 -0
  70. package/src/packageVersion.ts +1 -1
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@fluidframework/container-runtime",
3
- "version": "2.60.0",
3
+ "version": "2.61.0-355516",
4
4
  "description": "Fluid container runtime",
5
5
  "homepage": "https://fluidframework.com",
6
6
  "repository": {
@@ -119,18 +119,18 @@
119
119
  "temp-directory": "nyc/.nyc_output"
120
120
  },
121
121
  "dependencies": {
122
- "@fluid-internal/client-utils": "~2.60.0",
123
- "@fluidframework/container-definitions": "~2.60.0",
124
- "@fluidframework/container-runtime-definitions": "~2.60.0",
125
- "@fluidframework/core-interfaces": "~2.60.0",
126
- "@fluidframework/core-utils": "~2.60.0",
127
- "@fluidframework/datastore": "~2.60.0",
128
- "@fluidframework/driver-definitions": "~2.60.0",
129
- "@fluidframework/driver-utils": "~2.60.0",
130
- "@fluidframework/id-compressor": "~2.60.0",
131
- "@fluidframework/runtime-definitions": "~2.60.0",
132
- "@fluidframework/runtime-utils": "~2.60.0",
133
- "@fluidframework/telemetry-utils": "~2.60.0",
122
+ "@fluid-internal/client-utils": "2.61.0-355516",
123
+ "@fluidframework/container-definitions": "2.61.0-355516",
124
+ "@fluidframework/container-runtime-definitions": "2.61.0-355516",
125
+ "@fluidframework/core-interfaces": "2.61.0-355516",
126
+ "@fluidframework/core-utils": "2.61.0-355516",
127
+ "@fluidframework/datastore": "2.61.0-355516",
128
+ "@fluidframework/driver-definitions": "2.61.0-355516",
129
+ "@fluidframework/driver-utils": "2.61.0-355516",
130
+ "@fluidframework/id-compressor": "2.61.0-355516",
131
+ "@fluidframework/runtime-definitions": "2.61.0-355516",
132
+ "@fluidframework/runtime-utils": "2.61.0-355516",
133
+ "@fluidframework/telemetry-utils": "2.61.0-355516",
134
134
  "@tylerbu/sorted-btree-es6": "^1.8.0",
135
135
  "double-ended-queue": "^2.1.0-0",
136
136
  "lz4js": "^0.2.0",
@@ -140,23 +140,23 @@
140
140
  "devDependencies": {
141
141
  "@arethetypeswrong/cli": "^0.17.1",
142
142
  "@biomejs/biome": "~1.9.3",
143
- "@fluid-internal/mocha-test-setup": "~2.60.0",
144
- "@fluid-private/stochastic-test-utils": "~2.60.0",
145
- "@fluid-private/test-pairwise-generator": "~2.60.0",
143
+ "@fluid-internal/mocha-test-setup": "2.61.0-355516",
144
+ "@fluid-private/stochastic-test-utils": "2.61.0-355516",
145
+ "@fluid-private/test-pairwise-generator": "2.61.0-355516",
146
146
  "@fluid-tools/benchmark": "^0.51.0",
147
- "@fluid-tools/build-cli": "^0.57.0",
147
+ "@fluid-tools/build-cli": "^0.58.1",
148
148
  "@fluidframework/build-common": "^2.0.3",
149
- "@fluidframework/build-tools": "^0.57.0",
150
- "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.53.0",
149
+ "@fluidframework/build-tools": "^0.58.1",
150
+ "@fluidframework/container-runtime-previous": "npm:@fluidframework/container-runtime@2.60.0",
151
151
  "@fluidframework/eslint-config-fluid": "^6.0.0",
152
- "@fluidframework/test-runtime-utils": "~2.60.0",
153
- "@microsoft/api-extractor": "7.52.8",
152
+ "@fluidframework/test-runtime-utils": "2.61.0-355516",
153
+ "@microsoft/api-extractor": "7.52.11",
154
154
  "@types/double-ended-queue": "^2.1.0",
155
155
  "@types/lz4js": "^0.2.0",
156
156
  "@types/mocha": "^10.0.10",
157
157
  "@types/node": "^18.19.0",
158
158
  "@types/sinon": "^17.0.3",
159
- "c8": "^8.0.1",
159
+ "c8": "^10.1.3",
160
160
  "concurrently": "^8.2.1",
161
161
  "copyfiles": "^2.4.1",
162
162
  "cross-env": "^7.0.3",
@@ -173,8 +173,8 @@
173
173
  },
174
174
  "scripts": {
175
175
  "api": "fluid-build . --task api",
176
- "api-extractor:commonjs": "flub generate entrypoints --outDir ./dist",
177
- "api-extractor:esnext": "flub generate entrypoints --outDir ./lib --node10TypeCompat",
176
+ "api-extractor:commonjs": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./dist",
177
+ "api-extractor:esnext": "flub generate entrypoints --outFileLegacyBeta legacy --outDir ./lib --node10TypeCompat",
178
178
  "build": "fluid-build . --task build",
179
179
  "build:api-reports": "concurrently \"npm:build:api-reports:*\"",
180
180
  "build:api-reports:current": "api-extractor run --local --config api-extractor/api-extractor.current.json",
@@ -210,11 +210,11 @@
210
210
  "pack:tests": "tar -cf ./container-runtime.test-files.tar ./src/test ./dist/test ./lib/test",
211
211
  "place:cjs:package-stub": "copyfiles -f ../../../common/build/build-common/src/cjs/package.json ./dist",
212
212
  "test": "npm run test:mocha",
213
- "test:benchmark:report": "mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js \"./dist/**/*.perf.spec.*js\"",
213
+ "test:benchmark:report": "cross-env \"MOCHA_SPEC=dist/**/*.perf.spec.*js\" mocha --timeout 10s --perfMode --parentProcess --fgrep @Benchmark --fgrep @ExecutionTime --reporter @fluid-tools/benchmark/dist/MochaReporter.js",
214
214
  "test:coverage": "c8 npm test",
215
215
  "test:mocha": "npm run test:mocha:esm && echo skipping cjs to avoid overhead - npm run test:mocha:cjs",
216
- "test:mocha:cjs": "mocha --recursive \"dist/test/**/*.spec.*js\"",
217
- "test:mocha:esm": "mocha --recursive \"lib/test/**/*.spec.*js\"",
216
+ "test:mocha:cjs": "cross-env MOCHA_SPEC=dist/test mocha",
217
+ "test:mocha:esm": "mocha",
218
218
  "test:mocha:verbose": "cross-env FLUID_TEST_VERBOSE=1 npm run test:mocha",
219
219
  "tsc": "fluid-tsc commonjs --project ./tsconfig.cjs.json && npm run place:cjs:package-stub",
220
220
  "tsc:watch": "npm run place:cjs:package-stub && fluid-tsc commonjs --project ./tsconfig.cjs.json --watch",
@@ -129,7 +129,7 @@ export class BlobHandle
129
129
  // the contract explicit and reduces the amount of mocking required for tests.
130
130
  export type IBlobManagerRuntime = Pick<
131
131
  IContainerRuntime,
132
- "attachState" | "connected" | "baseLogger" | "clientDetails" | "disposed"
132
+ "attachState" | "baseLogger" | "disposed"
133
133
  > &
134
134
  IEventProvider<IContainerRuntimeEvents>;
135
135
 
@@ -181,13 +181,12 @@ export class BlobManager {
181
181
  private readonly internalEvents = createEmitter<IBlobManagerInternalEvents>();
182
182
 
183
183
  /**
184
- * Map of local IDs to storage IDs. Contains identity entries (storageId storageId) for storage IDs. All requested IDs should
185
- * be a key in this map. Blobs created while the container is detached are stored in IDetachedBlobStorage which
186
- * gives local IDs; the storage IDs are filled in at attach time.
187
- * Note: It contains mappings from all clients, i.e., from remote clients as well. local ID comes from the client
188
- * that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
184
+ * Map of local IDs to storage IDs. Also includes identity mappings of storage ID to storage ID for all known
185
+ * storage IDs. All requested IDs must be a key in this map. Blobs created while the container is detached are
186
+ * stored in IDetachedBlobStorage which gives pseudo storage IDs; the real storage IDs are filled in at attach
187
+ * time via setRedirectTable().
189
188
  */
190
- private readonly redirectTable: Map<string, string | undefined>;
189
+ private readonly redirectTable: Map<string, string>;
191
190
 
192
191
  /**
193
192
  * Blobs which we have not yet seen a BlobAttach op round-trip and not yet attached to a DDS.
@@ -206,13 +205,13 @@ export class BlobManager {
206
205
  private readonly routeContext: IFluidHandleContext;
207
206
  private readonly storage: Pick<IContainerStorageService, "createBlob" | "readBlob">;
208
207
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
209
- // blobPath's format - `/<basePath>/<blobId>`.
208
+ // blobPath's format - `/<basePath>/<localId>`.
210
209
  private readonly blobRequested: (blobPath: string) => void;
211
210
  // Called to check if a blob has been deleted by GC.
212
- // blobPath's format - `/<basePath>/<blobId>`.
211
+ // blobPath's format - `/<basePath>/<localId>`.
213
212
  private readonly isBlobDeleted: (blobPath: string) => boolean;
214
213
  private readonly runtime: IBlobManagerRuntime;
215
- private readonly localBlobIdGenerator: () => string;
214
+ private readonly localIdGenerator: () => string;
216
215
 
217
216
  private readonly createBlobPayloadPending: boolean;
218
217
 
@@ -233,14 +232,14 @@ export class BlobManager {
233
232
  */
234
233
  sendBlobAttachOp: (localId: string, storageId: string) => void;
235
234
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
236
- // blobPath's format - `/<basePath>/<blobId>`.
235
+ // blobPath's format - `/<basePath>/<localId>`.
237
236
  readonly blobRequested: (blobPath: string) => void;
238
237
  // Called to check if a blob has been deleted by GC.
239
- // blobPath's format - `/<basePath>/<blobId>`.
238
+ // blobPath's format - `/<basePath>/<localId>`.
240
239
  readonly isBlobDeleted: (blobPath: string) => boolean;
241
240
  readonly runtime: IBlobManagerRuntime;
242
241
  stashedBlobs: IPendingBlobs | undefined;
243
- readonly localBlobIdGenerator?: (() => string) | undefined;
242
+ readonly localIdGenerator?: (() => string) | undefined;
244
243
  readonly createBlobPayloadPending: boolean;
245
244
  }) {
246
245
  const {
@@ -251,7 +250,7 @@ export class BlobManager {
251
250
  blobRequested,
252
251
  isBlobDeleted,
253
252
  runtime,
254
- localBlobIdGenerator,
253
+ localIdGenerator,
255
254
  createBlobPayloadPending,
256
255
  } = props;
257
256
  this.routeContext = routeContext;
@@ -259,7 +258,7 @@ export class BlobManager {
259
258
  this.blobRequested = blobRequested;
260
259
  this.isBlobDeleted = isBlobDeleted;
261
260
  this.runtime = runtime;
262
- this.localBlobIdGenerator = localBlobIdGenerator ?? uuid;
261
+ this.localIdGenerator = localIdGenerator ?? uuid;
263
262
  this.createBlobPayloadPending = createBlobPayloadPending;
264
263
 
265
264
  this.mc = createChildMonitoringContext({
@@ -267,13 +266,9 @@ export class BlobManager {
267
266
  namespace: "BlobManager",
268
267
  });
269
268
 
270
- this.redirectTable = toRedirectTable(
271
- blobManagerLoadInfo,
272
- this.mc.logger,
273
- this.runtime.attachState,
274
- );
269
+ this.redirectTable = toRedirectTable(blobManagerLoadInfo, this.mc.logger);
275
270
 
276
- this.sendBlobAttachOp = (localId: string, blobId: string) => {
271
+ this.sendBlobAttachOp = (localId: string, storageId: string) => {
277
272
  const pendingEntry = this.pendingBlobs.get(localId);
278
273
  assert(
279
274
  pendingEntry !== undefined,
@@ -302,7 +297,7 @@ export class BlobManager {
302
297
  }
303
298
  }
304
299
  pendingEntry.opsent = true;
305
- sendBlobAttachOp(localId, blobId);
300
+ sendBlobAttachOp(localId, storageId);
306
301
  };
307
302
  }
308
303
 
@@ -329,59 +324,67 @@ export class BlobManager {
329
324
  });
330
325
  }
331
326
 
332
- public hasBlob(blobId: string): boolean {
333
- return this.redirectTable.get(blobId) !== undefined;
327
+ public hasBlob(localId: string): boolean {
328
+ return this.redirectTable.get(localId) !== undefined;
329
+ }
330
+
331
+ /**
332
+ * Lookup the blob storage ID for a given local blob id.
333
+ * @param localId - The local blob id. Likely coming from a handle.
334
+ * @returns The storage ID if found and the blob is not pending, undefined otherwise.
335
+ * @remarks
336
+ * For blobs with pending payloads (localId exists but upload hasn't finished), this is expected to return undefined.
337
+ * Consumers should use the observability APIs on the handle (handle.payloadState, payloadShared event)
338
+ * to understand/wait for storage ID availability.
339
+ * Similarly, when the runtime is detached, this will return undefined as no blobs have been uploaded to storage.
340
+ */
341
+ public lookupTemporaryBlobStorageId(localId: string): string | undefined {
342
+ if (this.runtime.attachState === AttachState.Detached) {
343
+ return undefined;
344
+ }
345
+ // Get the storage ID from the redirect table
346
+ return this.redirectTable.get(localId);
334
347
  }
335
348
 
336
349
  /**
337
350
  * Retrieve the blob with the given local blob id.
338
- * @param blobId - The local blob id. Likely coming from a handle.
351
+ * @param localId - The local blob id. Likely coming from a handle.
339
352
  * @param payloadPending - Whether we suspect the payload may be pending and not available yet.
340
353
  * @returns A promise which resolves to the blob contents
341
354
  */
342
- public async getBlob(blobId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
355
+ public async getBlob(localId: string, payloadPending: boolean): Promise<ArrayBufferLike> {
343
356
  // Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
344
357
  // an error, failing the call.
345
- this.verifyBlobNotDeleted(blobId);
358
+ this.verifyBlobNotDeleted(localId);
346
359
  // Let runtime know that the corresponding GC node was requested.
347
360
  // Note that this will throw if the blob is inactive or tombstoned and throwing on incorrect usage
348
361
  // is configured.
349
- this.blobRequested(getGCNodePathFromBlobId(blobId));
362
+ this.blobRequested(getGCNodePathFromLocalId(localId));
350
363
 
351
- const pending = this.pendingBlobs.get(blobId);
364
+ const pending = this.pendingBlobs.get(localId);
352
365
  if (pending) {
353
366
  return pending.blob;
354
367
  }
355
368
 
356
- let storageId: string;
357
- if (this.runtime.attachState === AttachState.Detached) {
358
- assert(this.redirectTable.has(blobId), 0x383 /* requesting unknown blobs */);
359
-
360
- // Blobs created while the container is detached are stored in IDetachedBlobStorage.
361
- // The 'IContainerStorageService.readBlob()' call below will retrieve these via localId.
362
- storageId = blobId;
363
- } else {
364
- const attachedStorageId = this.redirectTable.get(blobId);
365
- if (!payloadPending) {
366
- // Only blob handles explicitly marked with pending payload are permitted to exist without
367
- // yet knowing their storage id. Otherwise they must already be associated with a storage id.
368
- assert(attachedStorageId !== undefined, 0x11f /* "requesting unknown blobs" */);
369
- }
370
- // If we didn't find it in the redirectTable, assume the attach op is coming eventually and wait.
371
- // We do this even if the local client doesn't have the blob payloadPending flag enabled, in case a
372
- // remote client does have it enabled. This wait may be infinite if the uploading client failed
373
- // the upload and doesn't exist anymore.
374
- storageId =
375
- attachedStorageId ??
376
- (await new Promise<string>((resolve) => {
377
- const onProcessBlobAttach = (localId: string, _storageId: string): void => {
378
- if (localId === blobId) {
379
- this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
380
- resolve(_storageId);
381
- }
382
- };
383
- this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
384
- }));
369
+ let storageId = this.redirectTable.get(localId);
370
+ if (storageId === undefined) {
371
+ // Only blob handles explicitly marked with pending payload are permitted to exist without
372
+ // yet knowing their storage id. Otherwise they must already be associated with a storage id.
373
+ // Handles for detached blobs are not payload pending.
374
+ assert(payloadPending, 0x11f /* "requesting unknown blobs" */);
375
+ // If we didn't find it in the redirectTable and it's payloadPending, assume the attach op is coming
376
+ // eventually and wait. We do this even if the local client doesn't have the blob payloadPending flag
377
+ // enabled, in case a remote client does have it enabled. This wait may be infinite if the uploading
378
+ // client failed the upload and doesn't exist anymore.
379
+ storageId = await new Promise<string>((resolve) => {
380
+ const onProcessBlobAttach = (_localId: string, _storageId: string): void => {
381
+ if (_localId === localId) {
382
+ this.internalEvents.off("processedBlobAttach", onProcessBlobAttach);
383
+ resolve(_storageId);
384
+ }
385
+ };
386
+ this.internalEvents.on("processedBlobAttach", onProcessBlobAttach);
387
+ });
385
388
  }
386
389
 
387
390
  return PerformanceEvent.timedExecAsync(
@@ -417,7 +420,7 @@ export class BlobManager {
417
420
  }
418
421
  : undefined;
419
422
  return new BlobHandle(
420
- getGCNodePathFromBlobId(localId),
423
+ getGCNodePathFromLocalId(localId),
421
424
  this.routeContext,
422
425
  async () => this.getBlob(localId, false),
423
426
  false, // payloadPending
@@ -428,11 +431,13 @@ export class BlobManager {
428
431
  private async createBlobDetached(
429
432
  blob: ArrayBufferLike,
430
433
  ): Promise<IFluidHandleInternalPayloadPending<ArrayBufferLike>> {
434
+ const localId = this.localIdGenerator();
431
435
  // Blobs created while the container is detached are stored in IDetachedBlobStorage.
432
- // The 'IContainerStorageService.createBlob()' call below will respond with a localId.
433
- const response = await this.storage.createBlob(blob);
434
- this.setRedirection(response.id, undefined);
435
- return this.getBlobHandle(response.id);
436
+ // The 'IContainerStorageService.createBlob()' call below will respond with a pseudo storage ID.
437
+ // That pseudo storage ID will be replaced with the real storage ID at attach time.
438
+ const { id: detachedStorageId } = await this.storage.createBlob(blob);
439
+ this.setRedirection(localId, detachedStorageId);
440
+ return this.getBlobHandle(localId);
436
441
  }
437
442
 
438
443
  public async createBlob(
@@ -467,7 +472,7 @@ export class BlobManager {
467
472
 
468
473
  // Create a local ID for the blob. After uploading it to storage and before returning it, a local ID to
469
474
  // storage ID mapping is created.
470
- const localId = this.localBlobIdGenerator();
475
+ const localId = this.localIdGenerator();
471
476
  const pendingEntry: PendingBlob = {
472
477
  blob,
473
478
  handleP: new Deferred(),
@@ -494,10 +499,10 @@ export class BlobManager {
494
499
  private createBlobWithPayloadPending(
495
500
  blob: ArrayBufferLike,
496
501
  ): IFluidHandleInternalPayloadPending<ArrayBufferLike> {
497
- const localId = this.localBlobIdGenerator();
502
+ const localId = this.localIdGenerator();
498
503
 
499
504
  const blobHandle = new BlobHandle(
500
- getGCNodePathFromBlobId(localId),
505
+ getGCNodePathFromLocalId(localId),
501
506
  this.routeContext,
502
507
  async () => blob,
503
508
  true, // payloadPending
@@ -581,7 +586,7 @@ export class BlobManager {
581
586
  * Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
582
587
  * which is required for GC.
583
588
  */
584
- private setRedirection(fromId: string, toId: string | undefined): void {
589
+ private setRedirection(fromId: string, toId: string): void {
585
590
  this.redirectTable.set(fromId, toId);
586
591
  }
587
592
 
@@ -630,7 +635,7 @@ export class BlobManager {
630
635
  if (!entry.opsent) {
631
636
  this.sendBlobAttachOp(localId, response.id);
632
637
  }
633
- const storageIds = getStorageIds(this.redirectTable, this.runtime.attachState);
638
+ const storageIds = getStorageIds(this.redirectTable);
634
639
  if (storageIds.has(response.id)) {
635
640
  // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
636
641
  // an existing blob, we don't have to wait for the op to be ack'd since this step has already
@@ -663,7 +668,7 @@ export class BlobManager {
663
668
  */
664
669
  public reSubmit(metadata: Record<string, unknown> | undefined): void {
665
670
  assert(isBlobMetadata(metadata), 0xc01 /* Expected blob metadata for a BlobAttach op */);
666
- const { localId, blobId } = metadata;
671
+ const { localId, blobId: storageId } = metadata;
667
672
  // Any blob that we're actively trying to advance to attached state must have a
668
673
  // pendingBlobs entry. Decline to resubmit for anything else.
669
674
  // For example, we might be asked to resubmit stashed ops for blobs that never had
@@ -671,7 +676,7 @@ export class BlobManager {
671
676
  // try to attach them since they won't be accessible to the customer and would just
672
677
  // be considered garbage immediately.
673
678
  if (this.pendingBlobs.has(localId)) {
674
- this.sendBlobAttachOp(localId, blobId);
679
+ this.sendBlobAttachOp(localId, storageId);
675
680
  }
676
681
  }
677
682
 
@@ -680,19 +685,19 @@ export class BlobManager {
680
685
  isBlobMetadata(message.metadata),
681
686
  0xc02 /* Expected blob metadata for a BlobAttach op */,
682
687
  );
683
- const { localId, blobId } = message.metadata;
688
+ const { localId, blobId: storageId } = message.metadata;
684
689
  const pendingEntry = this.pendingBlobs.get(localId);
685
690
  if (pendingEntry?.abortSignal?.aborted) {
686
691
  this.deletePendingBlob(localId);
687
692
  return;
688
693
  }
689
694
 
690
- this.setRedirection(localId, blobId);
695
+ this.setRedirection(localId, storageId);
691
696
  // set identity (id -> id) entry
692
- this.setRedirection(blobId, blobId);
697
+ this.setRedirection(storageId, storageId);
693
698
 
694
699
  if (local) {
695
- const waitingBlobs = this.opsInFlight.get(blobId);
700
+ const waitingBlobs = this.opsInFlight.get(storageId);
696
701
  if (waitingBlobs !== undefined) {
697
702
  // For each op corresponding to this storage ID that we are waiting for, resolve the pending blob.
698
703
  // This is safe because the server will keep the blob alive and the op containing the local ID to
@@ -703,14 +708,14 @@ export class BlobManager {
703
708
  entry !== undefined,
704
709
  0x38f /* local online BlobAttach op with no pending blob entry */,
705
710
  );
706
- this.setRedirection(pendingLocalId, blobId);
711
+ this.setRedirection(pendingLocalId, storageId);
707
712
  entry.acked = true;
708
713
  const blobHandle = this.getBlobHandle(pendingLocalId);
709
714
  blobHandle.notifyShared();
710
715
  entry.handleP.resolve(blobHandle);
711
716
  this.deletePendingBlobMaybe(pendingLocalId);
712
717
  }
713
- this.opsInFlight.delete(blobId);
718
+ this.opsInFlight.delete(storageId);
714
719
  }
715
720
  const localEntry = this.pendingBlobs.get(localId);
716
721
  if (localEntry) {
@@ -721,11 +726,11 @@ export class BlobManager {
721
726
  this.deletePendingBlobMaybe(localId);
722
727
  }
723
728
  }
724
- this.internalEvents.emit("processedBlobAttach", localId, blobId);
729
+ this.internalEvents.emit("processedBlobAttach", localId, storageId);
725
730
  }
726
731
 
727
732
  public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
728
- return summarizeBlobManagerState(this.redirectTable, this.runtime.attachState);
733
+ return summarizeBlobManagerState(this.redirectTable);
729
734
  }
730
735
 
731
736
  /**
@@ -737,13 +742,12 @@ export class BlobManager {
737
742
  public getGCData(fullGC: boolean = false): IGarbageCollectionData {
738
743
  const gcData: IGarbageCollectionData = { gcNodes: {} };
739
744
  for (const [localId, storageId] of this.redirectTable) {
740
- assert(!!storageId, 0x390 /* Must be attached to get GC data */);
741
- // Only return local ids as GC nodes because a blob can only be referenced via its local id. The storage
742
- // id entries have the same key and value, ignore them.
743
- // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
744
- // by adding its handle to a referenced DDS.
745
+ // Don't report the identity mappings to GC - these exist to service old handles that referenced the storage
746
+ // IDs directly. We'll implicitly clean them up if all of their localId references get GC'd first.
745
747
  if (localId !== storageId) {
746
- gcData.gcNodes[getGCNodePathFromBlobId(localId)] = [];
748
+ // The outbound routes are empty because a blob node cannot reference other nodes. It can only be referenced
749
+ // by adding its handle to a referenced DDS.
750
+ gcData.gcNodes[getGCNodePathFromLocalId(localId)] = [];
747
751
  }
748
752
  }
749
753
  return gcData;
@@ -764,58 +768,55 @@ export class BlobManager {
764
768
  * Delete blobs with the given routes from the redirect table.
765
769
  *
766
770
  * @remarks
767
- * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<blobId>`. The blob ids are all local ids.
771
+ * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<localId>`.
768
772
  * Deleting the blobs involves 2 steps:
769
773
  *
770
774
  * 1. The redirect table entry for the local ids are deleted.
771
775
  *
772
- * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
773
- * for the storage ids are deleted as well.
776
+ * 2. If the storage ids corresponding to the deleted local ids are not referenced by any further local ids, the
777
+ * identity mappings in the redirect table are deleted as well.
774
778
  *
775
779
  * Note that this does not delete the blobs from storage service immediately. Deleting the blobs from redirect table
776
- * will remove them the next summary. The service would them delete them some time in the future.
780
+ * will ensure we don't create an attachment blob for them at the next summary. The service would then delete them
781
+ * some time in the future.
777
782
  */
778
783
  private deleteBlobsFromRedirectTable(blobRoutes: readonly string[]): void {
779
- if (blobRoutes.length === 0) {
780
- return;
781
- }
782
-
783
- // This tracks the storage ids of local ids that are deleted. After the local ids have been deleted, if any of
784
- // these storage ids are unused, they will be deleted as well.
784
+ // maybeUnusedStorageIds is used to compute the set of storage IDs that *used to have a local ID*, but that
785
+ // local ID is being deleted.
785
786
  const maybeUnusedStorageIds: Set<string> = new Set();
786
787
  for (const route of blobRoutes) {
787
- const blobId = getBlobIdFromGCNodePath(route);
788
+ const localId = getLocalIdFromGCNodePath(route);
788
789
  // If the blob hasn't already been deleted, log an error because this should never happen.
789
790
  // If the blob has already been deleted, log a telemetry event. This can happen because multiple GC
790
791
  // sweep ops can contain the same data store. It would be interesting to track how often this happens.
791
792
  const alreadyDeleted = this.isBlobDeleted(route);
792
- if (!this.redirectTable.has(blobId)) {
793
+ const storageId = this.redirectTable.get(localId);
794
+ if (storageId === undefined) {
793
795
  this.mc.logger.sendTelemetryEvent({
794
796
  eventName: "DeletedAttachmentBlobNotFound",
795
797
  category: alreadyDeleted ? "generic" : "error",
796
- blobId,
798
+ blobId: localId,
797
799
  details: { alreadyDeleted },
798
800
  });
799
801
  continue;
800
802
  }
801
- const storageId = this.redirectTable.get(blobId);
802
- assert(!!storageId, 0x5bb /* Must be attached to run GC */);
803
803
  maybeUnusedStorageIds.add(storageId);
804
- this.redirectTable.delete(blobId);
804
+ this.redirectTable.delete(localId);
805
805
  }
806
806
 
807
- // Find out storage ids that are in-use and remove them from maybeUnusedStorageIds. A storage id is in-use if
808
- // the redirect table has a local id -> storage id entry for it.
809
- for (const [localId, storageId] of this.redirectTable.entries()) {
810
- assert(!!storageId, 0x5bc /* Must be attached to run GC */);
811
- // For every storage id, the redirect table has a id -> id entry. These do not make the storage id in-use.
812
- if (maybeUnusedStorageIds.has(storageId) && localId !== storageId) {
807
+ // Remove any storage IDs that still have local IDs referring to them (excluding the identity mapping).
808
+ for (const [localId, storageId] of this.redirectTable) {
809
+ if (localId !== storageId) {
813
810
  maybeUnusedStorageIds.delete(storageId);
814
811
  }
815
812
  }
816
813
 
817
- // For unused storage ids, delete their id -> id entries from the redirect table.
818
- // This way they'll be absent from the next summary, and the service is free to delete them from storage.
814
+ // Now delete any identity mappings (storage ID -> storage ID) from the redirect table that used to be
815
+ // referenced by a distinct local ID. This way they'll be absent from the next summary, and the service
816
+ // is free to delete them from storage.
817
+ // WARNING: This can potentially delete identity mappings that are still referenced, if storage deduping
818
+ // has let us add a local ID -> storage ID mapping that is later deleted. AB#47337 tracks this issue
819
+ // and possible solutions.
819
820
  for (const storageId of maybeUnusedStorageIds) {
820
821
  this.redirectTable.delete(storageId);
821
822
  }
@@ -825,12 +826,12 @@ export class BlobManager {
825
826
  * Verifies that the blob with given id is not deleted, i.e., it has not been garbage collected. If the blob is GC'd,
826
827
  * log an error and throw if necessary.
827
828
  */
828
- private verifyBlobNotDeleted(blobId: string): void {
829
- if (!this.isBlobDeleted(getGCNodePathFromBlobId(blobId))) {
829
+ private verifyBlobNotDeleted(localId: string): void {
830
+ if (!this.isBlobDeleted(getGCNodePathFromLocalId(localId))) {
830
831
  return;
831
832
  }
832
833
 
833
- const request = { url: blobId };
834
+ const request = { url: localId };
834
835
  const error = responseToException(
835
836
  createResponseError(404, `Blob was deleted`, request),
836
837
  request,
@@ -846,22 +847,36 @@ export class BlobManager {
846
847
  throw error;
847
848
  }
848
849
 
849
- public setRedirectTable(table: Map<string, string>): void {
850
+ /**
851
+ * Called in detached state just prior to attaching, this will update the redirect table by
852
+ * converting the pseudo storage IDs into real storage IDs using the provided detachedStorageTable.
853
+ * The provided table must have exactly the same set of pseudo storage IDs as are found in the redirect table.
854
+ * @param detachedStorageTable - A map of pseudo storage IDs to real storage IDs.
855
+ */
856
+ public readonly patchRedirectTable = (detachedStorageTable: Map<string, string>): void => {
850
857
  assert(
851
858
  this.runtime.attachState === AttachState.Detached,
852
859
  0x252 /* "redirect table can only be set in detached container" */,
853
860
  );
861
+ // The values of the redirect table are the pseudo storage IDs, which are the keys of the
862
+ // detachedStorageTable. We expect to have a many:1 mapping from local IDs to pseudo
863
+ // storage IDs (many in the case that the storage dedupes the blob).
854
864
  assert(
855
- this.redirectTable.size === table.size,
865
+ new Set(this.redirectTable.values()).size === detachedStorageTable.size,
856
866
  0x391 /* Redirect table size must match BlobManager's local ID count */,
857
867
  );
858
- for (const [localId, storageId] of table) {
859
- assert(this.redirectTable.has(localId), 0x254 /* "unrecognized id in redirect table" */);
860
- this.setRedirection(localId, storageId);
868
+ // Taking a snapshot of the redirect table entries before iterating, because
869
+ // we will be adding identity mappings to the the redirect table as we iterate
870
+ // and we don't want to include those in the iteration.
871
+ const redirectTableEntries = [...this.redirectTable.entries()];
872
+ for (const [localId, detachedStorageId] of redirectTableEntries) {
873
+ const newStorageId = detachedStorageTable.get(detachedStorageId);
874
+ assert(newStorageId !== undefined, "Couldn't find a matching storage ID");
875
+ this.setRedirection(localId, newStorageId);
861
876
  // set identity (id -> id) entry
862
- this.setRedirection(storageId, storageId);
877
+ this.setRedirection(newStorageId, newStorageId);
863
878
  }
864
- }
879
+ };
865
880
 
866
881
  /**
867
882
  * To be used in getPendingLocalState flow. Get a serializable record of the blobs that are
@@ -896,17 +911,17 @@ export class BlobManager {
896
911
  }
897
912
 
898
913
  /**
899
- * For a blobId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<blobId>`.
914
+ * For a localId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<localId>`.
900
915
  * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
901
916
  * referenced by storing these handles in a referenced DDS.
902
917
  */
903
- const getGCNodePathFromBlobId = (blobId: string): string =>
904
- `/${blobManagerBasePath}/${blobId}`;
918
+ const getGCNodePathFromLocalId = (localId: string): string =>
919
+ `/${blobManagerBasePath}/${localId}`;
905
920
 
906
921
  /**
907
- * For a given GC node path, return the blobId. The node path is of the format `/<basePath>/<blobId>`.
922
+ * For a given GC node path, return the localId. The node path is of the format `/<basePath>/<localId>`.
908
923
  */
909
- const getBlobIdFromGCNodePath = (nodePath: string): string => {
924
+ const getLocalIdFromGCNodePath = (nodePath: string): string => {
910
925
  const pathParts = nodePath.split("/");
911
926
  assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
912
927
  return pathParts[2];