@fluidframework/container-runtime 2.1.0-276326 → 2.1.0-276985

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/README.md +5 -5
  2. package/container-runtime.test-files.tar +0 -0
  3. package/dist/{blobManager.d.ts → blobManager/blobManager.d.ts} +9 -29
  4. package/dist/blobManager/blobManager.d.ts.map +1 -0
  5. package/dist/{blobManager.js → blobManager/blobManager.js} +23 -83
  6. package/dist/blobManager/blobManager.js.map +1 -0
  7. package/dist/blobManager/blobManagerSnapSum.d.ts +30 -0
  8. package/dist/blobManager/blobManagerSnapSum.d.ts.map +1 -0
  9. package/dist/blobManager/blobManagerSnapSum.js +82 -0
  10. package/dist/blobManager/blobManagerSnapSum.js.map +1 -0
  11. package/dist/blobManager/index.d.ts +7 -0
  12. package/dist/blobManager/index.d.ts.map +1 -0
  13. package/dist/blobManager/index.js +16 -0
  14. package/dist/blobManager/index.js.map +1 -0
  15. package/dist/containerRuntime.d.ts +1 -5
  16. package/dist/containerRuntime.d.ts.map +1 -1
  17. package/dist/containerRuntime.js +68 -75
  18. package/dist/containerRuntime.js.map +1 -1
  19. package/dist/dataStoreContext.d.ts +1 -0
  20. package/dist/dataStoreContext.d.ts.map +1 -1
  21. package/dist/dataStoreContext.js +7 -2
  22. package/dist/dataStoreContext.js.map +1 -1
  23. package/dist/gc/garbageCollection.js +2 -2
  24. package/dist/gc/garbageCollection.js.map +1 -1
  25. package/dist/index.d.ts +1 -1
  26. package/dist/index.d.ts.map +1 -1
  27. package/dist/index.js.map +1 -1
  28. package/dist/opLifecycle/remoteMessageProcessor.d.ts +17 -1
  29. package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  30. package/dist/opLifecycle/remoteMessageProcessor.js +43 -5
  31. package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
  32. package/dist/packageVersion.d.ts +1 -1
  33. package/dist/packageVersion.js +1 -1
  34. package/dist/packageVersion.js.map +1 -1
  35. package/dist/pendingStateManager.d.ts +3 -1
  36. package/dist/pendingStateManager.d.ts.map +1 -1
  37. package/dist/pendingStateManager.js +16 -1
  38. package/dist/pendingStateManager.js.map +1 -1
  39. package/dist/summary/index.d.ts +1 -1
  40. package/dist/summary/index.d.ts.map +1 -1
  41. package/dist/summary/index.js +1 -2
  42. package/dist/summary/index.js.map +1 -1
  43. package/dist/summary/summaryFormat.d.ts +0 -1
  44. package/dist/summary/summaryFormat.d.ts.map +1 -1
  45. package/dist/summary/summaryFormat.js +3 -3
  46. package/dist/summary/summaryFormat.js.map +1 -1
  47. package/lib/{blobManager.d.ts → blobManager/blobManager.d.ts} +9 -29
  48. package/lib/blobManager/blobManager.d.ts.map +1 -0
  49. package/lib/{blobManager.js → blobManager/blobManager.js} +21 -83
  50. package/lib/blobManager/blobManager.js.map +1 -0
  51. package/lib/blobManager/blobManagerSnapSum.d.ts +30 -0
  52. package/lib/blobManager/blobManagerSnapSum.d.ts.map +1 -0
  53. package/lib/blobManager/blobManagerSnapSum.js +75 -0
  54. package/lib/blobManager/blobManagerSnapSum.js.map +1 -0
  55. package/lib/blobManager/index.d.ts +7 -0
  56. package/lib/blobManager/index.d.ts.map +1 -0
  57. package/lib/blobManager/index.js +7 -0
  58. package/lib/blobManager/index.js.map +1 -0
  59. package/lib/containerRuntime.d.ts +1 -5
  60. package/lib/containerRuntime.d.ts.map +1 -1
  61. package/lib/containerRuntime.js +17 -24
  62. package/lib/containerRuntime.js.map +1 -1
  63. package/lib/dataStoreContext.d.ts +1 -0
  64. package/lib/dataStoreContext.d.ts.map +1 -1
  65. package/lib/dataStoreContext.js +7 -2
  66. package/lib/dataStoreContext.js.map +1 -1
  67. package/lib/gc/garbageCollection.js +2 -2
  68. package/lib/gc/garbageCollection.js.map +1 -1
  69. package/lib/index.d.ts +1 -1
  70. package/lib/index.d.ts.map +1 -1
  71. package/lib/index.js.map +1 -1
  72. package/lib/opLifecycle/remoteMessageProcessor.d.ts +17 -1
  73. package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
  74. package/lib/opLifecycle/remoteMessageProcessor.js +41 -3
  75. package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
  76. package/lib/packageVersion.d.ts +1 -1
  77. package/lib/packageVersion.js +1 -1
  78. package/lib/packageVersion.js.map +1 -1
  79. package/lib/pendingStateManager.d.ts +3 -1
  80. package/lib/pendingStateManager.d.ts.map +1 -1
  81. package/lib/pendingStateManager.js +17 -2
  82. package/lib/pendingStateManager.js.map +1 -1
  83. package/lib/summary/index.d.ts +1 -1
  84. package/lib/summary/index.d.ts.map +1 -1
  85. package/lib/summary/index.js +1 -1
  86. package/lib/summary/index.js.map +1 -1
  87. package/lib/summary/summaryFormat.d.ts +0 -1
  88. package/lib/summary/summaryFormat.d.ts.map +1 -1
  89. package/lib/summary/summaryFormat.js +1 -1
  90. package/lib/summary/summaryFormat.js.map +1 -1
  91. package/package.json +21 -21
  92. package/src/{blobManager.ts → blobManager/blobManager.ts} +38 -123
  93. package/src/blobManager/blobManagerSnapSum.ts +133 -0
  94. package/src/blobManager/index.ts +19 -0
  95. package/src/containerRuntime.ts +34 -37
  96. package/src/dataStoreContext.ts +8 -2
  97. package/src/gc/garbageCollection.ts +2 -2
  98. package/src/index.ts +1 -1
  99. package/src/opLifecycle/remoteMessageProcessor.ts +63 -6
  100. package/src/packageVersion.ts +1 -1
  101. package/src/pendingStateManager.ts +18 -0
  102. package/src/summary/index.ts +0 -1
  103. package/src/summary/summaryFormat.ts +1 -1
  104. package/dist/blobManager.d.ts.map +0 -1
  105. package/dist/blobManager.js.map +0 -1
  106. package/lib/blobManager.d.ts.map +0 -1
  107. package/lib/blobManager.js.map +0 -1
@@ -21,7 +21,6 @@ import { assert, Deferred } from "@fluidframework/core-utils/internal";
21
21
  import {
22
22
  IDocumentStorageService,
23
23
  ICreateBlobResponse,
24
- ISnapshotTree,
25
24
  ISequencedDocumentMessage,
26
25
  } from "@fluidframework/driver-definitions/internal";
27
26
  import { canRetryOnError, runWithRetry } from "@fluidframework/driver-utils/internal";
@@ -32,7 +31,6 @@ import {
32
31
  } from "@fluidframework/runtime-definitions/internal";
33
32
  import {
34
33
  FluidHandleBase,
35
- SummaryTreeBuilder,
36
34
  createResponseError,
37
35
  generateHandleContextPath,
38
36
  responseToException,
@@ -47,7 +45,14 @@ import {
47
45
  } from "@fluidframework/telemetry-utils/internal";
48
46
  import { v4 as uuid } from "uuid";
49
47
 
50
- import { IBlobMetadata } from "./metadata.js";
48
+ import { IBlobMetadata } from "../metadata.js";
49
+
50
+ import {
51
+ getStorageIds,
52
+ summarizeBlobManagerState,
53
+ toRedirectTable,
54
+ type IBlobManagerLoadInfo,
55
+ } from "./blobManagerSnapSum.js";
51
56
 
52
57
  /**
53
58
  * This class represents blob (long string)
@@ -87,16 +92,6 @@ export class BlobHandle extends FluidHandleBase<ArrayBufferLike> {
87
92
  }
88
93
  }
89
94
 
90
- /**
91
- * Information from a snapshot needed to load BlobManager
92
- * @legacy
93
- * @alpha
94
- */
95
- export interface IBlobManagerLoadInfo {
96
- ids?: string[];
97
- redirectTable?: [string, string][];
98
- }
99
-
100
95
  // Restrict the IContainerRuntime interface to the subset required by BlobManager. This helps to make
101
96
  // the contract explicit and reduces the amount of mocking required for tests.
102
97
  export type IBlobManagerRuntime = Pick<
@@ -146,9 +141,9 @@ const stashedPendingBlobOverrides: Pick<
146
141
  uploadTime: undefined,
147
142
  } as const;
148
143
 
144
+ export const blobManagerBasePath = "_blobs" as const;
145
+
149
146
  export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
150
- public static readonly basePath = "_blobs";
151
- private static readonly redirectTableBlobName = ".redirectTable";
152
147
  private readonly mc: MonitoringContext;
153
148
 
154
149
  /**
@@ -178,10 +173,10 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
178
173
  private readonly routeContext: IFluidHandleContext;
179
174
  private readonly getStorage: () => IDocumentStorageService;
180
175
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
181
- // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
176
+ // blobPath's format - `/<basePath>/<blobId>`.
182
177
  private readonly blobRequested: (blobPath: string) => void;
183
178
  // Called to check if a blob has been deleted by GC.
184
- // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
179
+ // blobPath's format - `/<basePath>/<blobId>`.
185
180
  private readonly isBlobDeleted: (blobPath: string) => boolean;
186
181
  private readonly runtime: IBlobManagerRuntime;
187
182
  private readonly closeContainer: (error?: ICriticalContainerError) => void;
@@ -202,10 +197,10 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
202
197
  */
203
198
  sendBlobAttachOp: (localId: string, storageId?: string) => void;
204
199
  // Called when a blob node is requested. blobPath is the path of the blob's node in GC's graph.
205
- // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
200
+ // blobPath's format - `/<basePath>/<blobId>`.
206
201
  readonly blobRequested: (blobPath: string) => void;
207
202
  // Called to check if a blob has been deleted by GC.
208
- // blobPath's format - `/<BlobManager.basePath>/<blobId>`.
203
+ // blobPath's format - `/<basePath>/<blobId>`.
209
204
  readonly isBlobDeleted: (blobPath: string) => boolean;
210
205
  readonly runtime: IBlobManagerRuntime;
211
206
  stashedBlobs: IPendingBlobs | undefined;
@@ -235,7 +230,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
235
230
  namespace: "BlobManager",
236
231
  });
237
232
 
238
- this.redirectTable = this.load(snapshot);
233
+ this.redirectTable = toRedirectTable(snapshot, this.mc.logger, this.runtime.attachState);
239
234
 
240
235
  // Begin uploading stashed blobs from previous container instance
241
236
  Object.entries(stashedBlobs ?? {}).forEach(([localId, entry]) => {
@@ -347,27 +342,6 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
347
342
  );
348
343
  }
349
344
 
350
- /**
351
- * Set of actual storage IDs (i.e., IDs that can be requested from storage). This will be empty if the container is
352
- * detached or there are no (non-pending) attachment blobs in the document
353
- */
354
- private get storageIds(): Set<string> {
355
- const ids = new Set<string | undefined>(this.redirectTable.values());
356
-
357
- // If we are detached, we will not have storage IDs, only undefined
358
- const undefinedValueInTable = ids.delete(undefined);
359
-
360
- // For a detached container, entries are inserted into the redirect table with an undefined storage ID.
361
- // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
362
- assert(
363
- !undefinedValueInTable ||
364
- (this.runtime.attachState === AttachState.Detached && ids.size === 0),
365
- 0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */,
366
- );
367
-
368
- return ids as Set<string>;
369
- }
370
-
371
345
  public async getBlob(blobId: string): Promise<ArrayBufferLike> {
372
346
  // Verify that the blob is not deleted, i.e., it has not been garbage collected. If it is, this will throw
373
347
  // an error, failing the call.
@@ -419,7 +393,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
419
393
  }
420
394
  : undefined;
421
395
  return new BlobHandle(
422
- `${BlobManager.basePath}/${id}`,
396
+ getGCNodePathFromBlobId(id),
423
397
  this.routeContext,
424
398
  async () => this.getBlob(id),
425
399
  callback,
@@ -569,7 +543,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
569
543
  if (!entry.opsent) {
570
544
  this.sendBlobAttachOp(localId, response.id);
571
545
  }
572
- if (this.storageIds.has(response.id)) {
546
+ const storageIds = getStorageIds(this.redirectTable, this.runtime.attachState);
547
+ if (storageIds.has(response.id)) {
573
548
  // The blob is de-duped. Set up a local ID to storage ID mapping and return the blob. Since this is
574
549
  // an existing blob, we don't have to wait for the op to be ack'd since this step has already
575
550
  // happened before and so, the server won't delete it.
@@ -662,74 +637,8 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
662
637
  }
663
638
  }
664
639
 
665
- /**
666
- * Reads blobs needed to load BlobManager from storage.
667
- * @param blobsTree - Tree containing IDs of previously attached blobs. We
668
- * look for the IDs in the blob entries of the tree since the both the r11s
669
- * and SPO drivers replace the attachment types returned in snapshot() with blobs.
670
- */
671
- public static async load(
672
- blobsTree: ISnapshotTree | undefined,
673
- tryFetchBlob: (id: string) => Promise<[string, string][]>,
674
- ): Promise<IBlobManagerLoadInfo> {
675
- if (!blobsTree) {
676
- return {};
677
- }
678
- let redirectTable;
679
- const tableId = blobsTree.blobs[this.redirectTableBlobName];
680
- if (tableId) {
681
- redirectTable = await tryFetchBlob(tableId);
682
- }
683
- const ids = Object.entries(blobsTree.blobs)
684
- .filter(([k, _]) => k !== this.redirectTableBlobName)
685
- .map(([_, v]) => v);
686
- return { ids, redirectTable };
687
- }
688
-
689
- /**
690
- * Load a set of previously attached blob IDs and redirect table from a previous snapshot.
691
- */
692
- private load(snapshot: IBlobManagerLoadInfo): Map<string, string | undefined> {
693
- this.mc.logger.sendTelemetryEvent({
694
- eventName: "AttachmentBlobsLoaded",
695
- count: snapshot.ids?.length ?? 0,
696
- redirectTable: snapshot.redirectTable?.length,
697
- });
698
- const table = new Map<string, string | undefined>(snapshot.redirectTable);
699
- if (snapshot.ids) {
700
- const detached = this.runtime.attachState === AttachState.Detached;
701
- // If we are detached, we don't have storage IDs yet, so set to undefined
702
- // Otherwise, set identity (id -> id) entries
703
- snapshot.ids.forEach((entry) => table.set(entry, detached ? undefined : entry));
704
- }
705
- return table;
706
- }
707
-
708
640
  public summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
709
- // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
710
- const blobIds =
711
- this.storageIds.size > 0
712
- ? Array.from(this.storageIds)
713
- : Array.from(this.redirectTable.keys());
714
- const builder = new SummaryTreeBuilder();
715
- blobIds.forEach((blobId) => {
716
- builder.addAttachment(blobId);
717
- });
718
-
719
- // Any non-identity entries in the table need to be saved in the summary
720
- if (this.redirectTable.size > blobIds.length) {
721
- builder.addBlob(
722
- BlobManager.redirectTableBlobName,
723
- // filter out identity entries
724
- JSON.stringify(
725
- Array.from(this.redirectTable.entries()).filter(
726
- ([localId, storageId]) => localId !== storageId,
727
- ),
728
- ),
729
- );
730
- }
731
-
732
- return builder.getSummaryTree();
641
+ return summarizeBlobManagerState(this.redirectTable, this.runtime.attachState);
733
642
  }
734
643
 
735
644
  /**
@@ -766,7 +675,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
766
675
 
767
676
  /**
768
677
  * Delete blobs with the given routes from the redirect table.
769
- * The routes are GC nodes paths of format -`/<BlobManager.basePath>/<blobId>`. The blob ids are all local ids.
678
+ * The routes are GC nodes paths of format -`/<blobManagerBasePath>/<blobId>`. The blob ids are all local ids.
770
679
  * Deleting the blobs involves 2 steps:
771
680
  * 1. The redirect table entry for the local ids are deleted.
772
681
  * 2. If the storage ids corresponding to the deleted local ids are not in-use anymore, the redirect table entries
@@ -839,7 +748,7 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
839
748
  this.mc.logger.sendErrorEvent(
840
749
  {
841
750
  eventName: "GC_Deleted_Blob_Requested",
842
- pkg: BlobManager.basePath,
751
+ pkg: blobManagerBasePath,
843
752
  },
844
753
  error,
845
754
  );
@@ -936,22 +845,28 @@ export class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
936
845
  }
937
846
 
938
847
  /**
939
- * For a blobId, returns its path in GC's graph. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
848
+ * For a blobId, returns its path in GC's graph. The node path is of the format `/<blobManagerBasePath>/<blobId>`.
940
849
  * This path must match the path of the blob handle returned by the createBlob API because blobs are marked
941
850
  * referenced by storing these handles in a referenced DDS.
942
851
  */
943
- function getGCNodePathFromBlobId(blobId: string) {
944
- return `/${BlobManager.basePath}/${blobId}`;
945
- }
852
+ const getGCNodePathFromBlobId = (blobId: string) => `/${blobManagerBasePath}/${blobId}`;
946
853
 
947
854
  /**
948
- * For a given GC node path, return the blobId. The node path is of the format `/<BlobManager.basePath>/<blobId>`.
855
+ * For a given GC node path, return the blobId. The node path is of the format `/<basePath>/<blobId>`.
949
856
  */
950
- function getBlobIdFromGCNodePath(nodePath: string) {
857
+ const getBlobIdFromGCNodePath = (nodePath: string) => {
951
858
  const pathParts = nodePath.split("/");
952
- assert(
953
- pathParts.length === 3 && pathParts[1] === BlobManager.basePath,
954
- 0x5bd /* Invalid blob node path */,
955
- );
859
+ assert(areBlobPathParts(pathParts), 0x5bd /* Invalid blob node path */);
956
860
  return pathParts[2];
957
- }
861
+ };
862
+
863
+ /**
864
+ * Returns whether a given path is for attachment blobs that are in the format - "/blobManagerBasePath/...".
865
+ */
866
+ export const isBlobPath = (path: string): path is `/${typeof blobManagerBasePath}/${string}` =>
867
+ areBlobPathParts(path.split("/"));
868
+
869
+ export const areBlobPathParts = (
870
+ pathParts: string[],
871
+ ): pathParts is ["", typeof blobManagerBasePath, string] =>
872
+ pathParts.length === 3 && pathParts[1] === blobManagerBasePath;
@@ -0,0 +1,133 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import {
7
+ AttachState,
8
+ type IContainerContext,
9
+ } from "@fluidframework/container-definitions/internal";
10
+ import { assert } from "@fluidframework/core-utils/internal";
11
+ import { readAndParse } from "@fluidframework/driver-utils/internal";
12
+ import type { ISummaryTreeWithStats } from "@fluidframework/runtime-definitions/internal";
13
+ import { SummaryTreeBuilder } from "@fluidframework/runtime-utils/internal";
14
+ import type { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils/internal";
15
+
16
+ /**
17
+ * Information from a snapshot needed to load BlobManager
18
+ * @legacy
19
+ * @alpha
20
+ */
21
+ export interface IBlobManagerLoadInfo {
22
+ ids?: string[];
23
+ redirectTable?: [string, string][];
24
+ }
25
+
26
+ export const redirectTableBlobName = ".redirectTable";
27
+
28
+ /**
29
+ * @internal
30
+ */
31
+ export const blobsTreeName = ".blobs";
32
+
33
+ /**
34
+ * Reads blobs needed to load BlobManager from storage.
35
+ *
36
+ */
37
+ export const loadBlobManagerLoadInfo = async (
38
+ context: Pick<IContainerContext, "baseSnapshot" | "storage" | "attachState">,
39
+ ): Promise<IBlobManagerLoadInfo> => loadV1(context);
40
+
41
+ const loadV1 = async (
42
+ context: Pick<IContainerContext, "baseSnapshot" | "storage" | "attachState">,
43
+ ): Promise<IBlobManagerLoadInfo> => {
44
+ const blobsTree = context.baseSnapshot?.trees[blobsTreeName];
45
+
46
+ if (!blobsTree) {
47
+ return {};
48
+ }
49
+ let redirectTableEntries: [string, string][] = [];
50
+ const tableId = blobsTree.blobs[redirectTableBlobName];
51
+ if (tableId) {
52
+ redirectTableEntries = await readAndParse(context.storage, tableId);
53
+ }
54
+ const ids = Object.entries(blobsTree.blobs)
55
+ .filter(([k, _]) => k !== redirectTableBlobName)
56
+ .map(([_, v]) => v);
57
+
58
+ return { ids, redirectTable: redirectTableEntries };
59
+ };
60
+
61
+ export const toRedirectTable = (
62
+ snapshot: IBlobManagerLoadInfo,
63
+ logger: ITelemetryLoggerExt,
64
+ attachState: AttachState,
65
+ ): Map<string, string | undefined> => {
66
+ logger.sendTelemetryEvent({
67
+ eventName: "AttachmentBlobsLoaded",
68
+ count: snapshot.ids?.length ?? 0,
69
+ redirectTable: snapshot.redirectTable?.length,
70
+ });
71
+ const redirectTable = new Map<string, string | undefined>(snapshot.redirectTable);
72
+ const detached = attachState !== AttachState.Attached;
73
+ if (snapshot.ids) {
74
+ // If we are detached, we don't have storage IDs yet, so set to undefined
75
+ // Otherwise, set identity (id -> id) entries.
76
+ snapshot.ids.forEach((entry) => redirectTable.set(entry, detached ? undefined : entry));
77
+ }
78
+ return redirectTable;
79
+ };
80
+
81
+ export const summarizeBlobManagerState = (
82
+ redirectTable: Map<string, string | undefined>,
83
+ attachState: AttachState,
84
+ ): ISummaryTreeWithStats => summarizeV1(redirectTable, attachState);
85
+
86
+ const summarizeV1 = (
87
+ redirectTable: Map<string, string | undefined>,
88
+ attachState: AttachState,
89
+ ): ISummaryTreeWithStats => {
90
+ const storageIds = getStorageIds(redirectTable, attachState);
91
+
92
+ // if storageIds is empty, it means we are detached and have only local IDs, or that there are no blobs attached
93
+ const blobIds =
94
+ storageIds.size > 0 ? Array.from(storageIds) : Array.from(redirectTable.keys());
95
+ const builder = new SummaryTreeBuilder();
96
+ blobIds.forEach((blobId) => {
97
+ builder.addAttachment(blobId);
98
+ });
99
+
100
+ // Any non-identity entries in the table need to be saved in the summary
101
+ if (redirectTable.size > blobIds.length) {
102
+ builder.addBlob(
103
+ redirectTableBlobName,
104
+ // filter out identity entries
105
+ JSON.stringify(
106
+ Array.from(redirectTable.entries()).filter(
107
+ ([localId, storageId]) => localId !== storageId,
108
+ ),
109
+ ),
110
+ );
111
+ }
112
+
113
+ return builder.getSummaryTree();
114
+ };
115
+
116
+ export const getStorageIds = (
117
+ redirectTable: Map<string, string | undefined>,
118
+ attachState: AttachState,
119
+ ) => {
120
+ const ids = new Set<string | undefined>(redirectTable.values());
121
+
122
+ // If we are detached, we will not have storage IDs, only undefined
123
+ const undefinedValueInTable = ids.delete(undefined);
124
+
125
+ // For a detached container, entries are inserted into the redirect table with an undefined storage ID.
126
+ // For an attached container, entries are inserted w/storage ID after the BlobAttach op round-trips.
127
+ assert(
128
+ !undefinedValueInTable || (attachState === AttachState.Detached && ids.size === 0),
129
+ 0x382 /* 'redirectTable' must contain only undefined while detached / defined values while attached */,
130
+ );
131
+
132
+ return ids as Set<string>;
133
+ };
@@ -0,0 +1,19 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ export {
7
+ BlobManager,
8
+ IPendingBlobs,
9
+ IBlobManagerRuntime,
10
+ IBlobManagerEvents,
11
+ blobManagerBasePath,
12
+ isBlobPath,
13
+ } from "./blobManager.js";
14
+ export {
15
+ loadBlobManagerLoadInfo,
16
+ IBlobManagerLoadInfo,
17
+ blobsTreeName,
18
+ redirectTableBlobName,
19
+ } from "./blobManagerSnapSum.js";
@@ -126,7 +126,15 @@ import {
126
126
  import { v4 as uuid } from "uuid";
127
127
 
128
128
  import { BindBatchTracker } from "./batchTracker.js";
129
- import { BlobManager, IBlobManagerLoadInfo, IPendingBlobs } from "./blobManager.js";
129
+ import {
130
+ BlobManager,
131
+ IPendingBlobs,
132
+ blobManagerBasePath,
133
+ blobsTreeName,
134
+ isBlobPath,
135
+ loadBlobManagerLoadInfo,
136
+ type IBlobManagerLoadInfo,
137
+ } from "./blobManager/index.js";
130
138
  import {
131
139
  ChannelCollection,
132
140
  getSummaryForDatastores,
@@ -211,7 +219,6 @@ import {
211
219
  SummaryCollection,
212
220
  SummaryManager,
213
221
  aliasBlobName,
214
- blobsTreeName,
215
222
  chunksBlobName,
216
223
  createRootSummarizerNodeWithGC,
217
224
  electedSummarizerBlobName,
@@ -677,23 +684,26 @@ export const makeLegacySendBatchFn =
677
684
  };
678
685
 
679
686
  /** Helper type for type constraints passed through several functions.
687
+ * local - Did this client send the op?
688
+ * savedOp - Is this op being replayed after being serialized (having been sequenced previously)
689
+ * batchStartCsn - The clientSequenceNumber given on submit to the start of this batch
680
690
  * message - The unpacked message. Likely a TypedContainerRuntimeMessage, but could also be a system op
681
691
  * modernRuntimeMessage - Does this appear like a current TypedContainerRuntimeMessage?
682
- * local - Did this client send the op?
683
692
  */
684
- type MessageWithContext =
693
+ type MessageWithContext = {
694
+ local: boolean;
695
+ savedOp?: boolean;
696
+ batchStartCsn: number;
697
+ } & (
685
698
  | {
686
699
  message: InboundSequencedContainerRuntimeMessage;
687
700
  modernRuntimeMessage: true;
688
- local: boolean;
689
- savedOp?: boolean;
690
701
  }
691
702
  | {
692
703
  message: InboundSequencedContainerRuntimeMessageOrSystemMessage;
693
704
  modernRuntimeMessage: false;
694
- local: boolean;
695
- savedOp?: boolean;
696
- };
705
+ }
706
+ );
697
707
 
698
708
  const summarizerRequestUrl = "_summarizer";
699
709
 
@@ -857,18 +867,7 @@ export class ContainerRuntime
857
867
  ]);
858
868
 
859
869
  // read snapshot blobs needed for BlobManager to load
860
- const blobManagerSnapshot = await BlobManager.load(
861
- context.baseSnapshot?.trees[blobsTreeName],
862
- async (id) => {
863
- // IContainerContext storage api return type still has undefined in 0.39 package version.
864
- // So once we release 0.40 container-defn package we can remove this check.
865
- assert(
866
- context.storage !== undefined,
867
- 0x256 /* "storage undefined in attached container" */,
868
- );
869
- return readAndParse(context.storage, id);
870
- },
871
- );
870
+ const blobManagerSnapshot = await loadBlobManagerLoadInfo(context);
872
871
 
873
872
  const messageAtLastSummary = lastMessageFromMetadata(metadata);
874
873
 
@@ -2221,7 +2220,7 @@ export class ContainerRuntime
2221
2220
  return this.resolveHandle(requestParser.createSubRequest(1));
2222
2221
  }
2223
2222
 
2224
- if (id === BlobManager.basePath && requestParser.isLeaf(2)) {
2223
+ if (id === blobManagerBasePath && requestParser.isLeaf(2)) {
2225
2224
  const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
2226
2225
  return blob
2227
2226
  ? {
@@ -2619,7 +2618,13 @@ export class ContainerRuntime
2619
2618
  // but will not modify the contents object (likely it will replace it on the message).
2620
2619
  const messageCopy = { ...messageArg };
2621
2620
  const savedOp = (messageCopy.metadata as ISavedOpMetadata)?.savedOp;
2622
- for (const message of this.remoteMessageProcessor.process(messageCopy)) {
2621
+ const processResult = this.remoteMessageProcessor.process(messageCopy);
2622
+ if (processResult === undefined) {
2623
+ // This means the incoming message is an incomplete part of a message or batch
2624
+ // and we need to process more messages before the rest of the system can understand it.
2625
+ return;
2626
+ }
2627
+ for (const message of processResult.messages) {
2623
2628
  const msg: MessageWithContext = modernRuntimeMessage
2624
2629
  ? {
2625
2630
  // Cast it since we expect it to be this based on modernRuntimeMessage computation above.
@@ -2629,12 +2634,14 @@ export class ContainerRuntime
2629
2634
  message: message as InboundSequencedContainerRuntimeMessage,
2630
2635
  local,
2631
2636
  modernRuntimeMessage,
2637
+ batchStartCsn: processResult.batchStartCsn,
2632
2638
  }
2633
2639
  : // Unrecognized message will be ignored.
2634
2640
  {
2635
2641
  message,
2636
2642
  local,
2637
2643
  modernRuntimeMessage,
2644
+ batchStartCsn: processResult.batchStartCsn,
2638
2645
  };
2639
2646
  msg.savedOp = savedOp;
2640
2647
 
@@ -2682,6 +2689,7 @@ export class ContainerRuntime
2682
2689
  if (local && messageWithContext.modernRuntimeMessage) {
2683
2690
  localOpMetadata = this.pendingStateManager.processPendingLocalMessage(
2684
2691
  messageWithContext.message,
2692
+ messageWithContext.batchStartCsn,
2685
2693
  );
2686
2694
  }
2687
2695
 
@@ -3360,7 +3368,7 @@ export class ContainerRuntime
3360
3368
  * blob manager.
3361
3369
  */
3362
3370
  public getNodeType(nodePath: string): GCNodeType {
3363
- if (this.isBlobPath(nodePath)) {
3371
+ if (isBlobPath(nodePath)) {
3364
3372
  return GCNodeType.Blob;
3365
3373
  }
3366
3374
  return this.channelCollection.getGCNodeType(nodePath) ?? GCNodeType.Other;
@@ -3379,7 +3387,7 @@ export class ContainerRuntime
3379
3387
 
3380
3388
  switch (this.getNodeType(nodePath)) {
3381
3389
  case GCNodeType.Blob:
3382
- return [BlobManager.basePath];
3390
+ return [blobManagerBasePath];
3383
3391
  case GCNodeType.DataStore:
3384
3392
  case GCNodeType.SubDataStore:
3385
3393
  return this.channelCollection.getDataStorePackagePath(nodePath);
@@ -3388,17 +3396,6 @@ export class ContainerRuntime
3388
3396
  }
3389
3397
  }
3390
3398
 
3391
- /**
3392
- * Returns whether a given path is for attachment blobs that are in the format - "/BlobManager.basePath/...".
3393
- */
3394
- private isBlobPath(path: string): boolean {
3395
- const pathParts = path.split("/");
3396
- if (pathParts.length < 2 || pathParts[1] !== BlobManager.basePath) {
3397
- return false;
3398
- }
3399
- return true;
3400
- }
3401
-
3402
3399
  /**
3403
3400
  * From a given list of routes, separate and return routes that belong to blob manager and data stores.
3404
3401
  * @param routes - A list of routes that can belong to data stores or blob manager.
@@ -3409,7 +3406,7 @@ export class ContainerRuntime
3409
3406
  const blobManagerRoutes: string[] = [];
3410
3407
  const dataStoreRoutes: string[] = [];
3411
3408
  for (const route of routes) {
3412
- if (this.isBlobPath(route)) {
3409
+ if (isBlobPath(route)) {
3413
3410
  blobManagerRoutes.push(route);
3414
3411
  } else {
3415
3412
  dataStoreRoutes.push(route);
@@ -1058,6 +1058,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1058
1058
  private snapshotFetchRequired: boolean | undefined;
1059
1059
  private readonly runtime: IContainerRuntimeBase;
1060
1060
  private readonly blobContents: Map<string, ArrayBuffer> | undefined;
1061
+ private readonly isSnapshotInISnapshotFormat: boolean | undefined;
1061
1062
 
1062
1063
  constructor(props: IRemoteFluidDataStoreContextProps) {
1063
1064
  super(props, true /* existing */, false /* isLocalDataStore */, () => {
@@ -1068,8 +1069,10 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1068
1069
  if (isInstanceOfISnapshot(props.snapshot)) {
1069
1070
  this.blobContents = props.snapshot.blobContents;
1070
1071
  this._baseSnapshot = props.snapshot.snapshotTree;
1072
+ this.isSnapshotInISnapshotFormat = true;
1071
1073
  } else {
1072
1074
  this._baseSnapshot = props.snapshot;
1075
+ this.isSnapshotInISnapshotFormat = false;
1073
1076
  }
1074
1077
  if (this._baseSnapshot !== undefined) {
1075
1078
  this.summarizerNode.updateBaseSummaryState(this._baseSnapshot);
@@ -1091,10 +1094,13 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
1091
1094
  private readonly initialSnapshotDetailsP = new LazyPromise<ISnapshotDetails>(async () => {
1092
1095
  // Sequence number of the snapshot.
1093
1096
  let sequenceNumber: number | undefined;
1094
- // Check whether we need to fetch the snapshot first to load.
1097
+ // Check whether we need to fetch the snapshot first to load. The snapshot should be in new format to see
1098
+ // whether we want to evaluate to fetch snapshot or not for loadingGroupId. Otherwise, the snapshot
1099
+ // will contain all the blobs.
1095
1100
  if (
1096
1101
  this.snapshotFetchRequired === undefined &&
1097
- this._baseSnapshot?.groupId !== undefined
1102
+ this._baseSnapshot?.groupId !== undefined &&
1103
+ this.isSnapshotInISnapshotFormat
1098
1104
  ) {
1099
1105
  assert(
1100
1106
  this.blobContents !== undefined,
@@ -26,7 +26,7 @@ import {
26
26
  tagCodeArtifacts,
27
27
  } from "@fluidframework/telemetry-utils/internal";
28
28
 
29
- import { BlobManager } from "../blobManager.js";
29
+ import { blobManagerBasePath } from "../blobManager/index.js";
30
30
  import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
31
31
  import { ClientSessionExpiredError } from "../error.js";
32
32
  import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
@@ -1272,7 +1272,7 @@ export class GarbageCollector implements IGarbageCollector {
1272
1272
  // be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
1273
1273
  const getDeletedNodeType = (nodeId: string): GCNodeType => {
1274
1274
  const pathParts = nodeId.split("/");
1275
- if (pathParts[1] === BlobManager.basePath) {
1275
+ if (pathParts[1] === blobManagerBasePath) {
1276
1276
  return GCNodeType.Blob;
1277
1277
  }
1278
1278
  if (pathParts.length === 2) {
package/src/index.ts CHANGED
@@ -30,7 +30,7 @@ export {
30
30
  RecentlyAddedContainerRuntimeMessageDetails,
31
31
  UnknownContainerRuntimeMessage,
32
32
  } from "./messageTypes.js";
33
- export { IBlobManagerLoadInfo } from "./blobManager.js";
33
+ export { IBlobManagerLoadInfo } from "./blobManager/index.js";
34
34
  export { FluidDataStoreRegistry } from "./dataStoreRegistry.js";
35
35
  export {
36
36
  detectOutboundReferences,