@fluidframework/datastore 1.4.0-115997 → 2.0.0-dev-rc.1.0.0.224419

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (145) hide show
  1. package/.eslintrc.js +5 -7
  2. package/.mocharc.js +12 -0
  3. package/CHANGELOG.md +273 -0
  4. package/README.md +41 -0
  5. package/api-extractor-lint.json +4 -0
  6. package/api-extractor.json +2 -2
  7. package/api-report/datastore.api.md +168 -0
  8. package/dist/channelContext.cjs +86 -0
  9. package/dist/channelContext.cjs.map +1 -0
  10. package/dist/channelContext.d.ts +15 -9
  11. package/dist/channelContext.d.ts.map +1 -1
  12. package/dist/{channelDeltaConnection.js → channelDeltaConnection.cjs} +14 -15
  13. package/dist/channelDeltaConnection.cjs.map +1 -0
  14. package/dist/channelDeltaConnection.d.ts +4 -5
  15. package/dist/channelDeltaConnection.d.ts.map +1 -1
  16. package/dist/{channelStorageService.js → channelStorageService.cjs} +13 -16
  17. package/dist/channelStorageService.cjs.map +1 -0
  18. package/dist/channelStorageService.d.ts +2 -2
  19. package/dist/channelStorageService.d.ts.map +1 -1
  20. package/dist/{dataStoreRuntime.js → dataStoreRuntime.cjs} +302 -225
  21. package/dist/dataStoreRuntime.cjs.map +1 -0
  22. package/dist/dataStoreRuntime.d.ts +81 -37
  23. package/dist/dataStoreRuntime.d.ts.map +1 -1
  24. package/dist/datastore-alpha.d.ts +317 -0
  25. package/dist/datastore-beta.d.ts +47 -0
  26. package/dist/datastore-public.d.ts +47 -0
  27. package/dist/datastore-untrimmed.d.ts +324 -0
  28. package/dist/{fluidHandle.js → fluidHandle.cjs} +44 -16
  29. package/dist/fluidHandle.cjs.map +1 -0
  30. package/dist/fluidHandle.d.ts +33 -6
  31. package/dist/fluidHandle.d.ts.map +1 -1
  32. package/dist/index.cjs +15 -0
  33. package/dist/index.cjs.map +1 -0
  34. package/dist/index.d.ts +2 -2
  35. package/dist/index.d.ts.map +1 -1
  36. package/dist/localChannelContext.cjs +190 -0
  37. package/dist/localChannelContext.cjs.map +1 -0
  38. package/dist/localChannelContext.d.ts +12 -21
  39. package/dist/localChannelContext.d.ts.map +1 -1
  40. package/dist/{localChannelStorageService.js → localChannelStorageService.cjs} +3 -3
  41. package/dist/localChannelStorageService.cjs.map +1 -0
  42. package/dist/localChannelStorageService.d.ts.map +1 -1
  43. package/dist/remoteChannelContext.cjs +124 -0
  44. package/dist/remoteChannelContext.cjs.map +1 -0
  45. package/dist/remoteChannelContext.d.ts +5 -10
  46. package/dist/remoteChannelContext.d.ts.map +1 -1
  47. package/dist/tsdoc-metadata.json +11 -0
  48. package/lib/{channelContext.d.ts → channelContext.d.mts} +16 -10
  49. package/lib/channelContext.d.mts.map +1 -0
  50. package/lib/channelContext.mjs +78 -0
  51. package/lib/channelContext.mjs.map +1 -0
  52. package/lib/{channelDeltaConnection.d.ts → channelDeltaConnection.d.mts} +4 -5
  53. package/lib/channelDeltaConnection.d.mts.map +1 -0
  54. package/lib/{channelDeltaConnection.js → channelDeltaConnection.mjs} +11 -12
  55. package/lib/channelDeltaConnection.mjs.map +1 -0
  56. package/lib/{channelStorageService.d.ts → channelStorageService.d.mts} +2 -2
  57. package/lib/channelStorageService.d.mts.map +1 -0
  58. package/lib/{channelStorageService.js → channelStorageService.mjs} +13 -16
  59. package/lib/channelStorageService.mjs.map +1 -0
  60. package/lib/{dataStoreRuntime.d.ts → dataStoreRuntime.d.mts} +81 -37
  61. package/lib/dataStoreRuntime.d.mts.map +1 -0
  62. package/lib/{dataStoreRuntime.js → dataStoreRuntime.mjs} +286 -209
  63. package/lib/dataStoreRuntime.mjs.map +1 -0
  64. package/lib/datastore-alpha.d.mts +317 -0
  65. package/lib/datastore-beta.d.mts +47 -0
  66. package/lib/datastore-public.d.mts +47 -0
  67. package/lib/datastore-untrimmed.d.mts +324 -0
  68. package/lib/fluidHandle.d.mts +57 -0
  69. package/lib/fluidHandle.d.mts.map +1 -0
  70. package/lib/{fluidHandle.js → fluidHandle.mjs} +44 -16
  71. package/lib/fluidHandle.mjs.map +1 -0
  72. package/lib/index.d.mts +7 -0
  73. package/lib/index.d.mts.map +1 -0
  74. package/lib/index.mjs +7 -0
  75. package/lib/index.mjs.map +1 -0
  76. package/lib/{localChannelContext.d.ts → localChannelContext.d.mts} +13 -22
  77. package/lib/localChannelContext.d.mts.map +1 -0
  78. package/lib/{localChannelContext.js → localChannelContext.mjs} +73 -85
  79. package/lib/localChannelContext.mjs.map +1 -0
  80. package/lib/localChannelStorageService.d.mts.map +1 -0
  81. package/lib/{localChannelStorageService.js → localChannelStorageService.mjs} +2 -2
  82. package/lib/localChannelStorageService.mjs.map +1 -0
  83. package/lib/{remoteChannelContext.d.ts → remoteChannelContext.d.mts} +7 -12
  84. package/lib/remoteChannelContext.d.mts.map +1 -0
  85. package/lib/remoteChannelContext.mjs +120 -0
  86. package/lib/remoteChannelContext.mjs.map +1 -0
  87. package/package.json +107 -72
  88. package/{lib/index.js → prettier.config.cjs} +4 -3
  89. package/src/channelContext.ts +168 -71
  90. package/src/channelDeltaConnection.ts +52 -47
  91. package/src/channelStorageService.ts +59 -55
  92. package/src/dataStoreRuntime.ts +1158 -983
  93. package/src/fluidHandle.ts +92 -64
  94. package/src/index.ts +8 -2
  95. package/src/localChannelContext.ts +278 -272
  96. package/src/localChannelStorageService.ts +48 -46
  97. package/src/remoteChannelContext.ts +237 -300
  98. package/tsc-multi.test.json +4 -0
  99. package/tsconfig.json +11 -13
  100. package/dist/channelContext.js +0 -35
  101. package/dist/channelContext.js.map +0 -1
  102. package/dist/channelDeltaConnection.js.map +0 -1
  103. package/dist/channelStorageService.js.map +0 -1
  104. package/dist/dataStoreRuntime.js.map +0 -1
  105. package/dist/fluidHandle.js.map +0 -1
  106. package/dist/index.js +0 -19
  107. package/dist/index.js.map +0 -1
  108. package/dist/localChannelContext.js +0 -202
  109. package/dist/localChannelContext.js.map +0 -1
  110. package/dist/localChannelStorageService.js.map +0 -1
  111. package/dist/packageVersion.d.ts +0 -9
  112. package/dist/packageVersion.d.ts.map +0 -1
  113. package/dist/packageVersion.js +0 -12
  114. package/dist/packageVersion.js.map +0 -1
  115. package/dist/remoteChannelContext.js +0 -207
  116. package/dist/remoteChannelContext.js.map +0 -1
  117. package/lib/channelContext.d.ts.map +0 -1
  118. package/lib/channelContext.js +0 -29
  119. package/lib/channelContext.js.map +0 -1
  120. package/lib/channelDeltaConnection.d.ts.map +0 -1
  121. package/lib/channelDeltaConnection.js.map +0 -1
  122. package/lib/channelStorageService.d.ts.map +0 -1
  123. package/lib/channelStorageService.js.map +0 -1
  124. package/lib/dataStoreRuntime.d.ts.map +0 -1
  125. package/lib/dataStoreRuntime.js.map +0 -1
  126. package/lib/fluidHandle.d.ts +0 -30
  127. package/lib/fluidHandle.d.ts.map +0 -1
  128. package/lib/fluidHandle.js.map +0 -1
  129. package/lib/index.d.ts +0 -7
  130. package/lib/index.d.ts.map +0 -1
  131. package/lib/index.js.map +0 -1
  132. package/lib/localChannelContext.d.ts.map +0 -1
  133. package/lib/localChannelContext.js.map +0 -1
  134. package/lib/localChannelStorageService.d.ts.map +0 -1
  135. package/lib/localChannelStorageService.js.map +0 -1
  136. package/lib/packageVersion.d.ts +0 -9
  137. package/lib/packageVersion.d.ts.map +0 -1
  138. package/lib/packageVersion.js +0 -9
  139. package/lib/packageVersion.js.map +0 -1
  140. package/lib/remoteChannelContext.d.ts.map +0 -1
  141. package/lib/remoteChannelContext.js +0 -203
  142. package/lib/remoteChannelContext.js.map +0 -1
  143. package/src/packageVersion.ts +0 -9
  144. package/tsconfig.esnext.json +0 -7
  145. /package/lib/{localChannelStorageService.d.ts → localChannelStorageService.d.mts} +0 -0
@@ -3,969 +3,1131 @@
3
3
  * Licensed under the MIT License.
4
4
  */
5
5
 
6
- import { ITelemetryLogger } from "@fluidframework/common-definitions";
6
+ import { TypedEventEmitter } from "@fluid-internal/client-utils";
7
7
  import {
8
- IFluidHandle,
9
- IFluidHandleContext,
10
- IRequest,
11
- IResponse,
8
+ DataProcessingError,
9
+ ITelemetryLoggerExt,
10
+ generateStack,
11
+ LoggingError,
12
+ MonitoringContext,
13
+ raiseConnectedEvent,
14
+ createChildMonitoringContext,
15
+ tagCodeArtifacts,
16
+ UsageError,
17
+ } from "@fluidframework/telemetry-utils";
18
+ import {
19
+ FluidObject,
20
+ IFluidHandle,
21
+ IFluidHandleContext,
22
+ IRequest,
23
+ IResponse,
12
24
  } from "@fluidframework/core-interfaces";
25
+ import { assert, Deferred, LazyPromise, unreachableCase } from "@fluidframework/core-utils";
13
26
  import {
14
- IAudience,
15
- IDeltaManager,
16
- BindState,
17
- AttachState,
18
- ILoaderOptions,
27
+ IAudience,
28
+ IDeltaManager,
29
+ AttachState,
30
+ ILoaderOptions,
19
31
  } from "@fluidframework/container-definitions";
20
- import { DataProcessingError, UsageError } from "@fluidframework/container-utils";
21
- import {
22
- assert,
23
- Deferred,
24
- LazyPromise,
25
- TypedEventEmitter,
26
- unreachableCase,
27
- } from "@fluidframework/common-utils";
28
- import {
29
- ChildLogger,
30
- LoggingError,
31
- raiseConnectedEvent,
32
- } from "@fluidframework/telemetry-utils";
33
32
  import { buildSnapshotTree } from "@fluidframework/driver-utils";
34
33
  import {
35
- IClientDetails,
36
- IDocumentMessage,
37
- ISequencedDocumentMessage,
38
- SummaryType,
39
- ISummaryBlob,
40
- ISummaryTree,
41
- IQuorumClients,
34
+ IClientDetails,
35
+ IDocumentMessage,
36
+ ISequencedDocumentMessage,
37
+ SummaryType,
38
+ ISummaryBlob,
39
+ ISummaryTree,
40
+ IQuorumClients,
42
41
  } from "@fluidframework/protocol-definitions";
43
42
  import {
44
- CreateSummarizerNodeSource,
45
- IAttachMessage,
46
- IEnvelope,
47
- IFluidDataStoreContext,
48
- IFluidDataStoreChannel,
49
- IGarbageCollectionData,
50
- IGarbageCollectionDetailsBase,
51
- IInboundSignalMessage,
52
- ISummaryTreeWithStats,
53
- VisibilityState,
54
- ITelemetryContext,
43
+ CreateChildSummarizerNodeParam,
44
+ CreateSummarizerNodeSource,
45
+ IAttachMessage,
46
+ IEnvelope,
47
+ IFluidDataStoreContext,
48
+ IFluidDataStoreChannel,
49
+ IGarbageCollectionData,
50
+ IInboundSignalMessage,
51
+ ISummaryTreeWithStats,
52
+ VisibilityState,
53
+ ITelemetryContext,
54
+ IIdCompressor,
55
55
  } from "@fluidframework/runtime-definitions";
56
56
  import {
57
- convertSnapshotTreeToSummaryTree,
58
- convertSummaryTreeToITree,
59
- generateHandleContextPath,
60
- RequestParser,
61
- SummaryTreeBuilder,
62
- create404Response,
63
- createResponseError,
64
- exceptionToResponse,
57
+ convertSnapshotTreeToSummaryTree,
58
+ convertSummaryTreeToITree,
59
+ generateHandleContextPath,
60
+ RequestParser,
61
+ SummaryTreeBuilder,
62
+ create404Response,
63
+ createResponseError,
64
+ exceptionToResponse,
65
+ GCDataBuilder,
66
+ unpackChildNodesUsedRoutes,
65
67
  } from "@fluidframework/runtime-utils";
66
68
  import {
67
- IChannel,
68
- IFluidDataStoreRuntime,
69
- IFluidDataStoreRuntimeEvents,
70
- IChannelFactory,
69
+ IChannel,
70
+ IFluidDataStoreRuntime,
71
+ IFluidDataStoreRuntimeEvents,
72
+ IChannelFactory,
71
73
  } from "@fluidframework/datastore-definitions";
72
- import {
73
- GCDataBuilder,
74
- removeRouteFromAllNodes,
75
- unpackChildNodesGCDetails,
76
- unpackChildNodesUsedRoutes,
77
- } from "@fluidframework/garbage-collector";
78
74
  import { v4 as uuid } from "uuid";
79
75
  import { IChannelContext, summarizeChannel } from "./channelContext";
80
- import { LocalChannelContext, LocalChannelContextBase, RehydratedLocalChannelContext } from "./localChannelContext";
76
+ import {
77
+ LocalChannelContext,
78
+ LocalChannelContextBase,
79
+ RehydratedLocalChannelContext,
80
+ } from "./localChannelContext";
81
81
  import { RemoteChannelContext } from "./remoteChannelContext";
82
+ import { FluidObjectHandle } from "./fluidHandle";
82
83
 
84
+ /**
85
+ * @alpha
86
+ */
83
87
  export enum DataStoreMessageType {
84
- // Creates a new channel
85
- Attach = "attach",
86
- ChannelOp = "op",
88
+ // Creates a new channel
89
+ Attach = "attach",
90
+ ChannelOp = "op",
87
91
  }
88
92
 
93
+ /**
94
+ * @alpha
95
+ */
89
96
  export interface ISharedObjectRegistry {
90
- // TODO consider making this async. A consequence is that either the creation of a distributed data type
91
- // is async or we need a new API to split the synchronous vs. asynchronous creation.
92
- get(name: string): IChannelFactory | undefined;
97
+ // TODO consider making this async. A consequence is that either the creation of a distributed data type
98
+ // is async or we need a new API to split the synchronous vs. asynchronous creation.
99
+ get(name: string): IChannelFactory | undefined;
93
100
  }
94
101
 
95
102
  /**
96
103
  * Base data store class
104
+ * @alpha
97
105
  */
98
- export class FluidDataStoreRuntime extends
99
- TypedEventEmitter<IFluidDataStoreRuntimeEvents> implements
100
- IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext {
101
- /**
102
- * Loads the data store runtime
103
- * @param context - The data store context
104
- * @param sharedObjectRegistry - The registry of shared objects used by this data store
105
- * @param existing - If loading from an existing file.
106
- */
107
- public static load(
108
- context: IFluidDataStoreContext,
109
- sharedObjectRegistry: ISharedObjectRegistry,
110
- existing: boolean,
111
- ): FluidDataStoreRuntime {
112
- return new FluidDataStoreRuntime(context, sharedObjectRegistry, existing);
113
- }
114
-
115
- public get IFluidRouter() { return this; }
116
-
117
- public get connected(): boolean {
118
- return this.dataStoreContext.connected;
119
- }
120
-
121
- public get clientId(): string | undefined {
122
- return this.dataStoreContext.clientId;
123
- }
124
-
125
- public get clientDetails(): IClientDetails {
126
- return this.dataStoreContext.clientDetails;
127
- }
128
-
129
- public get isAttached(): boolean {
130
- return this.attachState !== AttachState.Detached;
131
- }
132
-
133
- public get attachState(): AttachState {
134
- return this._attachState;
135
- }
136
-
137
- public get absolutePath(): string {
138
- return generateHandleContextPath(this.id, this.routeContext);
139
- }
140
-
141
- public get routeContext(): IFluidHandleContext {
142
- return this.dataStoreContext.IFluidHandleContext;
143
- }
144
-
145
- public get IFluidHandleContext() { return this; }
146
-
147
- public get rootRoutingContext() { return this; }
148
- public get channelsRoutingContext() { return this; }
149
- public get objectsRoutingContext() { return this; }
150
-
151
- private _disposed = false;
152
- public get disposed() { return this._disposed; }
153
-
154
- private readonly contexts = new Map<string, IChannelContext>();
155
- private readonly contextsDeferred = new Map<string, Deferred<IChannelContext>>();
156
- private readonly pendingAttach = new Map<string, IAttachMessage>();
157
-
158
- private bindState: BindState;
159
- private readonly deferredAttached = new Deferred<void>();
160
- private readonly localChannelContextQueue = new Map<string, LocalChannelContextBase>();
161
- private readonly notBoundedChannelContextSet = new Set<string>();
162
- private _attachState: AttachState;
163
- public visibilityState: VisibilityState;
164
- // A list of handles that are bound when the data store is not visible. We have to make them visible when the data
165
- // store becomes visible.
166
- private readonly pendingHandlesToMakeVisible: Set<IFluidHandle> = new Set();
167
-
168
- public readonly id: string;
169
- public readonly options: ILoaderOptions;
170
- public readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>;
171
- private readonly quorum: IQuorumClients;
172
- private readonly audience: IAudience;
173
- public readonly logger: ITelemetryLogger;
174
-
175
- // A map of child channel context ids to the their base GC details. This is used to initialize the GC state of the
176
- // channel contexts.
177
- private readonly channelsBaseGCDetails: LazyPromise<Map<string, IGarbageCollectionDetailsBase>>;
178
-
179
- public constructor(
180
- private readonly dataStoreContext: IFluidDataStoreContext,
181
- private readonly sharedObjectRegistry: ISharedObjectRegistry,
182
- existing: boolean,
183
- ) {
184
- super();
185
-
186
- assert(!dataStoreContext.id.includes("/"),
187
- 0x30e /* Id cannot contain slashes. DataStoreContext should have validated this. */);
188
-
189
- this.logger = ChildLogger.create(
190
- dataStoreContext.logger,
191
- "FluidDataStoreRuntime",
192
- { all: { dataStoreId: uuid() } },
193
- );
194
-
195
- this.id = dataStoreContext.id;
196
- this.options = dataStoreContext.options;
197
- this.deltaManager = dataStoreContext.deltaManager;
198
- this.quorum = dataStoreContext.getQuorum();
199
- this.audience = dataStoreContext.getAudience();
200
-
201
- const tree = dataStoreContext.baseSnapshot;
202
-
203
- this.channelsBaseGCDetails = new LazyPromise(async () => {
204
- const baseGCDetails = await
205
- (this.dataStoreContext.getBaseGCDetails?.() ?? this.dataStoreContext.getInitialGCSummaryDetails());
206
- return unpackChildNodesGCDetails(baseGCDetails);
207
- });
208
-
209
- // Must always receive the data store type inside of the attributes
210
- if (tree?.trees !== undefined) {
211
- Object.keys(tree.trees).forEach((path) => {
212
- // Issue #4414
213
- if (path === "_search") { return; }
214
-
215
- let channelContext: IChannelContext;
216
- // If already exists on storage, then create a remote channel. However, if it is case of rehydrating a
217
- // container from snapshot where we load detached container from a snapshot, isLocalDataStore would be
218
- // true. In this case create a RehydratedLocalChannelContext.
219
- if (dataStoreContext.isLocalDataStore) {
220
- channelContext = new RehydratedLocalChannelContext(
221
- path,
222
- this.sharedObjectRegistry,
223
- this,
224
- this.dataStoreContext,
225
- this.dataStoreContext.storage,
226
- this.logger,
227
- (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata),
228
- (address: string) => this.setChannelDirty(address),
229
- (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
230
- this.addedGCOutboundReference(srcHandle, outboundHandle),
231
- tree.trees[path]);
232
- // This is the case of rehydrating a detached container from snapshot. Now due to delay loading of
233
- // data store, if the data store is loaded after the container is attached, then we missed making
234
- // the channel visible. So do it now. Otherwise, add it to local channel context queue, so
235
- // that it can be make it visible later with the data store.
236
- if (dataStoreContext.attachState !== AttachState.Detached) {
237
- (channelContext as LocalChannelContextBase).makeVisible();
238
- } else {
239
- this.localChannelContextQueue.set(path, channelContext as LocalChannelContextBase);
240
- }
241
- } else {
242
- channelContext = new RemoteChannelContext(
243
- this,
244
- dataStoreContext,
245
- dataStoreContext.storage,
246
- (content, localOpMetadata) => this.submitChannelOp(path, content, localOpMetadata),
247
- (address: string) => this.setChannelDirty(address),
248
- (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
249
- this.addedGCOutboundReference(srcHandle, outboundHandle),
250
- path,
251
- tree.trees[path],
252
- this.sharedObjectRegistry,
253
- undefined /* extraBlobs */,
254
- this.dataStoreContext.getCreateChildSummarizerNodeFn(
255
- path,
256
- { type: CreateSummarizerNodeSource.FromSummary },
257
- ),
258
- async () => this.getChannelBaseGCDetails(path));
259
- }
260
- const deferred = new Deferred<IChannelContext>();
261
- deferred.resolve(channelContext);
262
-
263
- this.contexts.set(path, channelContext);
264
- this.contextsDeferred.set(path, deferred);
265
- });
266
- }
267
-
268
- this.attachListener();
269
- // If exists on storage or loaded from a snapshot, it should already be bound.
270
- this.bindState = existing ? BindState.Bound : BindState.NotBound;
271
- this._attachState = dataStoreContext.attachState;
272
-
273
- /**
274
- * If existing flag is false, this is a new data store and is not visible. The existing flag can be true in two
275
- * conditions:
276
- * 1. It's a local data store that is created when a detached container is rehydrated. In this case, the data
277
- * store is locally visible because the snapshot it is loaded from contains locally visible data stores only.
278
- * 2. It's a remote data store that is created when an attached container is loaded is loaded from snapshot or
279
- * when an attach op comes in. In both these cases, the data store is already globally visible.
280
- */
281
- if (existing) {
282
- this.visibilityState = dataStoreContext.attachState === AttachState.Detached
283
- ? VisibilityState.LocallyVisible : VisibilityState.GloballyVisible;
284
- } else {
285
- this.visibilityState = VisibilityState.NotVisible;
286
- }
287
-
288
- // If it's existing we know it has been attached.
289
- if (existing) {
290
- this.deferredAttached.resolve();
291
- }
292
- }
293
-
294
- public dispose(): void {
295
- if (this._disposed) {
296
- return;
297
- }
298
- this._disposed = true;
299
-
300
- this.emit("dispose");
301
- this.removeAllListeners();
302
- }
303
-
304
- public async resolveHandle(request: IRequest): Promise<IResponse> {
305
- return this.request(request);
306
- }
307
-
308
- public async request(request: IRequest): Promise<IResponse> {
309
- try {
310
- const parser = RequestParser.create(request);
311
- const id = parser.pathParts[0];
312
-
313
- if (id === "_channels" || id === "_custom") {
314
- return this.request(parser.createSubRequest(1));
315
- }
316
-
317
- // Check for a data type reference first
318
- if (this.contextsDeferred.has(id) && parser.isLeaf(1)) {
319
- try {
320
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
321
- const value = await this.contextsDeferred.get(id)!.promise;
322
- const channel = await value.getChannel();
323
-
324
- return { mimeType: "fluid/object", status: 200, value: channel };
325
- } catch (error) {
326
- this.logger.sendErrorEvent({ eventName: "GetChannelFailedInRequest" }, error);
327
-
328
- return createResponseError(500, `Failed to get Channel: ${error}`, request);
329
- }
330
- }
331
-
332
- // Otherwise defer to an attached request handler
333
- return create404Response(request);
334
- } catch (error) {
335
- return exceptionToResponse(error);
336
- }
337
- }
338
-
339
- public async getChannel(id: string): Promise<IChannel> {
340
- this.verifyNotClosed();
341
-
342
- // TODO we don't assume any channels (even root) in the runtime. If you request a channel that doesn't exist
343
- // we will never resolve the promise. May want a flag to getChannel that doesn't wait for the promise if
344
- // it doesn't exist
345
- if (!this.contextsDeferred.has(id)) {
346
- this.contextsDeferred.set(id, new Deferred<IChannelContext>());
347
- }
348
-
349
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
350
- const context = await this.contextsDeferred.get(id)!.promise;
351
- const channel = await context.getChannel();
352
-
353
- return channel;
354
- }
355
-
356
- public createChannel(id: string = uuid(), type: string): IChannel {
357
- if (id.includes("/")) {
358
- throw new UsageError(`Id cannot contain slashes: ${id}`);
359
- }
360
-
361
- this.verifyNotClosed();
362
-
363
- assert(!this.contexts.has(id), 0x179 /* "createChannel() with existing ID" */);
364
- this.notBoundedChannelContextSet.add(id);
365
- const context = new LocalChannelContext(
366
- id,
367
- this.sharedObjectRegistry,
368
- type,
369
- this,
370
- this.dataStoreContext,
371
- this.dataStoreContext.storage,
372
- this.logger,
373
- (content, localOpMetadata) => this.submitChannelOp(id, content, localOpMetadata),
374
- (address: string) => this.setChannelDirty(address),
375
- (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
376
- this.addedGCOutboundReference(srcHandle, outboundHandle));
377
- this.contexts.set(id, context);
378
-
379
- if (this.contextsDeferred.has(id)) {
380
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
381
- this.contextsDeferred.get(id)!.resolve(context);
382
- } else {
383
- const deferred = new Deferred<IChannelContext>();
384
- deferred.resolve(context);
385
- this.contextsDeferred.set(id, deferred);
386
- }
387
-
388
- assert(!!context.channel, 0x17a /* "Channel should be loaded when created!!" */);
389
- return context.channel;
390
- }
391
-
392
- /**
393
- * Binds a channel with the runtime. If the runtime is attached we will attach the channel right away.
394
- * If the runtime is not attached we will defer the attach until the runtime attaches.
395
- * @param channel - channel to be registered.
396
- */
397
- public bindChannel(channel: IChannel): void {
398
- assert(this.notBoundedChannelContextSet.has(channel.id),
399
- 0x17b /* "Channel to be binded should be in not bounded set" */);
400
- this.notBoundedChannelContextSet.delete(channel.id);
401
- // If our data store is attached, then attach the channel.
402
- if (this.isAttached) {
403
- this.attachChannel(channel);
404
- return;
405
- }
406
-
407
- /**
408
- * If this channel is already waiting to be made visible, do nothing. This can happen during attachGraph() when
409
- * a channel's graph is attached. It calls bindToContext on the shared object which will end up back here.
410
- */
411
- if (this.pendingHandlesToMakeVisible.has(channel.handle)) {
412
- return;
413
- }
414
-
415
- this.bind(channel.handle);
416
-
417
- // If our data store is local then add the channel to the queue
418
- if (!this.localChannelContextQueue.has(channel.id)) {
419
- this.localChannelContextQueue.set(channel.id, this.contexts.get(channel.id) as LocalChannelContextBase);
420
- }
421
- }
422
-
423
- /**
424
- * This function is called when a data store becomes root. It does the following:
425
- * 1. Marks the data store locally visible in the container.
426
- * 2. Attaches the graph of all the handles bound to it.
427
- * 3. Calls into the data store context to mark it visible in the container too. If the container is globally
428
- * visible, it will mark us globally visible. Otherwise, it will mark us globally visible when it becomes
429
- * globally visible.
430
- */
431
- public makeVisibleAndAttachGraph() {
432
- if (this.visibilityState !== VisibilityState.NotVisible) {
433
- return;
434
- }
435
- this.visibilityState = VisibilityState.LocallyVisible;
436
-
437
- this.pendingHandlesToMakeVisible.forEach((handle) => {
438
- handle.attachGraph();
439
- });
440
- this.pendingHandlesToMakeVisible.clear();
441
- this.bindToContext();
442
- }
443
-
444
- /**
445
- * This function is called when a handle to this data store is added to a visible DDS.
446
- */
447
- public attachGraph() {
448
- this.makeVisibleAndAttachGraph();
449
- }
450
-
451
- /**
452
- * Binds this runtime to the container
453
- * This includes the following:
454
- * 1. Sending an Attach op that includes all existing state
455
- * 2. Attaching the graph if the data store becomes attached.
456
- */
457
- public bindToContext() {
458
- if (this.bindState !== BindState.NotBound) {
459
- return;
460
- }
461
- this.bindState = BindState.Binding;
462
- this.dataStoreContext.bindToContext();
463
- this.bindState = BindState.Bound;
464
- }
465
-
466
- public bind(handle: IFluidHandle): void {
467
- // If visible, attach the incoming handle's graph. Else, this will be done when we become visible.
468
- if (this.visibilityState !== VisibilityState.NotVisible) {
469
- handle.attachGraph();
470
- return;
471
- }
472
- this.pendingHandlesToMakeVisible.add(handle);
473
- }
474
-
475
- public setConnectionState(connected: boolean, clientId?: string) {
476
- this.verifyNotClosed();
477
-
478
- for (const [, object] of this.contexts) {
479
- object.setConnectionState(connected, clientId);
480
- }
481
-
482
- raiseConnectedEvent(this.logger, this, connected, clientId);
483
- }
484
-
485
- public getQuorum(): IQuorumClients {
486
- return this.quorum;
487
- }
488
-
489
- public getAudience(): IAudience {
490
- return this.audience;
491
- }
492
-
493
- public async uploadBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>> {
494
- this.verifyNotClosed();
495
-
496
- return this.dataStoreContext.uploadBlob(blob);
497
- }
498
-
499
- public process(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown) {
500
- this.verifyNotClosed();
501
-
502
- try {
503
- // catches as data processing error whether or not they come from async pending queues
504
- switch (message.type) {
505
- case DataStoreMessageType.Attach: {
506
- const attachMessage = message.contents as IAttachMessage;
507
- const id = attachMessage.id;
508
-
509
- // If a non-local operation then go and create the object
510
- // Otherwise mark it as officially attached.
511
- if (local) {
512
- assert(this.pendingAttach.has(id), 0x17c /* "Unexpected attach (local) channel OP" */);
513
- this.pendingAttach.delete(id);
514
- } else {
515
- assert(!this.contexts.has(id),
516
- 0x17d, /* `Unexpected attach channel OP,
517
- is in pendingAttach set: ${this.pendingAttach.has(id)},
518
- is local channel contexts: ${this.contexts.get(id) instanceof LocalChannelContextBase}` */);
519
-
520
- const flatBlobs = new Map<string, ArrayBufferLike>();
521
- const snapshotTree = buildSnapshotTree(attachMessage.snapshot.entries, flatBlobs);
522
-
523
- const remoteChannelContext = new RemoteChannelContext(
524
- this,
525
- this.dataStoreContext,
526
- this.dataStoreContext.storage,
527
- (content, localContentMetadata) => this.submitChannelOp(id, content, localContentMetadata),
528
- (address: string) => this.setChannelDirty(address),
529
- (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
530
- this.addedGCOutboundReference(srcHandle, outboundHandle),
531
- id,
532
- snapshotTree,
533
- this.sharedObjectRegistry,
534
- flatBlobs,
535
- this.dataStoreContext.getCreateChildSummarizerNodeFn(
536
- id,
537
- {
538
- type: CreateSummarizerNodeSource.FromAttach,
539
- sequenceNumber: message.sequenceNumber,
540
- snapshot: attachMessage.snapshot,
541
- },
542
- ),
543
- async () => this.getChannelBaseGCDetails(id),
544
- attachMessage.type);
545
-
546
- this.contexts.set(id, remoteChannelContext);
547
- if (this.contextsDeferred.has(id)) {
548
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
549
- this.contextsDeferred.get(id)!.resolve(remoteChannelContext);
550
- } else {
551
- const deferred = new Deferred<IChannelContext>();
552
- deferred.resolve(remoteChannelContext);
553
- this.contextsDeferred.set(id, deferred);
554
- }
555
- }
556
- break;
557
- }
558
-
559
- case DataStoreMessageType.ChannelOp:
560
- this.processChannelOp(message, local, localOpMetadata);
561
- break;
562
- default:
563
- }
564
-
565
- this.emit("op", message);
566
- } catch (error) {
567
- throw DataProcessingError.wrapIfUnrecognized(error, "fluidDataStoreRuntimeFailedToProcessMessage", message);
568
- }
569
- }
570
-
571
- public processSignal(message: IInboundSignalMessage, local: boolean) {
572
- this.emit("signal", message, local);
573
- }
574
-
575
- private isChannelAttached(id: string): boolean {
576
- return (
577
- // Added in createChannel
578
- // Removed when bindChannel is called
579
- !this.notBoundedChannelContextSet.has(id)
580
- // Added in bindChannel only if this is not attached yet
581
- // Removed when this is attached by calling attachGraph
582
- && !this.localChannelContextQueue.has(id)
583
- // Added in attachChannel called by bindChannel
584
- // Removed when attach op is broadcast
585
- && !this.pendingAttach.has(id)
586
- );
587
- }
588
-
589
- /**
590
- * Returns the outbound routes of this channel. Currently, all contexts in this channel are considered
591
- * referenced and are hence outbound. This will change when we have root and non-root channel contexts.
592
- * The only root contexts will be considered as referenced.
593
- */
594
- private getOutboundRoutes(): string[] {
595
- const outboundRoutes: string[] = [];
596
- for (const [contextId] of this.contexts) {
597
- outboundRoutes.push(`${this.absolutePath}/${contextId}`);
598
- }
599
- return outboundRoutes;
600
- }
601
-
602
- /**
603
- * Updates the GC nodes of this channel. It does the following:
604
- * - Adds a back route to self to all its child GC nodes.
605
- * - Adds a node for this channel.
606
- * @param builder - The builder that contains the GC nodes for this channel's children.
607
- */
608
- private updateGCNodes(builder: GCDataBuilder) {
609
- // Add a back route to self in each child's GC nodes. If any child is referenced, then its parent should
610
- // be considered referenced as well.
611
- builder.addRouteToAllNodes(this.absolutePath);
612
-
613
- // Get the outbound routes and add a GC node for this channel.
614
- builder.addNode("/", this.getOutboundRoutes());
615
- }
616
-
617
- /**
618
- * Generates data used for garbage collection. This includes a list of GC nodes that represent this channel
619
- * including any of its child channel contexts. Each node has a set of outbound routes to other GC nodes in the
620
- * document. It does the following:
621
- * 1. Calls into each child context to get its GC data.
622
- * 2. Prefixes the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
623
- * identified as belonging to the child.
624
- * 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
625
- * the GC data of this channel.
626
- * @param fullGC - true to bypass optimizations and force full generation of GC data.
627
- */
628
- public async getGCData(fullGC: boolean = false): Promise<IGarbageCollectionData> {
629
- const builder = new GCDataBuilder();
630
- // Iterate over each channel context and get their GC data.
631
- await Promise.all(Array.from(this.contexts)
632
- .filter(([contextId, _]) => {
633
- // Get GC data only for attached contexts. Detached contexts are not connected in the GC reference
634
- // graph so any references they might have won't be connected as well.
635
- return this.isChannelAttached(contextId);
636
- }).map(async ([contextId, context]) => {
637
- const contextGCData = await context.getGCData(fullGC);
638
- // Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
639
- // This also gradually builds the id of each node to be a path from the root.
640
- builder.prefixAndAddNodes(contextId, contextGCData.gcNodes);
641
- }));
642
-
643
- this.updateGCNodes(builder);
644
- return builder.getGCData();
645
- }
646
-
647
- /**
648
- * After GC has run, called to notify this channel of routes that are used in it. It calls the child contexts to
649
- * update their used routes.
650
- * @param usedRoutes - The routes that are used in all contexts in this channel.
651
- * @param gcTimestamp - The time when GC was run that generated these used routes. If any node becomes unreferenced
652
- * as part of this GC run, this should be used to update the time when it happens.
653
- */
654
- public updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number) {
655
- // Get a map of channel ids to routes used in it.
656
- const usedContextRoutes = unpackChildNodesUsedRoutes(usedRoutes);
657
-
658
- // Verify that the used routes are correct.
659
- for (const [id] of usedContextRoutes) {
660
- assert(this.contexts.has(id), 0x17e /* "Used route does not belong to any known context" */);
661
- }
662
-
663
- // Update the used routes in each context. Used routes is empty for unused context.
664
- for (const [contextId, context] of this.contexts) {
665
- context.updateUsedRoutes(usedContextRoutes.get(contextId) ?? [], gcTimestamp);
666
- }
667
- }
668
-
669
- /**
670
- * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
671
- * all references added in the system.
672
- * @param srcHandle - The handle of the node that added the reference.
673
- * @param outboundHandle - The handle of the outbound node that is referenced.
674
- */
675
- private addedGCOutboundReference(srcHandle: IFluidHandle, outboundHandle: IFluidHandle) {
676
- this.dataStoreContext.addedGCOutboundReference?.(srcHandle, outboundHandle);
677
- }
678
-
679
- /**
680
- * Returns the base GC details for the channel with the given id. This is used to initialize its GC state.
681
- * @param channelId - The id of the channel context that is asked for the initial GC details.
682
- * @returns the requested channel's base GC details.
683
- */
684
- private async getChannelBaseGCDetails(channelId: string): Promise<IGarbageCollectionDetailsBase> {
685
- let channelBaseGCDetails = (await this.channelsBaseGCDetails).get(channelId);
686
- if (channelBaseGCDetails === undefined) {
687
- channelBaseGCDetails = {};
688
- } else if (channelBaseGCDetails.gcData?.gcNodes !== undefined) {
689
- // Note: if the child channel has an explicit handle route to its parent, it will be removed here and
690
- // expected to be added back by the parent when getGCData is called.
691
- removeRouteFromAllNodes(channelBaseGCDetails.gcData.gcNodes, this.absolutePath);
692
- }
693
-
694
- // Currently, channel context's are always considered used. So, it there are no used routes for it, we still
695
- // need to mark it as used. Add self-route (empty string) to the channel context's used routes.
696
- if (channelBaseGCDetails.usedRoutes === undefined || channelBaseGCDetails.usedRoutes.length === 0) {
697
- channelBaseGCDetails.usedRoutes = [""];
698
- }
699
- return channelBaseGCDetails;
700
- }
701
-
702
- /**
703
- * Returns a summary at the current sequence number.
704
- * @param fullTree - true to bypass optimizations and force a full summary tree
705
- * @param trackState - This tells whether we should track state from this summary.
706
- * @param telemetryContext - summary data passed through the layers for telemetry purposes
707
- */
708
- public async summarize(
709
- fullTree: boolean = false,
710
- trackState: boolean = true,
711
- telemetryContext?: ITelemetryContext,
712
- ): Promise<ISummaryTreeWithStats> {
713
- const summaryBuilder = new SummaryTreeBuilder();
714
-
715
- // Iterate over each data store and ask it to summarize
716
- await Promise.all(Array.from(this.contexts)
717
- .filter(([contextId, _]) => {
718
- const isAttached = this.isChannelAttached(contextId);
719
- // We are not expecting local dds! Summary may not capture local state.
720
- assert(isAttached, 0x17f /* "Not expecting detached channels during summarize" */);
721
- // If the object is registered - and we have received the sequenced op creating the object
722
- // (i.e. it has a base mapping) - then we go ahead and summarize
723
- return isAttached;
724
- }).map(async ([contextId, context]) => {
725
- const contextSummary = await context.summarize(fullTree, trackState, telemetryContext);
726
- summaryBuilder.addWithStats(contextId, contextSummary);
727
- }));
728
-
729
- return summaryBuilder.getSummaryTree();
730
- }
731
-
732
- public getAttachSummary(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
733
- /**
734
- * back-compat 0.59.1000 - getAttachSummary() is called when making a data store globally visible (previously
735
- * attaching state). Ideally, attachGraph() should have already be called making it locally visible. However,
736
- * before visibility state was added, this may not have been the case and getAttachSummary() could be called:
737
- * 1) Before attaching the data store - When a detached container is attached.
738
- * 2) After attaching the data store - When a data store is created and bound in an attached container.
739
- *
740
- * The basic idea is that all local object should become locally visible before they are globally visible.
741
- */
742
- this.attachGraph();
743
-
744
- /**
745
- * This assert cannot be added now due to back-compat. To be uncommented when the following issue is fixed -
746
- * https://github.com/microsoft/FluidFramework/issues/9688.
747
- *
748
- * assert(this.visibilityState === VisibilityState.LocallyVisible,
749
- * "The data store should be locally visible when generating attach summary",
750
- * );
751
- */
752
-
753
- const summaryBuilder = new SummaryTreeBuilder();
754
-
755
- // Craft the .attributes file for each shared object
756
- for (const [contextId, context] of this.contexts) {
757
- if (!(context instanceof LocalChannelContextBase)) {
758
- throw new LoggingError("Should only be called with local channel handles");
759
- }
760
-
761
- if (!this.notBoundedChannelContextSet.has(contextId)) {
762
- let summaryTree: ISummaryTreeWithStats;
763
- if (context.isLoaded) {
764
- const contextSummary = context.getAttachSummary(telemetryContext);
765
- assert(
766
- contextSummary.summary.type === SummaryType.Tree,
767
- 0x180 /* "getAttachSummary should always return a tree" */);
768
- summaryTree = { stats: contextSummary.stats, summary: contextSummary.summary };
769
- } else {
770
- // If this channel is not yet loaded, then there should be no changes in the snapshot from which
771
- // it was created as it is detached container. So just use the previous snapshot.
772
- assert(!!this.dataStoreContext.baseSnapshot,
773
- 0x181 /* "BaseSnapshot should be there as detached container loaded from snapshot" */);
774
- summaryTree = convertSnapshotTreeToSummaryTree(this.dataStoreContext.baseSnapshot.trees[contextId]);
775
- }
776
- summaryBuilder.addWithStats(contextId, summaryTree);
777
- }
778
- }
779
-
780
- return summaryBuilder.getSummaryTree();
781
- }
782
-
783
- public submitMessage(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
784
- this.submit(type, content, localOpMetadata);
785
- }
786
-
787
- public submitSignal(type: string, content: any) {
788
- this.verifyNotClosed();
789
- return this.dataStoreContext.submitSignal(type, content);
790
- }
791
-
792
- /**
793
- * Will return when the data store is attached.
794
- */
795
- public async waitAttached(): Promise<void> {
796
- return this.deferredAttached.promise;
797
- }
798
-
799
- /**
800
- * Attach channel should only be called after the data store has been attached
801
- */
802
- private attachChannel(channel: IChannel): void {
803
- this.verifyNotClosed();
804
- // If this handle is already attached no need to attach again.
805
- if (channel.handle.isAttached) {
806
- return;
807
- }
808
-
809
- channel.handle.attachGraph();
810
-
811
- assert(this.isAttached, 0x182 /* "Data store should be attached to attach the channel." */);
812
- assert(this.visibilityState === VisibilityState.GloballyVisible,
813
- 0x2d0 /* "Data store should be globally visible to attach channels." */);
814
-
815
- const summarizeResult = summarizeChannel(channel, true /* fullTree */, false /* trackState */);
816
- // Attach message needs the summary in ITree format. Convert the ISummaryTree into an ITree.
817
- const snapshot = convertSummaryTreeToITree(summarizeResult.summary);
818
-
819
- const message: IAttachMessage = {
820
- id: channel.id,
821
- snapshot,
822
- type: channel.attributes.type,
823
- };
824
- this.pendingAttach.set(channel.id, message);
825
- this.submit(DataStoreMessageType.Attach, message);
826
-
827
- const context = this.contexts.get(channel.id) as LocalChannelContextBase;
828
- context.makeVisible();
829
- }
830
-
831
- private submitChannelOp(address: string, contents: any, localOpMetadata: unknown) {
832
- const envelope: IEnvelope = { address, contents };
833
- this.submit(DataStoreMessageType.ChannelOp, envelope, localOpMetadata);
834
- }
835
-
836
- private submit(
837
- type: DataStoreMessageType,
838
- content: any,
839
- localOpMetadata: unknown = undefined): void {
840
- this.verifyNotClosed();
841
- this.dataStoreContext.submitMessage(type, content, localOpMetadata);
842
- }
843
-
844
- /**
845
- * For messages of type MessageType.Operation, finds the right channel and asks it to resubmit the message.
846
- * For all other messages, just submit it again.
847
- * This typically happens when we reconnect and there are unacked messages.
848
- * @param content - The content of the original message.
849
- * @param localOpMetadata - The local metadata associated with the original message.
850
- */
851
- public reSubmit(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
852
- this.verifyNotClosed();
853
-
854
- switch (type) {
855
- case DataStoreMessageType.ChannelOp:
856
- {
857
- // For Operations, find the right channel and trigger resubmission on it.
858
- const envelope = content as IEnvelope;
859
- const channelContext = this.contexts.get(envelope.address);
860
- assert(!!channelContext, 0x183 /* "There should be a channel context for the op" */);
861
- channelContext.reSubmit(envelope.contents, localOpMetadata);
862
- break;
863
- }
864
- case DataStoreMessageType.Attach:
865
- // For Attach messages, just submit them again.
866
- this.submit(type, content, localOpMetadata);
867
- break;
868
- default:
869
- unreachableCase(type);
870
- }
871
- }
872
-
873
- /**
874
- * Revert a local op.
875
- * @param content - The content of the original message.
876
- * @param localOpMetadata - The local metadata associated with the original message.
877
- */
878
- public rollback?(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
879
- this.verifyNotClosed();
880
-
881
- switch (type) {
882
- case DataStoreMessageType.ChannelOp:
883
- {
884
- // For Operations, find the right channel and trigger resubmission on it.
885
- const envelope = content as IEnvelope;
886
- const channelContext = this.contexts.get(envelope.address);
887
- assert(!!channelContext, 0x2ed /* "There should be a channel context for the op" */);
888
- channelContext.rollback(envelope.contents, localOpMetadata);
889
- break;
890
- }
891
- default:
892
- throw new LoggingError(`Can't rollback ${type} message`);
893
- }
894
- }
895
-
896
- public async applyStashedOp(content: any): Promise<unknown> {
897
- const envelope = content as IEnvelope;
898
- const channelContext = this.contexts.get(envelope.address);
899
- assert(!!channelContext, 0x184 /* "There should be a channel context for the op" */);
900
- await channelContext.getChannel();
901
- return channelContext.applyStashedOp(envelope.contents);
902
- }
903
-
904
- private setChannelDirty(address: string): void {
905
- this.verifyNotClosed();
906
- this.dataStoreContext.setChannelDirty(address);
907
- }
908
-
909
- private processChannelOp(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown) {
910
- this.verifyNotClosed();
911
-
912
- const envelope = message.contents as IEnvelope;
913
-
914
- const transformed: ISequencedDocumentMessage = {
915
- ...message,
916
- contents: envelope.contents,
917
- };
918
-
919
- const channelContext = this.contexts.get(envelope.address);
920
- assert(!!channelContext, 0x185 /* "Channel not found" */);
921
- channelContext.processOp(transformed, local, localOpMetadata);
922
-
923
- return channelContext;
924
- }
925
-
926
- private attachListener() {
927
- this.setMaxListeners(Number.MAX_SAFE_INTEGER);
928
- this.dataStoreContext.once("attaching", () => {
929
- /**
930
- * back-compat 0.59.1000 - Ideally, attachGraph() should have already been called making the data store
931
- * locally visible. However, before visibility state was added, this may not have been the case and data
932
- * store can move to "attaching" state in 2 scenarios:
933
- * 1) Before attachGraph() is called - When a data store is created and bound in an attached container.
934
- * 2) After attachGraph() is called - When a detached container is attached.
935
- *
936
- * The basic idea is that all local object should become locally visible before they are globally visible.
937
- */
938
- this.attachGraph();
939
-
940
- this._attachState = AttachState.Attaching;
941
-
942
- assert(this.visibilityState === VisibilityState.LocallyVisible,
943
- 0x2d1 /* "Data store should be locally visible before it can become globally visible." */);
944
-
945
- // Mark the data store globally visible and make its child channels visible as well.
946
- this.visibilityState = VisibilityState.GloballyVisible;
947
- this.localChannelContextQueue.forEach((channel) => {
948
- channel.makeVisible();
949
- });
950
- this.localChannelContextQueue.clear();
951
-
952
- // This promise resolution will be moved to attached event once we fix the scheduler.
953
- this.deferredAttached.resolve();
954
- this.emit("attaching");
955
- });
956
- this.dataStoreContext.once("attached", () => {
957
- assert(this.visibilityState === VisibilityState.GloballyVisible,
958
- 0x2d2 /* "Data store should be globally visible when its attached." */);
959
- this._attachState = AttachState.Attached;
960
- this.emit("attached");
961
- });
962
- }
963
-
964
- private verifyNotClosed() {
965
- if (this._disposed) {
966
- throw new LoggingError("Runtime is closed");
967
- }
968
- }
106
+ export class FluidDataStoreRuntime
107
+ extends TypedEventEmitter<IFluidDataStoreRuntimeEvents>
108
+ implements IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext
109
+ {
110
+ /**
111
+ * {@inheritDoc @fluidframework/datastore-definitions#IFluidDataStoreRuntime.entryPoint}
112
+ */
113
+ public readonly entryPoint: IFluidHandle<FluidObject>;
114
+
115
+ public get connected(): boolean {
116
+ return this.dataStoreContext.connected;
117
+ }
118
+
119
+ public get clientId(): string | undefined {
120
+ return this.dataStoreContext.clientId;
121
+ }
122
+
123
+ public get clientDetails(): IClientDetails {
124
+ return this.dataStoreContext.clientDetails;
125
+ }
126
+
127
+ public get isAttached(): boolean {
128
+ return this.attachState !== AttachState.Detached;
129
+ }
130
+
131
+ public get attachState(): AttachState {
132
+ return this._attachState;
133
+ }
134
+
135
+ public get absolutePath(): string {
136
+ return generateHandleContextPath(this.id, this.routeContext);
137
+ }
138
+
139
+ public get routeContext(): IFluidHandleContext {
140
+ return this.dataStoreContext.IFluidHandleContext;
141
+ }
142
+
143
+ public get idCompressor(): IIdCompressor | undefined {
144
+ return this.dataStoreContext.idCompressor;
145
+ }
146
+
147
+ public get IFluidHandleContext() {
148
+ return this;
149
+ }
150
+
151
+ public get rootRoutingContext() {
152
+ return this;
153
+ }
154
+ public get channelsRoutingContext() {
155
+ return this;
156
+ }
157
+ public get objectsRoutingContext() {
158
+ return this;
159
+ }
160
+
161
+ private _disposed = false;
162
+ public get disposed() {
163
+ return this._disposed;
164
+ }
165
+
166
+ private readonly contexts = new Map<string, IChannelContext>();
167
+ private readonly pendingAttach = new Set<string>();
168
+
169
+ private readonly deferredAttached = new Deferred<void>();
170
+ private readonly localChannelContextQueue = new Map<string, LocalChannelContextBase>();
171
+ private readonly notBoundedChannelContextSet = new Set<string>();
172
+ private _attachState: AttachState;
173
+ public visibilityState: VisibilityState;
174
+ // A list of handles that are bound when the data store is not visible. We have to make them visible when the data
175
+ // store becomes visible.
176
+ private readonly pendingHandlesToMakeVisible: Set<IFluidHandle> = new Set();
177
+
178
+ public readonly id: string;
179
+ public readonly options: ILoaderOptions;
180
+ public readonly deltaManager: IDeltaManager<ISequencedDocumentMessage, IDocumentMessage>;
181
+ private readonly quorum: IQuorumClients;
182
+ private readonly audience: IAudience;
183
+ private readonly mc: MonitoringContext;
184
+ public get logger(): ITelemetryLoggerExt {
185
+ return this.mc.logger;
186
+ }
187
+
188
+ /**
189
+ * If the summarizer makes local changes, a telemetry event is logged. This has the potential to be very noisy.
190
+ * So, adding a count of how many telemetry events are logged per data store context. This can be
191
+ * controlled via feature flags.
192
+ */
193
+ private localChangesTelemetryCount: number;
194
+
195
+ /**
196
+ * Invokes the given callback and expects that no ops are submitted
197
+ * until execution finishes. If an op is submitted, an error will be raised.
198
+ *
199
+ * Can be disabled by feature gate `Fluid.ContainerRuntime.DisableOpReentryCheck`
200
+ *
201
+ * @param callback - the callback to be invoked
202
+ */
203
+ public ensureNoDataModelChanges<T>(callback: () => T): T {
204
+ // back-compat ADO:2309
205
+ return this.dataStoreContext.ensureNoDataModelChanges === undefined
206
+ ? callback()
207
+ : this.dataStoreContext.ensureNoDataModelChanges(callback);
208
+ }
209
+
210
+ /**
211
+ * Create an instance of a DataStore runtime.
212
+ *
213
+ * @param dataStoreContext - Context object for the runtime.
214
+ * @param sharedObjectRegistry - The registry of shared objects that this data store will be able to instantiate.
215
+ * @param existing - Pass 'true' if loading this datastore from an existing file; pass 'false' otherwise.
216
+ * @param provideEntryPoint - Function to initialize the entryPoint object for the data store runtime. The
217
+ * handle to this data store runtime will point to the object returned by this function. If this function is not
218
+ * provided, the handle will be left undefined. This is here so we can start making handles a first-class citizen
219
+ * and the primary way of interacting with some Fluid objects, and should be used if possible.
220
+ */
221
+ public constructor(
222
+ private readonly dataStoreContext: IFluidDataStoreContext,
223
+ private readonly sharedObjectRegistry: ISharedObjectRegistry,
224
+ existing: boolean,
225
+ provideEntryPoint: (runtime: IFluidDataStoreRuntime) => Promise<FluidObject>,
226
+ ) {
227
+ super();
228
+
229
+ assert(
230
+ !dataStoreContext.id.includes("/"),
231
+ 0x30e /* Id cannot contain slashes. DataStoreContext should have validated this. */,
232
+ );
233
+
234
+ this.mc = createChildMonitoringContext({
235
+ logger: dataStoreContext.logger,
236
+ namespace: "FluidDataStoreRuntime",
237
+ properties: {
238
+ all: { dataStoreId: uuid() },
239
+ },
240
+ });
241
+
242
+ this.id = dataStoreContext.id;
243
+ this.options = dataStoreContext.options;
244
+ this.deltaManager = dataStoreContext.deltaManager;
245
+ this.quorum = dataStoreContext.getQuorum();
246
+ this.audience = dataStoreContext.getAudience();
247
+
248
+ const tree = dataStoreContext.baseSnapshot;
249
+
250
+ // Must always receive the data store type inside of the attributes
251
+ if (tree?.trees !== undefined) {
252
+ Object.keys(tree.trees).forEach((path) => {
253
+ // Issue #4414
254
+ if (path === "_search") {
255
+ return;
256
+ }
257
+
258
+ let channelContext: RemoteChannelContext | RehydratedLocalChannelContext;
259
+ // If already exists on storage, then create a remote channel. However, if it is case of rehydrating a
260
+ // container from snapshot where we load detached container from a snapshot, isLocalDataStore would be
261
+ // true. In this case create a RehydratedLocalChannelContext.
262
+ if (dataStoreContext.isLocalDataStore) {
263
+ channelContext = new RehydratedLocalChannelContext(
264
+ path,
265
+ this.sharedObjectRegistry,
266
+ this,
267
+ this.dataStoreContext,
268
+ this.dataStoreContext.storage,
269
+ this.logger,
270
+ (content, localOpMetadata) =>
271
+ this.submitChannelOp(path, content, localOpMetadata),
272
+ (address: string) => this.setChannelDirty(address),
273
+ (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
274
+ this.addedGCOutboundReference(srcHandle, outboundHandle),
275
+ tree.trees[path],
276
+ );
277
+ // This is the case of rehydrating a detached container from snapshot. Now due to delay loading of
278
+ // data store, if the data store is loaded after the container is attached, then we missed making
279
+ // the channel visible. So do it now. Otherwise, add it to local channel context queue, so
280
+ // that it can be make it visible later with the data store.
281
+ if (dataStoreContext.attachState !== AttachState.Detached) {
282
+ channelContext.makeVisible();
283
+ } else {
284
+ this.localChannelContextQueue.set(path, channelContext);
285
+ }
286
+ } else {
287
+ channelContext = new RemoteChannelContext(
288
+ this,
289
+ dataStoreContext,
290
+ dataStoreContext.storage,
291
+ (content, localOpMetadata) =>
292
+ this.submitChannelOp(path, content, localOpMetadata),
293
+ (address: string) => this.setChannelDirty(address),
294
+ (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
295
+ this.addedGCOutboundReference(srcHandle, outboundHandle),
296
+ path,
297
+ tree.trees[path],
298
+ this.sharedObjectRegistry,
299
+ undefined /* extraBlobs */,
300
+ this.dataStoreContext.getCreateChildSummarizerNodeFn(path, {
301
+ type: CreateSummarizerNodeSource.FromSummary,
302
+ }),
303
+ );
304
+ }
305
+
306
+ this.contexts.set(path, channelContext);
307
+ });
308
+ }
309
+
310
+ this.entryPoint = new FluidObjectHandle<FluidObject>(
311
+ new LazyPromise(async () => provideEntryPoint(this)),
312
+ "",
313
+ this.objectsRoutingContext,
314
+ );
315
+
316
+ this.attachListener();
317
+ this._attachState = dataStoreContext.attachState;
318
+
319
+ /**
320
+ * If existing flag is false, this is a new data store and is not visible. The existing flag can be true in two
321
+ * conditions:
322
+ *
323
+ * 1. It's a local data store that is created when a detached container is rehydrated. In this case, the data
324
+ * store is locally visible because the snapshot it is loaded from contains locally visible data stores only.
325
+ *
326
+ * 2. It's a remote data store that is created when an attached container is loaded is loaded from snapshot or
327
+ * when an attach op comes in. In both these cases, the data store is already globally visible.
328
+ */
329
+ if (existing) {
330
+ this.visibilityState =
331
+ dataStoreContext.attachState === AttachState.Detached
332
+ ? VisibilityState.LocallyVisible
333
+ : VisibilityState.GloballyVisible;
334
+ } else {
335
+ this.visibilityState = VisibilityState.NotVisible;
336
+ }
337
+
338
+ // If it's existing we know it has been attached.
339
+ if (existing) {
340
+ this.deferredAttached.resolve();
341
+ }
342
+
343
+ // By default, a data store can log maximum 10 local changes telemetry in summarizer.
344
+ this.localChangesTelemetryCount =
345
+ this.mc.config.getNumber("Fluid.Telemetry.LocalChangesTelemetryCount") ?? 10;
346
+ }
347
+
348
+ public dispose(): void {
349
+ if (this._disposed) {
350
+ return;
351
+ }
352
+ this._disposed = true;
353
+
354
+ this.emit("dispose");
355
+ this.removeAllListeners();
356
+ }
357
+
358
+ public async resolveHandle(request: IRequest): Promise<IResponse> {
359
+ return this.request(request);
360
+ }
361
+
362
+ public async request(request: IRequest): Promise<IResponse> {
363
+ try {
364
+ const parser = RequestParser.create(request);
365
+ const id = parser.pathParts[0];
366
+
367
+ if (id === "_channels" || id === "_custom") {
368
+ return await this.request(parser.createSubRequest(1));
369
+ }
370
+
371
+ // Check for a data type reference first
372
+ const context = this.contexts.get(id);
373
+ if (context !== undefined && parser.isLeaf(1)) {
374
+ try {
375
+ const channel = await context.getChannel();
376
+
377
+ return { mimeType: "fluid/object", status: 200, value: channel };
378
+ } catch (error) {
379
+ this.mc.logger.sendErrorEvent(
380
+ { eventName: "GetChannelFailedInRequest" },
381
+ error,
382
+ );
383
+
384
+ return createResponseError(500, `Failed to get Channel: ${error}`, request);
385
+ }
386
+ }
387
+
388
+ // Otherwise defer to an attached request handler
389
+ return create404Response(request);
390
+ } catch (error) {
391
+ return exceptionToResponse(error);
392
+ }
393
+ }
394
+
395
+ public async getChannel(id: string): Promise<IChannel> {
396
+ this.verifyNotClosed();
397
+
398
+ const context = this.contexts.get(id);
399
+ if (context === undefined) {
400
+ throw new LoggingError("Channel does not exist");
401
+ }
402
+
403
+ return context.getChannel();
404
+ }
405
+
406
+ /**
407
+ * Api which allows caller to create the channel first and then add it to the runtime.
408
+ * The channel type should be present in the registry, otherwise the runtime would reject
409
+ * the channel. Also the runtime used to create the channel object should be same to which
410
+ * it is added.
411
+ * @param channel - channel which needs to be added to the runtime.
412
+ */
413
+ public addChannel(channel: IChannel): void {
414
+ const id = channel.id;
415
+ if (id.includes("/")) {
416
+ throw new UsageError(`Id cannot contain slashes: ${id}`);
417
+ }
418
+
419
+ this.verifyNotClosed();
420
+
421
+ assert(!this.contexts.has(id), 0x865 /* addChannel() with existing ID */);
422
+
423
+ const type = channel.attributes.type;
424
+ const factory = this.sharedObjectRegistry.get(channel.attributes.type);
425
+ if (factory === undefined) {
426
+ throw new Error(`Channel Factory ${type} not registered`);
427
+ }
428
+
429
+ this.createChannelContext(channel);
430
+ // Channels (DDS) should not be created in summarizer client.
431
+ this.identifyLocalChangeInSummarizer("DDSCreatedInSummarizer", id, type);
432
+ }
433
+
434
+ public createChannel(id: string = uuid(), type: string): IChannel {
435
+ if (id.includes("/")) {
436
+ throw new UsageError(`Id cannot contain slashes: ${id}`);
437
+ }
438
+
439
+ this.verifyNotClosed();
440
+ assert(!this.contexts.has(id), 0x179 /* "createChannel() with existing ID" */);
441
+
442
+ assert(type !== undefined, 0x209 /* "Factory Type should be defined" */);
443
+ const factory = this.sharedObjectRegistry.get(type);
444
+ if (factory === undefined) {
445
+ throw new Error(`Channel Factory ${type} not registered`);
446
+ }
447
+
448
+ const channel = factory.create(this, id);
449
+ this.createChannelContext(channel);
450
+ // Channels (DDS) should not be created in summarizer client.
451
+ this.identifyLocalChangeInSummarizer("DDSCreatedInSummarizer", id, type);
452
+ return channel;
453
+ }
454
+
455
+ private createChannelContext(channel: IChannel) {
456
+ this.notBoundedChannelContextSet.add(channel.id);
457
+ const context = new LocalChannelContext(
458
+ channel,
459
+ this,
460
+ this.dataStoreContext,
461
+ this.dataStoreContext.storage,
462
+ this.logger,
463
+ (content, localOpMetadata) =>
464
+ this.submitChannelOp(channel.id, content, localOpMetadata),
465
+ (address: string) => this.setChannelDirty(address),
466
+ (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
467
+ this.addedGCOutboundReference(srcHandle, outboundHandle),
468
+ );
469
+ this.contexts.set(channel.id, context);
470
+ }
471
+
472
+ /**
473
+ * Binds a channel with the runtime. If the runtime is attached we will attach the channel right away.
474
+ * If the runtime is not attached we will defer the attach until the runtime attaches.
475
+ * @param channel - channel to be registered.
476
+ */
477
+ public bindChannel(channel: IChannel): void {
478
+ assert(
479
+ this.notBoundedChannelContextSet.has(channel.id),
480
+ 0x17b /* "Channel to be bound should be in not bounded set" */,
481
+ );
482
+ this.notBoundedChannelContextSet.delete(channel.id);
483
+ // If our data store is attached, then attach the channel.
484
+ if (this.isAttached) {
485
+ this.attachChannel(channel);
486
+ return;
487
+ }
488
+
489
+ /**
490
+ * If this channel is already waiting to be made visible, do nothing. This can happen during attachGraph() when
491
+ * a channel's graph is attached. It calls bindToContext on the shared object which will end up back here.
492
+ */
493
+ if (this.pendingHandlesToMakeVisible.has(channel.handle)) {
494
+ return;
495
+ }
496
+
497
+ this.bind(channel.handle);
498
+
499
+ // If our data store is local then add the channel to the queue
500
+ if (!this.localChannelContextQueue.has(channel.id)) {
501
+ this.localChannelContextQueue.set(
502
+ channel.id,
503
+ this.contexts.get(channel.id) as LocalChannelContextBase,
504
+ );
505
+ }
506
+ }
507
+
508
+ /**
509
+ * This function is called when a data store becomes root. It does the following:
510
+ *
511
+ * 1. Marks the data store locally visible in the container.
512
+ *
513
+ * 2. Attaches the graph of all the handles bound to it.
514
+ *
515
+ * 3. Calls into the data store context to mark it visible in the container too. If the container is globally
516
+ * visible, it will mark us globally visible. Otherwise, it will mark us globally visible when it becomes
517
+ * globally visible.
518
+ */
519
+ public makeVisibleAndAttachGraph() {
520
+ if (this.visibilityState !== VisibilityState.NotVisible) {
521
+ return;
522
+ }
523
+ this.visibilityState = VisibilityState.LocallyVisible;
524
+
525
+ this.pendingHandlesToMakeVisible.forEach((handle) => {
526
+ handle.attachGraph();
527
+ });
528
+ this.pendingHandlesToMakeVisible.clear();
529
+ this.dataStoreContext.makeLocallyVisible();
530
+ }
531
+
532
+ /**
533
+ * This function is called when a handle to this data store is added to a visible DDS.
534
+ */
535
+ public attachGraph() {
536
+ this.makeVisibleAndAttachGraph();
537
+ }
538
+
539
+ public bind(handle: IFluidHandle): void {
540
+ // If visible, attach the incoming handle's graph. Else, this will be done when we become visible.
541
+ if (this.visibilityState !== VisibilityState.NotVisible) {
542
+ handle.attachGraph();
543
+ return;
544
+ }
545
+ this.pendingHandlesToMakeVisible.add(handle);
546
+ }
547
+
548
+ public setConnectionState(connected: boolean, clientId?: string) {
549
+ this.verifyNotClosed();
550
+
551
+ for (const [, object] of this.contexts) {
552
+ object.setConnectionState(connected, clientId);
553
+ }
554
+
555
+ raiseConnectedEvent(this.logger, this, connected, clientId);
556
+ }
557
+
558
+ public getQuorum(): IQuorumClients {
559
+ return this.quorum;
560
+ }
561
+
562
+ public getAudience(): IAudience {
563
+ return this.audience;
564
+ }
565
+
566
+ public async uploadBlob(
567
+ blob: ArrayBufferLike,
568
+ signal?: AbortSignal,
569
+ ): Promise<IFluidHandle<ArrayBufferLike>> {
570
+ this.verifyNotClosed();
571
+
572
+ return this.dataStoreContext.uploadBlob(blob, signal);
573
+ }
574
+
575
+ private createRemoteChannelContext(
576
+ attachMessage: IAttachMessage,
577
+ summarizerNodeParams: CreateChildSummarizerNodeParam,
578
+ ) {
579
+ const flatBlobs = new Map<string, ArrayBufferLike>();
580
+ const snapshotTree = buildSnapshotTree(attachMessage.snapshot.entries, flatBlobs);
581
+
582
+ return new RemoteChannelContext(
583
+ this,
584
+ this.dataStoreContext,
585
+ this.dataStoreContext.storage,
586
+ (content, localContentMetadata) =>
587
+ this.submitChannelOp(attachMessage.id, content, localContentMetadata),
588
+ (address: string) => this.setChannelDirty(address),
589
+ (srcHandle: IFluidHandle, outboundHandle: IFluidHandle) =>
590
+ this.addedGCOutboundReference(srcHandle, outboundHandle),
591
+ attachMessage.id,
592
+ snapshotTree,
593
+ this.sharedObjectRegistry,
594
+ flatBlobs,
595
+ this.dataStoreContext.getCreateChildSummarizerNodeFn(
596
+ attachMessage.id,
597
+ summarizerNodeParams,
598
+ ),
599
+ attachMessage.type,
600
+ );
601
+ }
602
+
603
+ public process(message: ISequencedDocumentMessage, local: boolean, localOpMetadata: unknown) {
604
+ this.verifyNotClosed();
605
+
606
+ try {
607
+ // catches as data processing error whether or not they come from async pending queues
608
+ switch (message.type) {
609
+ case DataStoreMessageType.Attach: {
610
+ const attachMessage = message.contents as IAttachMessage;
611
+ const id = attachMessage.id;
612
+
613
+ // If a non-local operation then go and create the object
614
+ // Otherwise mark it as officially attached.
615
+ if (local) {
616
+ assert(
617
+ this.pendingAttach.delete(id),
618
+ 0x17c /* "Unexpected attach (local) channel OP" */,
619
+ );
620
+ } else {
621
+ assert(!this.contexts.has(id), 0x17d /* "Unexpected attach channel OP" */);
622
+
623
+ const summarizerNodeParams = {
624
+ type: CreateSummarizerNodeSource.FromAttach,
625
+ sequenceNumber: message.sequenceNumber,
626
+ snapshot: attachMessage.snapshot,
627
+ };
628
+
629
+ const remoteChannelContext = this.createRemoteChannelContext(
630
+ attachMessage,
631
+ summarizerNodeParams,
632
+ );
633
+ this.contexts.set(id, remoteChannelContext);
634
+ }
635
+ break;
636
+ }
637
+
638
+ case DataStoreMessageType.ChannelOp:
639
+ this.processChannelOp(message, local, localOpMetadata);
640
+ break;
641
+ default:
642
+ }
643
+
644
+ this.emit("op", message);
645
+ } catch (error) {
646
+ throw DataProcessingError.wrapIfUnrecognized(
647
+ error,
648
+ "fluidDataStoreRuntimeFailedToProcessMessage",
649
+ message,
650
+ );
651
+ }
652
+ }
653
+
654
+ public processSignal(message: IInboundSignalMessage, local: boolean) {
655
+ this.emit("signal", message, local);
656
+ }
657
+
658
+ private isChannelAttached(id: string): boolean {
659
+ return (
660
+ // Added in createChannel
661
+ // Removed when bindChannel is called
662
+ !this.notBoundedChannelContextSet.has(id) &&
663
+ // Added in bindChannel only if this is not attached yet
664
+ // Removed when this is attached by calling attachGraph
665
+ !this.localChannelContextQueue.has(id) &&
666
+ // Added in attachChannel called by bindChannel
667
+ // Removed when attach op is broadcast
668
+ !this.pendingAttach.has(id)
669
+ );
670
+ }
671
+
672
+ /**
673
+ * Returns the outbound routes of this channel. Currently, all contexts in this channel are considered
674
+ * referenced and are hence outbound. This will change when we have root and non-root channel contexts.
675
+ * The only root contexts will be considered as referenced.
676
+ */
677
+ private getOutboundRoutes(): string[] {
678
+ const outboundRoutes: string[] = [];
679
+ for (const [contextId] of this.contexts) {
680
+ outboundRoutes.push(`${this.absolutePath}/${contextId}`);
681
+ }
682
+ return outboundRoutes;
683
+ }
684
+
685
+ /**
686
+ * Updates the GC nodes of this channel. It does the following:
687
+ * - Adds a back route to self to all its child GC nodes.
688
+ * - Adds a node for this channel.
689
+ * @param builder - The builder that contains the GC nodes for this channel's children.
690
+ */
691
+ private updateGCNodes(builder: GCDataBuilder) {
692
+ // Add a back route to self in each child's GC nodes. If any child is referenced, then its parent should
693
+ // be considered referenced as well.
694
+ builder.addRouteToAllNodes(this.absolutePath);
695
+
696
+ // Get the outbound routes and add a GC node for this channel.
697
+ builder.addNode("/", this.getOutboundRoutes());
698
+ }
699
+
700
+ /**
701
+ * Generates data used for garbage collection. This includes a list of GC nodes that represent this channel
702
+ * including any of its child channel contexts. Each node has a set of outbound routes to other GC nodes in the
703
+ * document. It does the following:
704
+ *
705
+ * 1. Calls into each child context to get its GC data.
706
+ *
707
+ * 2. Prefixes the child context's id to the GC nodes in the child's GC data. This makes sure that the node can be
708
+ * identified as belonging to the child.
709
+ *
710
+ * 3. Adds a GC node for this channel to the nodes received from the children. All these nodes together represent
711
+ * the GC data of this channel.
712
+ *
713
+ * @param fullGC - true to bypass optimizations and force full generation of GC data.
714
+ */
715
+ public async getGCData(fullGC: boolean = false): Promise<IGarbageCollectionData> {
716
+ const builder = new GCDataBuilder();
717
+ // Iterate over each channel context and get their GC data.
718
+ await Promise.all(
719
+ Array.from(this.contexts)
720
+ .filter(([contextId, _]) => {
721
+ // Get GC data only for attached contexts. Detached contexts are not connected in the GC reference
722
+ // graph so any references they might have won't be connected as well.
723
+ return this.isChannelAttached(contextId);
724
+ })
725
+ .map(async ([contextId, context]) => {
726
+ const contextGCData = await context.getGCData(fullGC);
727
+ // Prefix the child's id to the ids of its GC nodes so they can be identified as belonging to the child.
728
+ // This also gradually builds the id of each node to be a path from the root.
729
+ builder.prefixAndAddNodes(contextId, contextGCData.gcNodes);
730
+ }),
731
+ );
732
+
733
+ this.updateGCNodes(builder);
734
+ return builder.getGCData();
735
+ }
736
+
737
+ /**
738
+ * After GC has run, called to notify this channel of routes that are used in it. It calls the child contexts to
739
+ * update their used routes.
740
+ * @param usedRoutes - The routes that are used in all contexts in this channel.
741
+ */
742
+ public updateUsedRoutes(usedRoutes: string[]) {
743
+ // Get a map of channel ids to routes used in it.
744
+ const usedContextRoutes = unpackChildNodesUsedRoutes(usedRoutes);
745
+
746
+ // Verify that the used routes are correct.
747
+ for (const [id] of usedContextRoutes) {
748
+ assert(
749
+ this.contexts.has(id),
750
+ 0x17e /* "Used route does not belong to any known context" */,
751
+ );
752
+ }
753
+
754
+ // Update the used routes in each context. Used routes is empty for unused context.
755
+ for (const [contextId, context] of this.contexts) {
756
+ context.updateUsedRoutes(usedContextRoutes.get(contextId) ?? []);
757
+ }
758
+ }
759
+
760
+ /**
761
+ * Called when a new outbound reference is added to another node. This is used by garbage collection to identify
762
+ * all references added in the system.
763
+ * @param srcHandle - The handle of the node that added the reference.
764
+ * @param outboundHandle - The handle of the outbound node that is referenced.
765
+ */
766
+ private addedGCOutboundReference(srcHandle: IFluidHandle, outboundHandle: IFluidHandle) {
767
+ this.dataStoreContext.addedGCOutboundReference?.(srcHandle, outboundHandle);
768
+ }
769
+
770
+ /**
771
+ * Returns a summary at the current sequence number.
772
+ * @param fullTree - true to bypass optimizations and force a full summary tree
773
+ * @param trackState - This tells whether we should track state from this summary.
774
+ * @param telemetryContext - summary data passed through the layers for telemetry purposes
775
+ */
776
+ public async summarize(
777
+ fullTree: boolean = false,
778
+ trackState: boolean = true,
779
+ telemetryContext?: ITelemetryContext,
780
+ ): Promise<ISummaryTreeWithStats> {
781
+ const summaryBuilder = new SummaryTreeBuilder();
782
+
783
+ // Iterate over each data store and ask it to summarize
784
+ await Promise.all(
785
+ Array.from(this.contexts)
786
+ .filter(([contextId, _]) => {
787
+ const isAttached = this.isChannelAttached(contextId);
788
+ // We are not expecting local dds! Summary may not capture local state.
789
+ assert(
790
+ isAttached,
791
+ 0x17f /* "Not expecting detached channels during summarize" */,
792
+ );
793
+ // If the object is registered - and we have received the sequenced op creating the object
794
+ // (i.e. it has a base mapping) - then we go ahead and summarize
795
+ return isAttached;
796
+ })
797
+ .map(async ([contextId, context]) => {
798
+ const contextSummary = await context.summarize(
799
+ fullTree,
800
+ trackState,
801
+ telemetryContext,
802
+ );
803
+ summaryBuilder.addWithStats(contextId, contextSummary);
804
+ }),
805
+ );
806
+
807
+ return summaryBuilder.getSummaryTree();
808
+ }
809
+
810
+ public getAttachSummary(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats {
811
+ /**
812
+ * back-compat 0.59.1000 - getAttachSummary() is called when making a data store globally visible (previously
813
+ * attaching state). Ideally, attachGraph() should have already be called making it locally visible. However,
814
+ * before visibility state was added, this may not have been the case and getAttachSummary() could be called:
815
+ *
816
+ * 1. Before attaching the data store - When a detached container is attached.
817
+ *
818
+ * 2. After attaching the data store - When a data store is created and bound in an attached container.
819
+ *
820
+ * The basic idea is that all local object should become locally visible before they are globally visible.
821
+ */
822
+ this.attachGraph();
823
+
824
+ // This assert cannot be added now due to back-compat. To be uncommented when the following issue is fixed -
825
+ // https://github.com/microsoft/FluidFramework/issues/9688.
826
+ //
827
+ // assert(this.visibilityState === VisibilityState.LocallyVisible,
828
+ // "The data store should be locally visible when generating attach summary",
829
+ // );
830
+
831
+ const summaryBuilder = new SummaryTreeBuilder();
832
+
833
+ // Craft the .attributes file for each shared object
834
+ for (const [contextId, context] of this.contexts) {
835
+ if (!(context instanceof LocalChannelContextBase)) {
836
+ throw new LoggingError("Should only be called with local channel handles");
837
+ }
838
+
839
+ if (!this.notBoundedChannelContextSet.has(contextId)) {
840
+ let summaryTree: ISummaryTreeWithStats;
841
+ if (context.isLoaded) {
842
+ const contextSummary = context.getAttachSummary(telemetryContext);
843
+ assert(
844
+ contextSummary.summary.type === SummaryType.Tree,
845
+ 0x180 /* "getAttachSummary should always return a tree" */,
846
+ );
847
+ summaryTree = { stats: contextSummary.stats, summary: contextSummary.summary };
848
+ } else {
849
+ // If this channel is not yet loaded, then there should be no changes in the snapshot from which
850
+ // it was created as it is detached container. So just use the previous snapshot.
851
+ assert(
852
+ !!this.dataStoreContext.baseSnapshot,
853
+ 0x181 /* "BaseSnapshot should be there as detached container loaded from snapshot" */,
854
+ );
855
+ summaryTree = convertSnapshotTreeToSummaryTree(
856
+ this.dataStoreContext.baseSnapshot.trees[contextId],
857
+ );
858
+ }
859
+ summaryBuilder.addWithStats(contextId, summaryTree);
860
+ }
861
+ }
862
+
863
+ return summaryBuilder.getSummaryTree();
864
+ }
865
+
866
+ public submitMessage(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
867
+ this.submit(type, content, localOpMetadata);
868
+ }
869
+
870
+ /**
871
+ * Submits the signal to be sent to other clients.
872
+ * @param type - Type of the signal.
873
+ * @param content - Content of the signal.
874
+ * @param targetClientId - When specified, the signal is only sent to the provided client id.
875
+ */
876
+ public submitSignal(type: string, content: any, targetClientId?: string) {
877
+ this.verifyNotClosed();
878
+ return this.dataStoreContext.submitSignal(type, content, targetClientId);
879
+ }
880
+
881
+ /**
882
+ * Will return when the data store is attached.
883
+ */
884
+ public async waitAttached(): Promise<void> {
885
+ return this.deferredAttached.promise;
886
+ }
887
+
888
+ /**
889
+ * Attach channel should only be called after the data store has been attached
890
+ */
891
+ private attachChannel(channel: IChannel): void {
892
+ this.verifyNotClosed();
893
+ // If this handle is already attached no need to attach again.
894
+ if (channel.handle.isAttached) {
895
+ return;
896
+ }
897
+
898
+ channel.handle.attachGraph();
899
+
900
+ assert(this.isAttached, 0x182 /* "Data store should be attached to attach the channel." */);
901
+ assert(
902
+ this.visibilityState === VisibilityState.GloballyVisible,
903
+ 0x2d0 /* "Data store should be globally visible to attach channels." */,
904
+ );
905
+
906
+ const summarizeResult = summarizeChannel(
907
+ channel,
908
+ true /* fullTree */,
909
+ false /* trackState */,
910
+ );
911
+ // Attach message needs the summary in ITree format. Convert the ISummaryTree into an ITree.
912
+ const snapshot = convertSummaryTreeToITree(summarizeResult.summary);
913
+
914
+ const message: IAttachMessage = {
915
+ id: channel.id,
916
+ snapshot,
917
+ type: channel.attributes.type,
918
+ };
919
+ this.pendingAttach.add(channel.id);
920
+ this.submit(DataStoreMessageType.Attach, message);
921
+
922
+ const context = this.contexts.get(channel.id) as LocalChannelContextBase;
923
+ context.makeVisible();
924
+ }
925
+
926
+ private submitChannelOp(address: string, contents: any, localOpMetadata: unknown) {
927
+ const envelope: IEnvelope = { address, contents };
928
+ this.submit(DataStoreMessageType.ChannelOp, envelope, localOpMetadata);
929
+ }
930
+
931
+ private submit(
932
+ type: DataStoreMessageType,
933
+ content: any,
934
+ localOpMetadata: unknown = undefined,
935
+ ): void {
936
+ this.verifyNotClosed();
937
+ this.dataStoreContext.submitMessage(type, content, localOpMetadata);
938
+ }
939
+
940
+ /**
941
+ * For messages of type MessageType.Operation, finds the right channel and asks it to resubmit the message.
942
+ * For all other messages, just submit it again.
943
+ * This typically happens when we reconnect and there are unacked messages.
944
+ * @param content - The content of the original message.
945
+ * @param localOpMetadata - The local metadata associated with the original message.
946
+ */
947
+ public reSubmit(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
948
+ this.verifyNotClosed();
949
+
950
+ switch (type) {
951
+ case DataStoreMessageType.ChannelOp: {
952
+ // For Operations, find the right channel and trigger resubmission on it.
953
+ const envelope = content as IEnvelope;
954
+ const channelContext = this.contexts.get(envelope.address);
955
+ assert(
956
+ !!channelContext,
957
+ 0x183 /* "There should be a channel context for the op" */,
958
+ );
959
+ channelContext.reSubmit(envelope.contents, localOpMetadata);
960
+ break;
961
+ }
962
+ case DataStoreMessageType.Attach:
963
+ // For Attach messages, just submit them again.
964
+ this.submit(type, content, localOpMetadata);
965
+ break;
966
+ default:
967
+ unreachableCase(type);
968
+ }
969
+ }
970
+
971
+ /**
972
+ * Revert a local op.
973
+ * @param content - The content of the original message.
974
+ * @param localOpMetadata - The local metadata associated with the original message.
975
+ */
976
+ public rollback?(type: DataStoreMessageType, content: any, localOpMetadata: unknown) {
977
+ this.verifyNotClosed();
978
+
979
+ switch (type) {
980
+ case DataStoreMessageType.ChannelOp: {
981
+ // For Operations, find the right channel and trigger resubmission on it.
982
+ const envelope = content as IEnvelope;
983
+ const channelContext = this.contexts.get(envelope.address);
984
+ assert(
985
+ !!channelContext,
986
+ 0x2ed /* "There should be a channel context for the op" */,
987
+ );
988
+ channelContext.rollback(envelope.contents, localOpMetadata);
989
+ break;
990
+ }
991
+ default:
992
+ throw new LoggingError(`Can't rollback ${type} message`);
993
+ }
994
+ }
995
+
996
+ public async applyStashedOp(content: any): Promise<unknown> {
997
+ const type = content?.type as DataStoreMessageType;
998
+ switch (type) {
999
+ case DataStoreMessageType.Attach: {
1000
+ const attachMessage = content.content as IAttachMessage;
1001
+ // local means this node will throw if summarized; this is fine because only interactive clients will have stashed ops
1002
+ const summarizerNodeParams: CreateChildSummarizerNodeParam = {
1003
+ type: CreateSummarizerNodeSource.Local,
1004
+ };
1005
+ const context = this.createRemoteChannelContext(
1006
+ attachMessage,
1007
+ summarizerNodeParams,
1008
+ );
1009
+ this.pendingAttach.add(attachMessage.id);
1010
+ this.contexts.set(attachMessage.id, context);
1011
+ return;
1012
+ }
1013
+ case DataStoreMessageType.ChannelOp: {
1014
+ const envelope = content.content as IEnvelope;
1015
+ const channelContext = this.contexts.get(envelope.address);
1016
+ assert(
1017
+ !!channelContext,
1018
+ 0x184 /* "There should be a channel context for the op" */,
1019
+ );
1020
+ await channelContext.getChannel();
1021
+ return channelContext.applyStashedOp(envelope.contents);
1022
+ }
1023
+ default:
1024
+ unreachableCase(type);
1025
+ }
1026
+ }
1027
+
1028
+ private setChannelDirty(address: string): void {
1029
+ this.verifyNotClosed();
1030
+ this.dataStoreContext.setChannelDirty(address);
1031
+ }
1032
+
1033
+ private processChannelOp(
1034
+ message: ISequencedDocumentMessage,
1035
+ local: boolean,
1036
+ localOpMetadata: unknown,
1037
+ ) {
1038
+ this.verifyNotClosed();
1039
+
1040
+ const envelope = message.contents as IEnvelope;
1041
+
1042
+ const transformed: ISequencedDocumentMessage = {
1043
+ ...message,
1044
+ contents: envelope.contents,
1045
+ };
1046
+
1047
+ const channelContext = this.contexts.get(envelope.address);
1048
+ assert(!!channelContext, 0x185 /* "Channel not found" */);
1049
+ channelContext.processOp(transformed, local, localOpMetadata);
1050
+
1051
+ return channelContext;
1052
+ }
1053
+
1054
+ private attachListener() {
1055
+ this.setMaxListeners(Number.MAX_SAFE_INTEGER);
1056
+ this.dataStoreContext.once("attaching", () => {
1057
+ /**
1058
+ * back-compat 0.59.1000 - Ideally, attachGraph() should have already been called making the data store
1059
+ * locally visible. However, before visibility state was added, this may not have been the case and data
1060
+ * store can move to "attaching" state in 2 scenarios:
1061
+ * 1) Before attachGraph() is called - When a data store is created and bound in an attached container.
1062
+ * 2) After attachGraph() is called - When a detached container is attached.
1063
+ *
1064
+ * The basic idea is that all local object should become locally visible before they are globally visible.
1065
+ */
1066
+ this.attachGraph();
1067
+
1068
+ this._attachState = AttachState.Attaching;
1069
+
1070
+ assert(
1071
+ this.visibilityState === VisibilityState.LocallyVisible,
1072
+ 0x2d1 /* "Data store should be locally visible before it can become globally visible." */,
1073
+ );
1074
+
1075
+ // Mark the data store globally visible and make its child channels visible as well.
1076
+ this.visibilityState = VisibilityState.GloballyVisible;
1077
+ this.localChannelContextQueue.forEach((channel) => {
1078
+ channel.makeVisible();
1079
+ });
1080
+ this.localChannelContextQueue.clear();
1081
+
1082
+ // This promise resolution will be moved to attached event once we fix the scheduler.
1083
+ this.deferredAttached.resolve();
1084
+ this.emit("attaching");
1085
+ });
1086
+ this.dataStoreContext.once("attached", () => {
1087
+ assert(
1088
+ this.visibilityState === VisibilityState.GloballyVisible,
1089
+ 0x2d2 /* "Data store should be globally visible when its attached." */,
1090
+ );
1091
+ this._attachState = AttachState.Attached;
1092
+ this.emit("attached");
1093
+ });
1094
+ }
1095
+
1096
+ private verifyNotClosed() {
1097
+ if (this._disposed) {
1098
+ throw new LoggingError("Runtime is closed");
1099
+ }
1100
+ }
1101
+
1102
+ /**
1103
+ * Summarizer client should not have local changes. These changes can become part of the summary and can break
1104
+ * eventual consistency. For example, the next summary (say at ref seq# 100) may contain these changes whereas
1105
+ * other clients that are up-to-date till seq# 100 may not have them yet.
1106
+ */
1107
+ private identifyLocalChangeInSummarizer(
1108
+ eventName: string,
1109
+ channelId: string,
1110
+ channelType: string,
1111
+ ) {
1112
+ if (this.clientDetails.type !== "summarizer" || this.localChangesTelemetryCount <= 0) {
1113
+ return;
1114
+ }
1115
+
1116
+ // Log a telemetry if there are local changes in the summarizer. This will give us data on how often
1117
+ // this is happening and which data stores do this. The eventual goal is to disallow local changes
1118
+ // in the summarizer and the data will help us plan this.
1119
+ this.mc.logger.sendTelemetryEvent({
1120
+ eventName,
1121
+ ...tagCodeArtifacts({
1122
+ channelType,
1123
+ channelId,
1124
+ fluidDataStoreId: this.id,
1125
+ fluidDataStorePackagePath: this.dataStoreContext.packagePath.join("/"),
1126
+ }),
1127
+ stack: generateStack(),
1128
+ });
1129
+ this.localChangesTelemetryCount--;
1130
+ }
969
1131
  }
970
1132
 
971
1133
  /**
@@ -973,59 +1135,72 @@ IFluidDataStoreChannel, IFluidDataStoreRuntime, IFluidHandleContext {
973
1135
  * Request handler is only called when data store can't resolve request, i.e. for custom requests.
974
1136
  * @param Base - base class, inherits from FluidDataStoreRuntime
975
1137
  * @param requestHandler - request handler to mix in
1138
+ * @internal
976
1139
  */
977
1140
  export const mixinRequestHandler = (
978
- requestHandler: (request: IRequest, runtime: FluidDataStoreRuntime) => Promise<IResponse>,
979
- Base: typeof FluidDataStoreRuntime = FluidDataStoreRuntime,
980
- ) => class RuntimeWithRequestHandler extends Base {
981
- public async request(request: IRequest) {
982
- const response = await super.request(request);
983
- if (response.status === 404) {
984
- return requestHandler(request, this);
985
- }
986
- return response;
987
- }
988
- } as typeof FluidDataStoreRuntime;
1141
+ requestHandler: (request: IRequest, runtime: FluidDataStoreRuntime) => Promise<IResponse>,
1142
+ Base: typeof FluidDataStoreRuntime = FluidDataStoreRuntime,
1143
+ ) =>
1144
+ class RuntimeWithRequestHandler extends Base {
1145
+ public async request(request: IRequest) {
1146
+ const response = await super.request(request);
1147
+ if (response.status === 404) {
1148
+ return requestHandler(request, this);
1149
+ }
1150
+ return response;
1151
+ }
1152
+ } as typeof FluidDataStoreRuntime;
989
1153
 
990
1154
  /**
991
1155
  * Mixin class that adds await for DataObject to finish initialization before we proceed to summary.
992
1156
  * @param handler - handler that returns info about blob to be added to summary.
993
1157
  * Or undefined not to add anything to summary.
994
1158
  * @param Base - base class, inherits from FluidDataStoreRuntime
1159
+ * @alpha
995
1160
  */
996
1161
  export const mixinSummaryHandler = (
997
- handler: (runtime: FluidDataStoreRuntime) => Promise<{ path: string[]; content: string; } | undefined >,
998
- Base: typeof FluidDataStoreRuntime = FluidDataStoreRuntime,
999
- ) => class RuntimeWithSummarizerHandler extends Base {
1000
- private addBlob(summary: ISummaryTreeWithStats, path: string[], content: string) {
1001
- const firstName = path.shift();
1002
- if (firstName === undefined) {
1003
- throw new LoggingError("Path can't be empty");
1004
- }
1005
-
1006
- let blob: ISummaryTree | ISummaryBlob = {
1007
- type: SummaryType.Blob,
1008
- content,
1009
- };
1010
- summary.stats.blobNodeCount++;
1011
- summary.stats.totalBlobSize += content.length;
1012
-
1013
- for (const name of path.reverse()) {
1014
- blob = {
1015
- type: SummaryType.Tree,
1016
- tree: { [name]: blob },
1017
- };
1018
- summary.stats.treeNodeCount++;
1019
- }
1020
- summary.summary.tree[firstName] = blob;
1021
- }
1022
-
1023
- async summarize(...args: any[]) {
1024
- const summary = await super.summarize(...args);
1025
- const content = await handler(this);
1026
- if (content !== undefined) {
1027
- this.addBlob(summary, content.path, content.content);
1028
- }
1029
- return summary;
1030
- }
1031
- } as typeof FluidDataStoreRuntime;
1162
+ handler: (
1163
+ runtime: FluidDataStoreRuntime,
1164
+ ) => Promise<{ path: string[]; content: string } | undefined>,
1165
+ Base: typeof FluidDataStoreRuntime = FluidDataStoreRuntime,
1166
+ ) =>
1167
+ class RuntimeWithSummarizerHandler extends Base {
1168
+ private addBlob(summary: ISummaryTreeWithStats, path: string[], content: string) {
1169
+ const firstName = path.shift();
1170
+ if (firstName === undefined) {
1171
+ throw new LoggingError("Path can't be empty");
1172
+ }
1173
+
1174
+ let blob: ISummaryTree | ISummaryBlob = {
1175
+ type: SummaryType.Blob,
1176
+ content,
1177
+ };
1178
+ summary.stats.blobNodeCount++;
1179
+ summary.stats.totalBlobSize += content.length;
1180
+
1181
+ for (const name of path.reverse()) {
1182
+ blob = {
1183
+ type: SummaryType.Tree,
1184
+ tree: { [name]: blob },
1185
+ };
1186
+ summary.stats.treeNodeCount++;
1187
+ }
1188
+ summary.summary.tree[firstName] = blob;
1189
+ }
1190
+
1191
+ async summarize(...args: any[]) {
1192
+ const summary = await super.summarize(...args);
1193
+
1194
+ try {
1195
+ const content = await handler(this);
1196
+ if (content !== undefined) {
1197
+ this.addBlob(summary, content.path, content.content);
1198
+ }
1199
+ } catch (e) {
1200
+ // Any error coming from app-provided handler should be marked as DataProcessingError
1201
+ throw DataProcessingError.wrapIfUnrecognized(e, "mixinSummaryHandler");
1202
+ }
1203
+
1204
+ return summary;
1205
+ }
1206
+ } as typeof FluidDataStoreRuntime;