@fluidframework/container-runtime 0.54.2 → 0.56.0-49831

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (94) hide show
  1. package/.eslintrc.js +1 -1
  2. package/dist/blobManager.js +1 -1
  3. package/dist/blobManager.js.map +1 -1
  4. package/dist/containerRuntime.d.ts +20 -14
  5. package/dist/containerRuntime.d.ts.map +1 -1
  6. package/dist/containerRuntime.js +51 -58
  7. package/dist/containerRuntime.js.map +1 -1
  8. package/dist/dataStoreContext.d.ts +57 -33
  9. package/dist/dataStoreContext.d.ts.map +1 -1
  10. package/dist/dataStoreContext.js +44 -54
  11. package/dist/dataStoreContext.js.map +1 -1
  12. package/dist/dataStores.d.ts +12 -2
  13. package/dist/dataStores.d.ts.map +1 -1
  14. package/dist/dataStores.js +106 -32
  15. package/dist/dataStores.js.map +1 -1
  16. package/dist/garbageCollection.d.ts +49 -14
  17. package/dist/garbageCollection.d.ts.map +1 -1
  18. package/dist/garbageCollection.js +122 -26
  19. package/dist/garbageCollection.js.map +1 -1
  20. package/dist/index.d.ts +7 -7
  21. package/dist/index.d.ts.map +1 -1
  22. package/dist/index.js +12 -17
  23. package/dist/index.js.map +1 -1
  24. package/dist/packageVersion.d.ts +1 -1
  25. package/dist/packageVersion.d.ts.map +1 -1
  26. package/dist/packageVersion.js +1 -1
  27. package/dist/packageVersion.js.map +1 -1
  28. package/dist/runningSummarizer.d.ts.map +1 -1
  29. package/dist/runningSummarizer.js +2 -9
  30. package/dist/runningSummarizer.js.map +1 -1
  31. package/dist/summaryFormat.d.ts +2 -0
  32. package/dist/summaryFormat.d.ts.map +1 -1
  33. package/dist/summaryFormat.js +2 -4
  34. package/dist/summaryFormat.js.map +1 -1
  35. package/dist/summaryGenerator.d.ts +0 -5
  36. package/dist/summaryGenerator.d.ts.map +1 -1
  37. package/dist/summaryGenerator.js.map +1 -1
  38. package/dist/summaryManager.d.ts +1 -0
  39. package/dist/summaryManager.d.ts.map +1 -1
  40. package/dist/summaryManager.js +7 -2
  41. package/dist/summaryManager.js.map +1 -1
  42. package/garbageCollection.md +33 -0
  43. package/lib/blobManager.js +1 -1
  44. package/lib/blobManager.js.map +1 -1
  45. package/lib/containerRuntime.d.ts +20 -14
  46. package/lib/containerRuntime.d.ts.map +1 -1
  47. package/lib/containerRuntime.js +51 -58
  48. package/lib/containerRuntime.js.map +1 -1
  49. package/lib/dataStoreContext.d.ts +57 -33
  50. package/lib/dataStoreContext.d.ts.map +1 -1
  51. package/lib/dataStoreContext.js +42 -52
  52. package/lib/dataStoreContext.js.map +1 -1
  53. package/lib/dataStores.d.ts +12 -2
  54. package/lib/dataStores.d.ts.map +1 -1
  55. package/lib/dataStores.js +107 -33
  56. package/lib/dataStores.js.map +1 -1
  57. package/lib/garbageCollection.d.ts +49 -14
  58. package/lib/garbageCollection.d.ts.map +1 -1
  59. package/lib/garbageCollection.js +122 -26
  60. package/lib/garbageCollection.js.map +1 -1
  61. package/lib/index.d.ts +7 -7
  62. package/lib/index.d.ts.map +1 -1
  63. package/lib/index.js +6 -7
  64. package/lib/index.js.map +1 -1
  65. package/lib/packageVersion.d.ts +1 -1
  66. package/lib/packageVersion.d.ts.map +1 -1
  67. package/lib/packageVersion.js +1 -1
  68. package/lib/packageVersion.js.map +1 -1
  69. package/lib/runningSummarizer.d.ts.map +1 -1
  70. package/lib/runningSummarizer.js +2 -9
  71. package/lib/runningSummarizer.js.map +1 -1
  72. package/lib/summaryFormat.d.ts +2 -0
  73. package/lib/summaryFormat.d.ts.map +1 -1
  74. package/lib/summaryFormat.js +2 -4
  75. package/lib/summaryFormat.js.map +1 -1
  76. package/lib/summaryGenerator.d.ts +0 -5
  77. package/lib/summaryGenerator.d.ts.map +1 -1
  78. package/lib/summaryGenerator.js.map +1 -1
  79. package/lib/summaryManager.d.ts +1 -0
  80. package/lib/summaryManager.d.ts.map +1 -1
  81. package/lib/summaryManager.js +7 -2
  82. package/lib/summaryManager.js.map +1 -1
  83. package/package.json +24 -23
  84. package/src/blobManager.ts +1 -1
  85. package/src/containerRuntime.ts +69 -65
  86. package/src/dataStoreContext.ts +105 -129
  87. package/src/dataStores.ts +118 -68
  88. package/src/garbageCollection.ts +156 -31
  89. package/src/index.ts +52 -6
  90. package/src/packageVersion.ts +1 -1
  91. package/src/runningSummarizer.ts +2 -7
  92. package/src/summaryFormat.ts +4 -4
  93. package/src/summaryGenerator.ts +0 -5
  94. package/src/summaryManager.ts +9 -3
@@ -5,6 +5,8 @@
5
5
 
6
6
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
8
+ import { ICriticalContainerError } from "@fluidframework/container-definitions";
9
+ import { ClientSessionExpiredError } from "@fluidframework/container-utils";
8
10
  import {
9
11
  cloneGCData,
10
12
  concatGarbageCollectionStates,
@@ -18,7 +20,7 @@ import {
18
20
  gcBlobKey,
19
21
  IGarbageCollectionData,
20
22
  IGarbageCollectionState,
21
- IGarbageCollectionSummaryDetails,
23
+ IGarbageCollectionDetailsBase,
22
24
  ISummaryTreeWithStats,
23
25
  } from "@fluidframework/runtime-definitions";
24
26
  import {
@@ -52,12 +54,14 @@ export const gcTreeKey = "gc";
52
54
  // They prefix for GC blobs in the GC tree in summary.
53
55
  export const gcBlobPrefix = "__gc";
54
56
 
55
- // Local storage key to turn GC on / off.
57
+ // Feature gate key to turn GC on / off.
56
58
  const runGCKey = "Fluid.GarbageCollection.RunGC";
57
- // Local storage key to turn GC test mode on / off.
59
+ // Feature gate key to turn GC test mode on / off.
58
60
  const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
59
- // Local storage key to turn GC sweep on / off.
61
+ // Feature gate key to turn GC sweep on / off.
60
62
  const runSweepKey = "Fluid.GarbageCollection.RunSweep";
63
+ // Feature gate key to write GC data at the root of the summary tree.
64
+ const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
61
65
 
62
66
  const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
63
67
 
@@ -77,6 +81,8 @@ export interface IGCStats {
77
81
 
78
82
  /** Defines the APIs for the runtime object to be passed to the garbage collector. */
79
83
  export interface IGarbageCollectionRuntime {
84
+ /** Before GC runs, called to notify the runtime to update any pending GC state. */
85
+ updateStateBeforeGC(): Promise<void>;
80
86
  /** Returns the garbage collection data of the runtime. */
81
87
  getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
82
88
  /** After GC has run, called to notify the runtime of routes that are used in it. */
@@ -87,14 +93,16 @@ export interface IGarbageCollectionRuntime {
87
93
  export interface IGarbageCollector {
88
94
  /** Tells whether GC should run or not. */
89
95
  readonly shouldRunGC: boolean;
96
+ /** The time in ms to expire a session for a client for gc. */
97
+ readonly sessionExpiryTimeoutMs: number | undefined;
90
98
  /**
91
99
  * This tracks two things:
92
100
  * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
93
101
  * 2. If GC is enabled, the version of GC used to generate the GC data written in a summary.
94
102
  */
95
103
  readonly gcSummaryFeatureVersion: number;
96
- /** Tells whether the GC version has changed compared to the version in the latest summary. */
97
- readonly hasGCVersionChanged: boolean;
104
+ /** Tells whether the GC state in summary needs to be reset in the next summary. */
105
+ readonly summaryStateNeedsReset: boolean;
98
106
  /** Tells whether GC data should be written to the root of the summary tree. */
99
107
  readonly writeDataAtRoot: boolean;
100
108
  /** Run garbage collection and update the reference / used state of the system. */
@@ -104,13 +112,14 @@ export interface IGarbageCollector {
104
112
  /** Summarizes the GC data and returns it as a summary tree. */
105
113
  summarize(): ISummaryTreeWithStats | undefined;
106
114
  /** Returns a map of each data store id to its GC details in the base summary. */
107
- getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>>;
115
+ getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>>;
108
116
  /** Called when the latest summary of the system has been refreshed. */
109
117
  latestSummaryStateRefreshed(result: RefreshSummaryResult, readAndParseBlob: ReadAndParseBlob): Promise<void>;
110
118
  /** Called when a node is changed. Used to detect and log when an inactive node is changed. */
111
119
  nodeChanged(id: string): void;
112
120
  /** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
113
121
  addedOutboundReference(fromNodeId: string, toNodeId: string): void;
122
+ dispose(): void;
114
123
  }
115
124
 
116
125
  /**
@@ -174,6 +183,7 @@ export class GarbageCollector implements IGarbageCollector {
174
183
  gcOptions: IGCRuntimeOptions,
175
184
  deleteUnusedRoutes: (unusedRoutes: string[]) => void,
176
185
  getCurrentTimestampMs: () => number,
186
+ closeFn: (error?: ICriticalContainerError) => void,
177
187
  baseSnapshot: ISnapshotTree | undefined,
178
188
  readAndParseBlob: ReadAndParseBlob,
179
189
  baseLogger: ITelemetryLogger,
@@ -185,6 +195,7 @@ export class GarbageCollector implements IGarbageCollector {
185
195
  gcOptions,
186
196
  deleteUnusedRoutes,
187
197
  getCurrentTimestampMs,
198
+ closeFn,
188
199
  baseSnapshot,
189
200
  readAndParseBlob,
190
201
  baseLogger,
@@ -198,6 +209,11 @@ export class GarbageCollector implements IGarbageCollector {
198
209
  */
199
210
  public readonly shouldRunGC: boolean;
200
211
 
212
+ /**
213
+ * The time in ms to expire a session for a client for gc.
214
+ */
215
+ public readonly sessionExpiryTimeoutMs: number | undefined;
216
+
201
217
  /**
202
218
  * This tracks two things:
203
219
  * 1. Whether GC is enabled - If this is 0, GC is disabled. If this is greater than 0, GC is enabled.
@@ -208,13 +224,16 @@ export class GarbageCollector implements IGarbageCollector {
208
224
  }
209
225
 
210
226
  /**
211
- * Tells whether the GC version has changed compared to the version in the latest summary.
227
+ * Tells whether the GC state needs to be reset in the next summary. We need to do this if:
228
+ * 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
229
+ * 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
230
+ * 3. The GC version in the latest summary is different from the current GC version. This can happen if:
231
+ * 3.1. The summary this client loaded with has data from a different GC version.
232
+ * 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
212
233
  */
213
- public get hasGCVersionChanged(): boolean {
214
- // The current version can differ from the latest summary version in two cases:
215
- // 1. The summary this client loaded with has data from a different GC version.
216
- // 2. This client's latest summary was updated from a snapshot that has a different GC version.
217
- return this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion;
234
+ public get summaryStateNeedsReset(): boolean {
235
+ return this.initialStateNeedsReset ||
236
+ (this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
218
237
  }
219
238
 
220
239
  /**
@@ -227,15 +246,24 @@ export class GarbageCollector implements IGarbageCollector {
227
246
  private readonly mc: MonitoringContext;
228
247
 
229
248
  /**
230
- * Tells whether the GC data should be written to the root of the summary tree. We do this under 2 conditions:
231
- * 1. If `writeDataAtRoot` GC option is enabled.
232
- * 2. If the base summary has the GC data written at the root. This is to support forward compatibility where when
233
- * we start writing the GC data at root, older versions can detect that and write at root too.
249
+ * Tells whether the GC data should be written to the root of the summary tree.
234
250
  */
235
251
  private _writeDataAtRoot: boolean = false;
236
252
  public get writeDataAtRoot(): boolean {
237
253
  return this._writeDataAtRoot;
238
- }
254
+ }
255
+
256
+ /**
257
+ * Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
258
+ * 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
259
+ * it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
260
+ * 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
261
+ * a document and the first time GC is enabled after is was disabled before.
262
+ *
263
+ * Note that the state needs reset only for the very first time summary is generated by this client. After that, the
264
+ * state will be up-to-date and this flag will be reset.
265
+ */
266
+ private initialStateNeedsReset: boolean = false;
239
267
 
240
268
  // The current GC version that this container is running.
241
269
  private readonly currentGCVersion = GCVersion;
@@ -251,11 +279,13 @@ export class GarbageCollector implements IGarbageCollector {
251
279
  // Promise when resolved initializes the base state of the nodes from the base summary state.
252
280
  private readonly initializeBaseStateP: Promise<void>;
253
281
  // The map of data store ids to their GC details in the base summary returned in getDataStoreGCDetails().
254
- private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionSummaryDetails>>;
282
+ private readonly dataStoreGCDetailsP: Promise<Map<string, IGarbageCollectionDetailsBase>>;
255
283
  // The time after which an unreferenced node can be deleted. Currently, we only set the node's state to expired.
256
284
  private readonly deleteTimeoutMs: number;
257
285
  // Map of node ids to their unreferenced state tracker.
258
286
  private readonly unreferencedNodesState: Map<string, UnreferencedStateTracker> = new Map();
287
+ // The timeout responsible for closing the container when the session has expired
288
+ private sessionExpiryTimer?: ReturnType<typeof setTimeout>;
259
289
 
260
290
  protected constructor(
261
291
  private readonly provider: IGarbageCollectionRuntime,
@@ -264,6 +294,7 @@ export class GarbageCollector implements IGarbageCollector {
264
294
  private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,
265
295
  /** Returns the current timestamp to be assigned to nodes that become unreferenced. */
266
296
  private readonly getCurrentTimestampMs: () => number,
297
+ private readonly closeFn: (error?: ICriticalContainerError) => void,
267
298
  baseSnapshot: ISnapshotTree | undefined,
268
299
  readAndParseBlob: ReadAndParseBlob,
269
300
  baseLogger: ITelemetryLogger,
@@ -276,17 +307,29 @@ export class GarbageCollector implements IGarbageCollector {
276
307
  this.deleteTimeoutMs = this.gcOptions.deleteTimeoutMs ?? defaultDeleteTimeoutMs;
277
308
 
278
309
  let prevSummaryGCVersion: number | undefined;
310
+
279
311
  // GC can only be enabled during creation. After that, it can never be enabled again. So, for existing
280
- // documents, we get this information from the metadata blob.
312
+ // documents, we get this information from the metadata blob. Similarly the session timeout should be
313
+ // consistent across all clients, thus we grab it as well from the metadata blob, and set it once on creation.
281
314
  if (existing) {
282
315
  prevSummaryGCVersion = getGCVersion(metadata);
283
316
  // Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
284
317
  // other exsiting documents, GC is enabled.
285
318
  this.gcEnabled = prevSummaryGCVersion > 0;
319
+ this.sessionExpiryTimeoutMs = metadata?.sessionExpiryTimeoutMs;
286
320
  } else {
287
321
  // For new documents, GC has to be exlicitly enabled via the gcAllowed flag in GC options.
288
322
  this.gcEnabled = gcOptions.gcAllowed === true;
323
+ this.sessionExpiryTimeoutMs = this.gcOptions.gcTestSessionTimeoutMs;
289
324
  }
325
+
326
+ // If session expiry is enabled, we need to close the container when the timeout expires
327
+ if (this.sessionExpiryTimeoutMs !== undefined) {
328
+ const expiryMs = this.sessionExpiryTimeoutMs;
329
+ this.sessionExpiryTimer = setTimeout(() => this.closeFn(
330
+ new ClientSessionExpiredError(`Client session expired.`, expiryMs)), expiryMs);
331
+ }
332
+
290
333
  // For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
291
334
  // latest tracked GC version. For new documents, we will be writing the first summary with the current version.
292
335
  this.latestSummaryGCVersion = prevSummaryGCVersion ?? this.currentGCVersion;
@@ -302,14 +345,25 @@ export class GarbageCollector implements IGarbageCollector {
302
345
  // Whether GC sweep phase should run or not. If this is false, only GC mark phase is run. Can override with
303
346
  // localStorage flag.
304
347
  this.shouldRunSweep = this.shouldRunGC &&
305
- (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true);
348
+ (this.mc.config.getBoolean(runSweepKey) ?? gcOptions.runSweep === true)
349
+ && this.sessionExpiryTimer !== undefined;
306
350
 
307
351
  // Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
308
352
  this.testMode = this.mc.config.getBoolean(gcTestModeKey) ?? gcOptions.runGCInTestMode === true;
309
353
 
310
- // If `writeDataAtRoot` GC option is true, we should write the GC data into the root of the summary tree. This
311
- // GC option is used for testing only. It will be removed once we start writing GC data into root by default.
312
- this._writeDataAtRoot = this.gcOptions.writeDataAtRoot === true;
354
+ /**
355
+ * Enable resetting initial state once the following issue is resolved:
356
+ * https://github.com/microsoft/FluidFramework/issues/8878.
357
+ * Currently, the GC tree is not written at root, so we don't know if the base snapshot contains GC tree or not.
358
+ */
359
+ // The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't contain
360
+ // GC tree and GC is enabled.
361
+ // const gcTreePresent = baseSnapshot?.trees[gcTreeKey] !== undefined;
362
+ // this.initialStateNeedsReset = gcTreePresent ? !this.shouldRunGC : this.shouldRunGC;
363
+
364
+ // If `writeDataAtRoot` setting is true, write the GC data into the root of the summary tree. We do this so that
365
+ // the roll out can be staged. Once its rolled out everywhere, we will start writing at root by default.
366
+ this._writeDataAtRoot = this.mc.config.getBoolean(writeAtRootKey) ?? this.gcOptions.writeDataAtRoot === true;
313
367
 
314
368
  // Get the GC state from the GC blob in the base snapshot. Use LazyPromise because we only want to do
315
369
  // this once since it involves fetching blobs from storage which is expensive.
@@ -321,7 +375,7 @@ export class GarbageCollector implements IGarbageCollector {
321
375
  // For newer documents, GC data should be present in the GC tree in the root of the snapshot.
322
376
  const gcSnapshotTree = baseSnapshot.trees[gcTreeKey];
323
377
  if (gcSnapshotTree !== undefined) {
324
- // forward-compat - If a newer version has written the GC tree at root, we should also do the same.
378
+ // If the GC tree is written at root, we should also do the same.
325
379
  this._writeDataAtRoot = true;
326
380
  return getGCStateFromSnapshot(gcSnapshotTree, readAndParseBlob);
327
381
  }
@@ -339,7 +393,7 @@ export class GarbageCollector implements IGarbageCollector {
339
393
  continue;
340
394
  }
341
395
 
342
- const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionSummaryDetails>(blobId);
396
+ const gcSummaryDetails = await readAndParseBlob<IGarbageCollectionDetailsBase>(blobId);
343
397
  // If there are no nodes for this data store, skip it.
344
398
  if (gcSummaryDetails.gcData?.gcNodes === undefined) {
345
399
  continue;
@@ -401,8 +455,8 @@ export class GarbageCollector implements IGarbageCollector {
401
455
  });
402
456
 
403
457
  // Get the GC details for each data store from the GC state in the base summary. This is returned in
404
- // getDataStoreBaseGCDetails and is used to initialize each data store's base GC details.
405
- this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionSummaryDetails>>(async () => {
458
+ // getDataStoreBaseGCDetails and is used to initialize each data store's GC state.
459
+ this.dataStoreGCDetailsP = new LazyPromise<Map<string, IGarbageCollectionDetailsBase>>(async () => {
406
460
  const baseState = await baseSummaryStateP;
407
461
  if (baseState === undefined) {
408
462
  return new Map();
@@ -453,12 +507,15 @@ export class GarbageCollector implements IGarbageCollector {
453
507
  const {
454
508
  logger = this.mc.logger,
455
509
  runSweep = this.shouldRunSweep,
456
- fullGC = this.gcOptions.runFullGC === true || this.hasGCVersionChanged,
510
+ fullGC = this.gcOptions.runFullGC === true || this.summaryStateNeedsReset,
457
511
  } = options;
458
512
 
459
513
  return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
460
514
  await this.initializeBaseStateP;
461
515
 
516
+ // Let the runtime update its pending state before GC runs.
517
+ await this.provider.updateStateBeforeGC();
518
+
462
519
  const gcStats: {
463
520
  deletedNodes?: number,
464
521
  totalNodes?: number,
@@ -530,9 +587,9 @@ export class GarbageCollector implements IGarbageCollector {
530
587
 
531
588
  /**
532
589
  * Returns a map of data store ids to their base GC details generated from the base summary.This is used to
533
- * initialize the data stores with their base GC state.
590
+ * initialize the GC state of data stores.
534
591
  */
535
- public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionSummaryDetails>> {
592
+ public async getDataStoreBaseGCDetails(): Promise<Map<string, IGarbageCollectionDetailsBase>> {
536
593
  return this.dataStoreGCDetailsP;
537
594
  }
538
595
 
@@ -544,6 +601,10 @@ export class GarbageCollector implements IGarbageCollector {
544
601
  result: RefreshSummaryResult,
545
602
  readAndParseBlob: ReadAndParseBlob,
546
603
  ): Promise<void> {
604
+ // After a summary is successfully submitted and ack'd by this client, the GC state should have been reset in
605
+ // the summary and doesn't need to be reset anymore.
606
+ this.initialStateNeedsReset = false;
607
+
547
608
  if (!this.shouldRunGC || !result.latestSummaryUpdated) {
548
609
  return;
549
610
  }
@@ -574,6 +635,13 @@ export class GarbageCollector implements IGarbageCollector {
574
635
  );
575
636
  }
576
637
 
638
+ public dispose(): void {
639
+ if (this.sessionExpiryTimer !== undefined) {
640
+ clearTimeout(this.sessionExpiryTimer);
641
+ this.sessionExpiryTimer = undefined;
642
+ }
643
+ }
644
+
577
645
  /**
578
646
  * Called when an outbound reference is added to a node. This is used to identify all nodes that have been
579
647
  * referenced between summaries so that their unreferenced timestamp can be reset.
@@ -662,6 +730,9 @@ export class GarbageCollector implements IGarbageCollector {
662
730
  return;
663
731
  }
664
732
 
733
+ // Validate that we have identified all references correctly.
734
+ this.validateReferenceCorrectness(currentGCData);
735
+
665
736
  /**
666
737
  * Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
667
738
  * edges that have been added since then. To do this, combine the GC data from the last run and the current
@@ -702,6 +773,60 @@ export class GarbageCollector implements IGarbageCollector {
702
773
  }
703
774
  }
704
775
  }
776
+
777
+ /**
778
+ * Validates that all new references are correctly identified and processed. The basic principle for validation is
779
+ * that we should not have new references in the reference graph (GC data) that have not been notified to the
780
+ * garbage collector via `referenceAdded`.
781
+ * We validate that the references in the current reference graph should be a subset of the references in the last
782
+ * run's reference graph + references since the last run.
783
+ * @param currentGCData - The GC data (reference graph) from the current GC run.
784
+ */
785
+ private validateReferenceCorrectness(currentGCData: IGarbageCollectionData) {
786
+ assert(this.gcDataFromLastRun !== undefined, 0x2b7
787
+ /* "Can't validate correctness without GC data from last run" */);
788
+
789
+ // Get a list of all the outbound routes (or references) in the current GC data.
790
+ const currentReferences: string[] = [];
791
+ for (const [nodeId, outboundRoutes] of Object.entries(currentGCData.gcNodes)) {
792
+ /**
793
+ * Remove routes from a child node to its parent which is added implicitly by the runtime. For instance,
794
+ * each adds its data store as an outbound route to mark it as referenced if the DDS is referenced.
795
+ * We won't get any explicit notification for these references so they must be removed before validation.
796
+ */
797
+ const explicitRoutes = outboundRoutes.filter((route) => !nodeId.startsWith(route));
798
+ currentReferences.push(...explicitRoutes);
799
+ }
800
+
801
+ // Get a list of outbound routes (or references) from the last run's GC data plus references added since the
802
+ // last run that were notified via `referenceAdded`.
803
+ const explicitReferences: string[] = [];
804
+ for (const [, outboundRoutes] of Object.entries(this.gcDataFromLastRun.gcNodes)) {
805
+ explicitReferences.push(...outboundRoutes);
806
+ }
807
+ this.referencesSinceLastRun.forEach((outboundRoutes: string[]) => {
808
+ explicitReferences.push(...outboundRoutes);
809
+ });
810
+
811
+ // Validate that the current reference graph doesn't have references that we are not already aware of. If this
812
+ // happens, it might indicate data corruption since we may delete objects prematurely.
813
+ currentReferences.forEach((route: string) => {
814
+ // Validate references for data stores only whose routes are of the format "/dataStoreId". Currently, layers
815
+ // below data stores don't have GC implemented so there is no guarantee their references will be notified.
816
+ if (route.split("/").length === 2 && !explicitReferences.includes(route)) {
817
+ /**
818
+ * The following log will be enabled once this issue is resolved:
819
+ * https://github.com/microsoft/FluidFramework/issues/8878.
820
+ */
821
+ // We should ideally throw a data corruption error here. However, send an error for now until we have
822
+ // implemented sweep and have reasonable confidence in the sweep process.
823
+ // this.mc.logger.sendErrorEvent({
824
+ // eventName: "gcUnknownOutboundRoute",
825
+ // route,
826
+ // });
827
+ }
828
+ });
829
+ }
705
830
  }
706
831
 
707
832
  /**
package/src/index.ts CHANGED
@@ -11,13 +11,14 @@ export {
11
11
  ISummaryRuntimeOptions,
12
12
  IContainerRuntimeOptions,
13
13
  isRuntimeMessage,
14
+ RuntimeMessage,
14
15
  unpackRuntimeMessage,
15
16
  ScheduleManager,
16
17
  agentSchedulerId,
17
18
  ContainerRuntime,
18
19
  } from "./containerRuntime";
19
- export * from "./deltaScheduler";
20
- export * from "./dataStoreRegistry";
20
+ export { DeltaScheduler } from "./deltaScheduler";
21
+ export { FluidDataStoreRegistry } from "./dataStoreRegistry";
21
22
  export {
22
23
  gcBlobPrefix,
23
24
  gcTreeKey,
@@ -25,8 +26,53 @@ export {
25
26
  IGCStats,
26
27
  IUsedStateStats,
27
28
  } from "./garbageCollection";
28
- export * from "./pendingStateManager";
29
- export * from "./summarizer";
30
- export * from "./summarizerTypes";
31
- export * from "./summaryCollection";
29
+ export {
30
+ IPendingFlush,
31
+ IPendingFlushMode,
32
+ IPendingLocalState,
33
+ IPendingMessage,
34
+ IPendingState,
35
+ } from "./pendingStateManager";
36
+ export { Summarizer } from "./summarizer";
37
+ export {
38
+ EnqueueSummarizeResult,
39
+ IAckSummaryResult,
40
+ IBaseSummarizeResult,
41
+ IBroadcastSummaryResult,
42
+ ICancellationToken,
43
+ IConnectableRuntime,
44
+ IEnqueueSummarizeOptions,
45
+ IGenerateSummaryTreeResult,
46
+ IGeneratedSummaryStats,
47
+ INackSummaryResult,
48
+ IOnDemandSummarizeOptions,
49
+ IProvideSummarizer,
50
+ ISubmitSummaryOpResult,
51
+ ISubmitSummaryOptions,
52
+ ISummarizeOptions,
53
+ ISummarizeResults,
54
+ ISummarizer,
55
+ ISummarizerEvents,
56
+ ISummarizerInternalsProvider,
57
+ ISummarizerOptions,
58
+ ISummarizerRuntime,
59
+ ISummarizingWarning,
60
+ ISummaryCancellationToken,
61
+ IUploadSummaryResult,
62
+ SubmitSummaryResult,
63
+ SummarizeResultPart,
64
+ SummarizerStopReason,
65
+ } from "./summarizerTypes";
66
+ export {
67
+ IAckedSummary,
68
+ IClientSummaryWatcher,
69
+ ISummary,
70
+ ISummaryCollectionOpEvents,
71
+ ISummaryAckMessage,
72
+ ISummaryNackMessage,
73
+ ISummaryOpMessage,
74
+ OpActionEventListener,
75
+ OpActionEventName,
76
+ SummaryCollection,
77
+ } from "./summaryCollection";
32
78
  export { ICancellableSummarizerController, neverCancelledSummaryToken } from "./runWhileConnectedCoordinator";
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "0.54.2";
9
+ export const pkgVersion = "0.56.0-49831";
@@ -201,13 +201,8 @@ export class RunningSummarizer implements IDisposable {
201
201
  }
202
202
  this.heuristicData.lastOpSequenceNumber = sequenceNumber;
203
203
 
204
- if (this.tryRunEnqueuedSummary()) {
205
- // Intentionally do nothing; check for enqueued on-demand summaries
206
- } else if (type === MessageType.Save) {
207
- // Check for ops requesting summary
208
- // Note: as const is only required until TypeScript version 4.3
209
- this.trySummarize(`save;${clientId}: ${contents}` as const);
210
- } else {
204
+ // Check for enqueued on-demand summaries; Intentionally do nothing otherwise
205
+ if (!this.tryRunEnqueuedSummary()) {
211
206
  this.heuristicRunner?.run();
212
207
  }
213
208
  }
@@ -8,6 +8,7 @@ import { IDocumentStorageService } from "@fluidframework/driver-definitions";
8
8
  import { readAndParse } from "@fluidframework/driver-utils";
9
9
  import { ISequencedDocumentMessage, ISnapshotTree, SummaryType } from "@fluidframework/protocol-definitions";
10
10
  import { channelsTreeName, ISummaryTreeWithStats } from "@fluidframework/runtime-definitions";
11
+ import { gcTreeKey } from "./garbageCollection";
11
12
 
12
13
  type OmitAttributesVersions<T> = Omit<T, "snapshotFormatVersion" | "summaryFormatVersion">;
13
14
  interface IFluidDataStoreAttributes0 {
@@ -69,7 +70,6 @@ export function getAttributesFormatVersion(attributes: ReadFluidDataStoreAttribu
69
70
  return 0;
70
71
  }
71
72
 
72
- // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
73
73
  export function hasIsolatedChannels(attributes: ReadFluidDataStoreAttributes): boolean {
74
74
  return !!attributes.summaryFormatVersion && !attributes.disableIsolatedChannels;
75
75
  }
@@ -85,6 +85,8 @@ export interface IContainerRuntimeMetadata extends ICreateContainerMetadata {
85
85
  readonly gcFeature?: GCVersion;
86
86
  /** Counter of the last summary happened, increments every time we summarize */
87
87
  readonly summaryCount?: number;
88
+ /** If this is present, the session for this container will expire after this time and the container will close */
89
+ readonly sessionExpiryTimeoutMs?: number;
88
90
  }
89
91
 
90
92
  export interface ICreateContainerMetadata {
@@ -120,7 +122,6 @@ export const extractSummaryMetadataMessage = (
120
122
  type: message.type,
121
123
  };
122
124
 
123
- // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
124
125
  export function getMetadataFormatVersion(metadata?: IContainerRuntimeMetadata): number {
125
126
  /**
126
127
  * Version 2+: Introduces runtime sequence number for data verification.
@@ -141,7 +142,6 @@ export const chunksBlobName = ".chunks";
141
142
  export const electedSummarizerBlobName = ".electedSummarizer";
142
143
  export const blobsTreeName = ".blobs";
143
144
 
144
- // eslint-disable-next-line prefer-arrow/prefer-arrow-functions
145
145
  export function rootHasIsolatedChannels(metadata?: IContainerRuntimeMetadata): boolean {
146
146
  return !!metadata && !metadata.disableIsolatedChannels;
147
147
  }
@@ -162,7 +162,7 @@ export const protocolTreeName = ".protocol";
162
162
  * isolated data stores namespace. Without the namespace, this must
163
163
  * be used to prevent name collisions with data store IDs.
164
164
  */
165
- export const nonDataStorePaths = [protocolTreeName, ".logTail", ".serviceProtocol", blobsTreeName];
165
+ export const nonDataStorePaths = [protocolTreeName, ".logTail", ".serviceProtocol", blobsTreeName, gcTreeKey];
166
166
 
167
167
  export const dataStoreAttributesBlobName = ".component";
168
168
 
@@ -76,11 +76,6 @@ export type SummarizeReason =
76
76
  * ack op.
77
77
  */
78
78
  | "maxOps"
79
- /**
80
- * Special case to generate a summary in response to a Save op.
81
- * @deprecated - do not use save ops
82
- */
83
- | `save;${string}: ${string}`
84
79
  /**
85
80
  * Special case to attempt to summarize one last time before the
86
81
  * summarizer client closes itself. This is to prevent cases where
@@ -141,6 +141,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
141
141
  this.refreshSummarizer();
142
142
  };
143
143
 
144
+ private static readonly isStartingOrRunning = (state: SummaryManagerState) =>
145
+ state === SummaryManagerState.Starting || state === SummaryManagerState.Running;
146
+
144
147
  private getShouldSummarizeState(): ShouldSummarizeState {
145
148
  if (!this.connectedState.connected) {
146
149
  return { shouldSummarize: false, stopReason: "parentNotConnected" };
@@ -253,7 +256,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
253
256
  // Note that summarizer may keep going (like doing last summary).
254
257
  // Ideally we await stopping process, but this code path is due to a bug
255
258
  // that needs to be fixed either way.
256
- this.stop("summarizerException");
259
+ if (SummaryManager.isStartingOrRunning(this.state)) {
260
+ this.stop("summarizerException");
261
+ }
257
262
  }
258
263
  }).finally(() => {
259
264
  assert(this.state !== SummaryManagerState.Off, 0x264 /* "Expected: Not Off" */);
@@ -273,8 +278,9 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
273
278
  }
274
279
 
275
280
  private stop(reason: SummarizerStopReason) {
276
- assert(this.state === SummaryManagerState.Running || this.state === SummaryManagerState.Starting,
277
- 0x265 /* "Expected: Starting or Running" */);
281
+ if (!SummaryManager.isStartingOrRunning(this.state)) {
282
+ return;
283
+ }
278
284
  this.state = SummaryManagerState.Stopping;
279
285
 
280
286
  // Stopping the running summarizer client should trigger a change