@fluidframework/container-runtime 2.0.0-internal.8.0.0 → 2.0.0-internal.8.0.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/dist/containerRuntime.d.ts.map +1 -1
  2. package/dist/containerRuntime.js +0 -3
  3. package/dist/containerRuntime.js.map +1 -1
  4. package/dist/gc/garbageCollection.d.ts +8 -12
  5. package/dist/gc/garbageCollection.d.ts.map +1 -1
  6. package/dist/gc/garbageCollection.js +71 -103
  7. package/dist/gc/garbageCollection.js.map +1 -1
  8. package/dist/gc/gcDefinitions.d.ts +0 -1
  9. package/dist/gc/gcDefinitions.d.ts.map +1 -1
  10. package/dist/gc/gcDefinitions.js.map +1 -1
  11. package/dist/gc/gcSummaryStateTracker.d.ts +1 -1
  12. package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
  13. package/dist/gc/gcSummaryStateTracker.js +3 -0
  14. package/dist/gc/gcSummaryStateTracker.js.map +1 -1
  15. package/dist/gc/gcTelemetry.d.ts.map +1 -1
  16. package/dist/gc/gcTelemetry.js +4 -14
  17. package/dist/gc/gcTelemetry.js.map +1 -1
  18. package/dist/packageVersion.d.ts +1 -1
  19. package/dist/packageVersion.js +1 -1
  20. package/dist/packageVersion.js.map +1 -1
  21. package/lib/containerRuntime.d.ts.map +1 -1
  22. package/lib/containerRuntime.js +0 -3
  23. package/lib/containerRuntime.js.map +1 -1
  24. package/lib/gc/garbageCollection.d.ts +8 -12
  25. package/lib/gc/garbageCollection.d.ts.map +1 -1
  26. package/lib/gc/garbageCollection.js +72 -104
  27. package/lib/gc/garbageCollection.js.map +1 -1
  28. package/lib/gc/gcDefinitions.d.ts +0 -1
  29. package/lib/gc/gcDefinitions.d.ts.map +1 -1
  30. package/lib/gc/gcDefinitions.js.map +1 -1
  31. package/lib/gc/gcSummaryStateTracker.d.ts +1 -1
  32. package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
  33. package/lib/gc/gcSummaryStateTracker.js +3 -0
  34. package/lib/gc/gcSummaryStateTracker.js.map +1 -1
  35. package/lib/gc/gcTelemetry.d.ts.map +1 -1
  36. package/lib/gc/gcTelemetry.js +4 -14
  37. package/lib/gc/gcTelemetry.js.map +1 -1
  38. package/lib/packageVersion.d.ts +1 -1
  39. package/lib/packageVersion.js +1 -1
  40. package/lib/packageVersion.js.map +1 -1
  41. package/package.json +16 -16
  42. package/src/containerRuntime.ts +0 -3
  43. package/src/gc/garbageCollection.ts +92 -119
  44. package/src/gc/gcDefinitions.ts +0 -1
  45. package/src/gc/gcSummaryStateTracker.ts +5 -1
  46. package/src/gc/gcTelemetry.ts +4 -13
  47. package/src/packageVersion.ts +1 -1
@@ -60,30 +60,26 @@ export declare class GarbageCollector implements IGarbageCollector {
60
60
  private readonly getNodePackagePath;
61
61
  /** Returns the timestamp of the last summary generated for this container. */
62
62
  private readonly getLastSummaryTimestampMs;
63
- /** Returns true if connection is active, i.e. it's "write" connection and the runtime is connected. */
64
- private readonly activeConnection;
65
63
  private readonly submitMessage;
66
64
  get summaryStateNeedsReset(): boolean;
67
65
  /** Returns the count of data stores whose GC state updated since the last summary. */
68
66
  get updatedDSCountSinceLastSummary(): number;
69
67
  protected constructor(createParams: IGarbageCollectorCreateParams);
70
68
  /**
71
- * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
72
- * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
69
+ * Called during container initialization. Initializes the tombstone and deleted nodes state from the base snapshot.
70
+ * Also, initializes the GC state including unreferenced nodes tracking if a current reference timestamp exists.
71
+ * Note that if there is any GC state in the base snapshot, then there will definitely be a reference timestamp
72
+ * to work with - The GC state would have been generated using a timestamp which is part of the snapshot.
73
73
  */
74
74
  initializeBaseState(): Promise<void>;
75
75
  /**
76
- * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
77
- * All current tracking is reset and updated from the data in the snapshot.
78
- * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
79
- * is reset.
80
- * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
81
- * timestamp.
76
+ * Initialize the GC state if not already initialized. If GC state is already initialized, update the unreferenced
77
+ * state tracking as per the current reference timestamp.
82
78
  */
83
- private updateStateFromSnapshotData;
79
+ private initializeOrUpdateGCState;
84
80
  /**
85
81
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
86
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
82
+ * to initialize or update the unreference state tracking.
87
83
  * @param connected - Whether the runtime connected / disconnected.
88
84
  * @param clientId - The clientId of this runtime.
89
85
  */
@@ -1 +1 @@
1
- {"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AACzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAwB,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAMX,MAAM,iBAAiB,CAAC;AAazB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IACrE,uGAAuG;IACvG,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAgB;IAEjD,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IAsKjE;;;OAGG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA2BjD;;;;;;;OAOG;IACH,OAAO,CAAC,2BAA2B;IA6EnC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAmBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IAwEhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA2CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAqDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyBxE;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAkD1B"}
1
+ {"version":3,"file":"garbageCollection.d.ts","sourceRoot":"","sources":["../../src/gc/garbageCollection.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,QAAQ,EAAE,MAAM,iCAAiC,CAAC;AAC3D,OAAO,EAGN,6BAA6B,EAC7B,gBAAgB,EAChB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAE7C,OAAO,EAIN,mBAAmB,EAGnB,MAAM,iCAAiC,CAAC;AAEzC,OAAO,EAEN,iBAAiB,EAEjB,MAAM,qBAAqB,CAAC;AAE7B,OAAO,EAAwB,yBAAyB,EAAE,MAAM,iBAAiB,CAAC;AAClF,OAAO,EAAE,qBAAqB,EAAE,MAAM,YAAY,CAAC;AAEnD,OAAO,EAEN,iBAAiB,EACjB,6BAA6B,EAG7B,QAAQ,EAER,WAAW,EAMX,MAAM,iBAAiB,CAAC;AAazB;;;;;;;;;;;;;;;;;;;;;GAqBG;AACH,qBAAa,gBAAiB,YAAW,iBAAiB;WAC3C,MAAM,CAAC,YAAY,EAAE,6BAA6B,GAAG,iBAAiB;IAIpF,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA2B;IAEnD,IAAW,WAAW,IAAI,OAAO,CAEhC;IAGD,OAAO,CAAC,iBAAiB,CAAqC;IAG9D,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAAoC;IAE9E,OAAO,CAAC,UAAU,CAAgB;IAElC,OAAO,CAAC,YAAY,CAA0B;IAG9C,OAAO,CAAC,QAAQ,CAAC,iBAAiB,CAAsD;IAExF,OAAO,CAAC,QAAQ,CAAC,kCAAkC,CAAgB;IAEnE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAyC;IAExE,OAAO,CAAC,QAAQ,CAAC,sBAAsB,CAAoD;IAE3F,OAAO,CAAC,kBAAkB,CAAoB;IAG9C,OAAO,CAAC,aAAa,CAAK;IAE1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAA4B;IACpD,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAU;IAE7C,OAAO,CAAC,QAAQ,CAAC,mBAAmB,CAAwB;IAC5D,OAAO,CAAC,QAAQ,CAAC,gBAAgB,CAAqB;IAEtD,iFAAiF;IACjF,IAAW,2BAA2B,IAAI,OAAO,CAEhD;IACD,uEAAuE;IACvE,IAAW,oBAAoB,IAAI,OAAO,CAEzC;IACD,kEAAkE;IAClE,IAAW,qBAAqB,IAAI,OAAO,CAE1C;IAED,8DAA8D;IAC9D,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAES;IAC5C,8EAA8E;IAC9E,OAAO,CAAC,QAAQ,CAAC,yBAAyB,CAA2B;IAErE,OAAO,CAAC,QAAQ,CAAC,aAAa,CAA+C;IAE7E,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,sFAAsF;IACtF,IAAW,8BAA8B,IAAI,MAAM,CAElD;IAED,SAAS,aAAa,YAAY,EAAE,6BAA6B;IA+KjE;;;;;OAKG;IACU,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;IA6BjD;;;OAGG;YACW,yBAAyB;IAmBvC;;;;;OAKG;IACI,kBAAkB,CAAC,SAAS,EAAE,OAAO,EAAE,QAAQ,CAAC,EAAE,MAAM,GAAG,SAAS,GAAG,IAAI;IAuBlF;;;OAGG;IACU,gBAAgB,IAAI,OAAO,CAAC,6BAA6B,CAAC;IAIvE;;;OAGG;IACU,cAAc,CAC1B,OAAO,EAAE;QACR,0CAA0C;QAC1C,MAAM,CAAC,EAAE,mBAAmB,CAAC;QAC7B,sDAAsD;QACtD,QAAQ,CAAC,EAAE,OAAO,CAAC;QACnB,oCAAoC;QACpC,MAAM,CAAC,EAAE,OAAO,CAAC;KACjB,EACD,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,OAAO,CAAC,QAAQ,GAAG,SAAS,CAAC;IA2EhC;;;;;;;;;;;;OAYG;YACW,KAAK;IA2CnB;;;;;;;;;;;;;OAaG;IACH,OAAO,CAAC,YAAY;IAqDpB;;;;;;;;;;OAUG;IACH,OAAO,CAAC,aAAa;IAgErB;;;;;;;;;;;;;;OAcG;IACH,OAAO,CAAC,gCAAgC;IAqExC;;;;OAIG;IACI,SAAS,CACf,QAAQ,EAAE,OAAO,EACjB,UAAU,EAAE,OAAO,EACnB,gBAAgB,CAAC,EAAE,iBAAiB,GAClC,gBAAgB,GAAG,SAAS;IAuBxB,WAAW,IAAI,WAAW;IAcjC;;OAEG;IACU,oBAAoB,CAAC,MAAM,EAAE,qBAAqB,GAAG,OAAO,CAAC,IAAI,CAAC;IAI/E;;;;OAIG;IACI,cAAc,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IAyBxE;;;;OAIG;IACH,OAAO,CAAC,qBAAqB;IAqC7B;;;;;;;;;OASG;IACI,WAAW,CACjB,QAAQ,EAAE,MAAM,EAChB,MAAM,EAAE,QAAQ,GAAG,SAAS,EAC5B,WAAW,CAAC,EAAE,MAAM,EACpB,WAAW,CAAC,EAAE,SAAS,MAAM,EAAE,EAC/B,OAAO,CAAC,EAAE,QAAQ,EAClB,UAAU,CAAC,EAAE,iBAAiB;IAyD/B;;;;;;OAMG;IACI,sBAAsB,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM;IAqBtE;;;OAGG;IACI,aAAa,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO;IAIxC,OAAO,IAAI,IAAI;IAKtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IA0DzB;;;;;;OAMG;IACH,OAAO,CAAC,kBAAkB;CAkE1B"}
@@ -9,6 +9,7 @@ const core_utils_1 = require("@fluidframework/core-utils");
9
9
  const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
10
10
  const runtime_utils_1 = require("@fluidframework/runtime-utils");
11
11
  const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
12
+ const blobManager_1 = require("../blobManager");
12
13
  const containerRuntime_1 = require("../containerRuntime");
13
14
  const error_1 = require("../error");
14
15
  const messageTypes_1 = require("../messageTypes");
@@ -83,7 +84,6 @@ class GarbageCollector {
83
84
  this.isSummarizerClient = createParams.isSummarizerClient;
84
85
  this.getNodePackagePath = createParams.getNodePackagePath;
85
86
  this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
86
- this.activeConnection = createParams.activeConnection;
87
87
  this.submitMessage = createParams.submitMessage;
88
88
  const baseSnapshot = createParams.baseSnapshot;
89
89
  const readAndParseBlob = createParams.readAndParseBlob;
@@ -144,28 +144,14 @@ class GarbageCollector {
144
144
  }
145
145
  });
146
146
  /**
147
- * Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
148
- * connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
149
- * GC state and updates their inactive or sweep-ready state.
147
+ * Set up the initializer which initializes the GC state from the data in base snapshot. It sets up GC data
148
+ * from the base GC state and starts tracking the state of unreferenced nodes.
149
+ *
150
+ * Must only be called if there is a current reference timestamp.
150
151
  */
151
152
  this.initializeGCStateFromBaseSnapshotP = new core_utils_1.LazyPromise(async () => {
152
- /**
153
- * If there is no current reference timestamp, skip initialization. We need the current timestamp to track
154
- * how long objects have been unreferenced and if they can be deleted.
155
- *
156
- * Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
157
- * for this container and it is in read mode. In this scenario, there is no point in running GC anyway
158
- * because references in the container do not change without any ops, i.e., there is nothing to collect.
159
- */
160
153
  const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
161
- if (currentReferenceTimestampMs === undefined) {
162
- // Log an event so we can evaluate how often we run into this scenario.
163
- this.mc.logger.sendErrorEvent({
164
- eventName: "GarbageCollectorInitializedWithoutTimestamp",
165
- gcConfigs: JSON.stringify(this.configs),
166
- });
167
- return;
168
- }
154
+ (0, core_utils_1.assert)(currentReferenceTimestampMs !== undefined, "Trying to initialize GC state without current timestamp");
169
155
  /**
170
156
  * The base snapshot data will not be present if the container is loaded from:
171
157
  * 1. The first summary created by the detached container.
@@ -173,11 +159,20 @@ class GarbageCollector {
173
159
  * 3. A summary that was generated before GC even existed.
174
160
  */
175
161
  const baseSnapshotData = await this.baseSnapshotDataP;
176
- if (baseSnapshotData === undefined) {
162
+ this.summaryStateTracker.initializeBaseState(baseSnapshotData);
163
+ if (baseSnapshotData?.gcState === undefined) {
177
164
  return;
178
165
  }
179
- this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
180
- this.summaryStateTracker.initializeBaseState(baseSnapshotData);
166
+ // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
167
+ // to the GC data from the snapshot data.
168
+ const gcNodes = {};
169
+ for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
170
+ if (nodeData.unreferencedTimestampMs !== undefined) {
171
+ this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
172
+ }
173
+ gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
174
+ }
175
+ this.gcDataFromLastRun = { gcNodes };
181
176
  });
182
177
  // Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
183
178
  // used to initialize the GC state of all the nodes in the container.
@@ -206,8 +201,10 @@ class GarbageCollector {
206
201
  });
207
202
  }
208
203
  /**
209
- * Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
210
- * during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
204
+ * Called during container initialization. Initializes the tombstone and deleted nodes state from the base snapshot.
205
+ * Also, initializes the GC state including unreferenced nodes tracking if a current reference timestamp exists.
206
+ * Note that if there is any GC state in the base snapshot, then there will definitely be a reference timestamp
207
+ * to work with - The GC state would have been generated using a timestamp which is part of the snapshot.
211
208
  */
212
209
  async initializeBaseState() {
213
210
  const baseSnapshotData = await this.baseSnapshotDataP;
@@ -232,98 +229,51 @@ class GarbageCollector {
232
229
  this.tombstones = Array.from(baseSnapshotData.tombstones);
233
230
  this.runtime.updateTombstonedRoutes(this.tombstones);
234
231
  }
232
+ await this.initializeOrUpdateGCState();
235
233
  }
236
234
  /**
237
- * Update state from the given snapshot data. This is done during load and during refreshing state from a snapshot.
238
- * All current tracking is reset and updated from the data in the snapshot.
239
- * @param snapshotData - The snapshot data to update state from. If this is undefined, all GC state and tracking
240
- * is reset.
241
- * @param currentReferenceTimestampMs - The current reference timestamp for marking unreferenced nodes' unreferenced
242
- * timestamp.
235
+ * Initialize the GC state if not already initialized. If GC state is already initialized, update the unreferenced
236
+ * state tracking as per the current reference timestamp.
243
237
  */
244
- updateStateFromSnapshotData(snapshotData, currentReferenceTimestampMs) {
245
- /**
246
- * Note: "newReferencesSinceLastRun" is not reset here. This is done because there may be references since the
247
- * snapshot that we are updating state from. For example, this client may have processed ops till seq#1000 and
248
- * its refreshing state from a summary that happened at seq#900. In this case, there may be references between
249
- * seq#901 and seq#1000 that we don't want to reset.
250
- * Unfortunately, there is no way to track the seq# of ops that add references, so we choose to not reset any
251
- * references here. This should be fine because, in the worst case, we may end up updating the unreferenced
252
- * timestamp of a node which will delay its deletion. Although not ideal, this will only happen in rare
253
- * scenarios, so it should be okay.
254
- */
255
- // Clear all existing unreferenced state tracking.
256
- for (const [, nodeStateTracker] of this.unreferencedNodesState) {
257
- nodeStateTracker.stopTracking();
258
- }
259
- this.unreferencedNodesState.clear();
260
- // If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
261
- // If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
262
- // tombstones.
263
- // If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
264
- // in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
265
- if (this.configs.shouldRunSweep) {
266
- const snapshotDeletedNodes = snapshotData?.deletedNodes
267
- ? new Set(snapshotData.deletedNodes)
268
- : undefined;
269
- // If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
270
- // delete them.
271
- if (snapshotDeletedNodes !== undefined) {
272
- const newDeletedNodes = [];
273
- for (const nodeId of snapshotDeletedNodes) {
274
- if (!this.deletedNodes.has(nodeId)) {
275
- newDeletedNodes.push(nodeId);
276
- }
277
- }
278
- if (newDeletedNodes.length > 0) {
279
- // Call container runtime to delete these nodes and add deleted nodes to this.deletedNodes.
280
- }
281
- }
282
- }
283
- else if (this.configs.tombstoneMode) {
284
- // The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
285
- // notify the runtime to update its state as well.
286
- this.tombstones = snapshotData?.tombstones ? Array.from(snapshotData.tombstones) : [];
287
- this.runtime.updateTombstonedRoutes(this.tombstones);
238
+ async initializeOrUpdateGCState() {
239
+ const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
240
+ if (currentReferenceTimestampMs === undefined) {
241
+ return;
288
242
  }
289
- // If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
290
- if (snapshotData?.gcState === undefined) {
291
- this.gcDataFromLastRun = undefined;
243
+ // If the GC state hasn't been initialized yet, initialize it and return.
244
+ if (this.gcDataFromLastRun === undefined) {
245
+ await this.initializeGCStateFromBaseSnapshotP;
292
246
  return;
293
247
  }
294
- // Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
295
- // to the GC data from the snapshot data.
296
- const gcNodes = {};
297
- for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
298
- if (nodeData.unreferencedTimestampMs !== undefined) {
299
- this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
300
- }
301
- gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
248
+ // If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
249
+ // reference timestamp.
250
+ for (const [, nodeStateTracker] of this.unreferencedNodesState) {
251
+ nodeStateTracker.updateTracking(currentReferenceTimestampMs);
302
252
  }
303
- this.gcDataFromLastRun = { gcNodes };
304
253
  }
305
254
  /**
306
255
  * Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
307
- * to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
256
+ * to initialize or update the unreference state tracking.
308
257
  * @param connected - Whether the runtime connected / disconnected.
309
258
  * @param clientId - The clientId of this runtime.
310
259
  */
311
260
  setConnectionState(connected, clientId) {
312
261
  /**
313
- * For all clients, initialize the base state when the container becomes active, i.e., it transitions
314
- * to "write" mode. This will ensure that the container's own join op is processed and there is a recent
315
- * reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
316
- * could affect the GC state will have been processed.
262
+ * When the client connects (or reconnects), attempt to initialize or update the GC state. This will keep
263
+ * the unreferenced state tracking updated as per the reference timestamp at the time of connection.
317
264
  *
318
- * If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
319
- * InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
320
- * the receiving summarizer client.
321
- *
322
- * Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
323
- * sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
265
+ * During GC initialization and during connections in read mode, it is possible that either no ops are
266
+ * processed or only trailing ops are processed. This means that the GC state is not initialized or initialized
267
+ * with an older reference timestamp. So, doing this on every connection will keep the unreferenced state
268
+ * tracking up-to-date.
324
269
  */
325
- if (this.activeConnection() && this.configs.shouldRunGC) {
326
- this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
270
+ if (connected && this.configs.shouldRunGC) {
271
+ this.initializeOrUpdateGCState().catch((error) => {
272
+ this.mc.logger.sendErrorEvent({
273
+ eventName: "GCInitializationOrUpdateFailed",
274
+ gcConfigs: JSON.stringify(this.configs),
275
+ }, error);
276
+ });
327
277
  }
328
278
  }
329
279
  /**
@@ -380,8 +330,11 @@ class GarbageCollector {
380
330
  const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
381
331
  event.end({
382
332
  ...gcStats,
383
- timestamp: currentReferenceTimestampMs,
384
- sweep: this.configs.shouldRunSweep,
333
+ details: {
334
+ timestamp: currentReferenceTimestampMs,
335
+ sweep: this.configs.shouldRunSweep,
336
+ tombstone: this.configs.throwOnTombstoneLoad,
337
+ },
385
338
  });
386
339
  /** Post-GC steps */
387
340
  // Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
@@ -874,9 +827,24 @@ class GarbageCollector {
874
827
  deletedDataStoreCount: 0,
875
828
  deletedAttachmentBlobCount: 0,
876
829
  };
830
+ // The runtime can't reliably identify the type of deleted nodes. So, get the type here. This should
831
+ // be good enough because the only types that participate in GC today are data stores, DDSes and blobs.
832
+ const getDeletedNodeType = (nodeId) => {
833
+ const pathParts = nodeId.split("/");
834
+ if (pathParts[1] === blobManager_1.BlobManager.basePath) {
835
+ return gcDefinitions_1.GCNodeType.Blob;
836
+ }
837
+ if (pathParts.length === 2) {
838
+ return gcDefinitions_1.GCNodeType.DataStore;
839
+ }
840
+ if (pathParts.length === 3) {
841
+ return gcDefinitions_1.GCNodeType.SubDataStore;
842
+ }
843
+ return gcDefinitions_1.GCNodeType.Other;
844
+ };
877
845
  for (const nodeId of deletedNodes) {
878
846
  sweepPhaseStats.deletedNodeCount++;
879
- const nodeType = this.runtime.getNodeType(nodeId);
847
+ const nodeType = getDeletedNodeType(nodeId);
880
848
  if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
881
849
  sweepPhaseStats.deletedDataStoreCount++;
882
850
  }