@fluidframework/container-runtime 2.0.0-internal.7.3.0 → 2.0.0-internal.8.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +50 -0
- package/api-extractor-lint.json +13 -0
- package/api-extractor.json +9 -1
- package/api-report/container-runtime.api.md +123 -123
- package/dist/blobManager.d.ts +4 -4
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js.map +1 -1
- package/dist/container-runtime-alpha.d.ts +1444 -0
- package/dist/container-runtime-beta.d.ts +292 -0
- package/dist/container-runtime-public.d.ts +292 -0
- package/dist/container-runtime-untrimmed.d.ts +1792 -0
- package/dist/containerRuntime.d.ts +36 -66
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +68 -104
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.js +0 -12
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts +1 -1
- package/dist/dataStoreRegistry.js +1 -1
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +10 -15
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +77 -40
- package/dist/dataStores.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +41 -13
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +215 -78
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +34 -37
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +121 -46
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +26 -18
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +18 -25
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +29 -45
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +0 -5
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +14 -42
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.d.ts +11 -5
- package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +43 -19
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +1 -1
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +4 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +15 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +16 -6
- package/dist/index.js.map +1 -1
- package/dist/messageTypes.d.ts +15 -7
- package/dist/messageTypes.d.ts.map +1 -1
- package/dist/messageTypes.js +6 -1
- package/dist/messageTypes.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +1 -0
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +1 -0
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +1 -1
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.d.ts +2 -2
- package/dist/summary/runWhileConnectedCoordinator.js +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/summarizer.d.ts +1 -13
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +1 -47
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +30 -30
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +10 -10
- package/dist/summary/summaryCollection.js +1 -1
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +3 -3
- package/dist/summary/summaryFormat.js.map +1 -1
- package/lib/blobManager.d.ts +4 -4
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js.map +1 -1
- package/lib/container-runtime-alpha.d.ts +1444 -0
- package/lib/container-runtime-beta.d.ts +292 -0
- package/lib/container-runtime-public.d.ts +292 -0
- package/lib/container-runtime-untrimmed.d.ts +1792 -0
- package/lib/containerRuntime.d.ts +36 -66
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +69 -104
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.js +0 -12
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +10 -15
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +80 -43
- package/lib/dataStores.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +41 -13
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +217 -80
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +37 -40
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +121 -46
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +25 -17
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +18 -25
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +27 -43
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +0 -5
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +15 -43
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.d.ts +11 -5
- package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +43 -19
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +1 -1
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +15 -3
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +16 -2
- package/lib/index.js.map +1 -1
- package/lib/messageTypes.d.ts +15 -7
- package/lib/messageTypes.d.ts.map +1 -1
- package/lib/messageTypes.js +6 -1
- package/lib/messageTypes.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +1 -0
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +1 -0
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +1 -1
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.d.ts +2 -2
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/summarizer.d.ts +1 -13
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +1 -47
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +30 -30
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +10 -10
- package/lib/summary/summaryCollection.js +1 -1
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +3 -3
- package/lib/summary/summaryFormat.js.map +1 -1
- package/package.json +46 -19
- package/src/blobManager.ts +5 -5
- package/src/containerRuntime.ts +93 -141
- package/src/dataStore.ts +1 -15
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +140 -69
- package/src/gc/garbageCollection.md +14 -15
- package/src/gc/garbageCollection.ts +256 -96
- package/src/gc/gcConfigs.ts +50 -52
- package/src/gc/gcDefinitions.ts +137 -52
- package/src/gc/gcHelpers.ts +31 -52
- package/src/gc/gcTelemetry.ts +16 -57
- package/src/gc/gcUnreferencedStateTracker.ts +61 -22
- package/src/gc/index.ts +6 -4
- package/src/index.ts +19 -2
- package/src/messageTypes.ts +19 -4
- package/src/opLifecycle/definitions.ts +1 -1
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +1 -0
- package/src/summary/orderedClientElection.ts +1 -1
- package/src/summary/runWhileConnectedCoordinator.ts +2 -2
- package/src/summary/summarizer.ts +2 -51
- package/src/summary/summarizerTypes.ts +30 -30
- package/src/summary/summaryCollection.ts +10 -10
- package/src/summary/summaryFormat.ts +3 -3
- package/dist/id-compressor/appendOnlySortedMap.d.ts +0 -124
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
- package/dist/id-compressor/appendOnlySortedMap.js +0 -318
- package/dist/id-compressor/appendOnlySortedMap.js.map +0 -1
- package/dist/id-compressor/finalSpace.d.ts +0 -29
- package/dist/id-compressor/finalSpace.d.ts.map +0 -1
- package/dist/id-compressor/finalSpace.js +0 -62
- package/dist/id-compressor/finalSpace.js.map +0 -1
- package/dist/id-compressor/idCompressor.d.ts +0 -54
- package/dist/id-compressor/idCompressor.d.ts.map +0 -1
- package/dist/id-compressor/idCompressor.js +0 -495
- package/dist/id-compressor/idCompressor.js.map +0 -1
- package/dist/id-compressor/identifiers.d.ts +0 -32
- package/dist/id-compressor/identifiers.d.ts.map +0 -1
- package/dist/id-compressor/identifiers.js +0 -15
- package/dist/id-compressor/identifiers.js.map +0 -1
- package/dist/id-compressor/index.d.ts +0 -13
- package/dist/id-compressor/index.d.ts.map +0 -1
- package/dist/id-compressor/index.js +0 -32
- package/dist/id-compressor/index.js.map +0 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +0 -22
- package/dist/id-compressor/persistanceUtilities.d.ts.map +0 -1
- package/dist/id-compressor/persistanceUtilities.js +0 -43
- package/dist/id-compressor/persistanceUtilities.js.map +0 -1
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionSpaceNormalizer.js +0 -80
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +0 -1
- package/dist/id-compressor/sessions.d.ts +0 -115
- package/dist/id-compressor/sessions.d.ts.map +0 -1
- package/dist/id-compressor/sessions.js +0 -305
- package/dist/id-compressor/sessions.js.map +0 -1
- package/dist/id-compressor/utilities.d.ts +0 -52
- package/dist/id-compressor/utilities.d.ts.map +0 -1
- package/dist/id-compressor/utilities.js +0 -169
- package/dist/id-compressor/utilities.js.map +0 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +0 -124
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +0 -1
- package/lib/id-compressor/appendOnlySortedMap.js +0 -314
- package/lib/id-compressor/appendOnlySortedMap.js.map +0 -1
- package/lib/id-compressor/finalSpace.d.ts +0 -29
- package/lib/id-compressor/finalSpace.d.ts.map +0 -1
- package/lib/id-compressor/finalSpace.js +0 -58
- package/lib/id-compressor/finalSpace.js.map +0 -1
- package/lib/id-compressor/idCompressor.d.ts +0 -54
- package/lib/id-compressor/idCompressor.d.ts.map +0 -1
- package/lib/id-compressor/idCompressor.js +0 -491
- package/lib/id-compressor/idCompressor.js.map +0 -1
- package/lib/id-compressor/identifiers.d.ts +0 -32
- package/lib/id-compressor/identifiers.d.ts.map +0 -1
- package/lib/id-compressor/identifiers.js +0 -11
- package/lib/id-compressor/identifiers.js.map +0 -1
- package/lib/id-compressor/index.d.ts +0 -13
- package/lib/id-compressor/index.d.ts.map +0 -1
- package/lib/id-compressor/index.js +0 -13
- package/lib/id-compressor/index.js.map +0 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +0 -22
- package/lib/id-compressor/persistanceUtilities.d.ts.map +0 -1
- package/lib/id-compressor/persistanceUtilities.js +0 -34
- package/lib/id-compressor/persistanceUtilities.js.map +0 -1
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +0 -46
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionSpaceNormalizer.js +0 -76
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +0 -1
- package/lib/id-compressor/sessions.d.ts +0 -115
- package/lib/id-compressor/sessions.d.ts.map +0 -1
- package/lib/id-compressor/sessions.js +0 -290
- package/lib/id-compressor/sessions.js.map +0 -1
- package/lib/id-compressor/utilities.d.ts +0 -52
- package/lib/id-compressor/utilities.d.ts.map +0 -1
- package/lib/id-compressor/utilities.js +0 -151
- package/lib/id-compressor/utilities.js.map +0 -1
- package/src/id-compressor/README.md +0 -3
- package/src/id-compressor/appendOnlySortedMap.ts +0 -366
- package/src/id-compressor/finalSpace.ts +0 -67
- package/src/id-compressor/idCompressor.ts +0 -630
- package/src/id-compressor/identifiers.ts +0 -42
- package/src/id-compressor/index.ts +0 -26
- package/src/id-compressor/persistanceUtilities.ts +0 -58
- package/src/id-compressor/sessionSpaceNormalizer.ts +0 -83
- package/src/id-compressor/sessions.ts +0 -405
- package/src/id-compressor/utilities.ts +0 -190
|
@@ -8,9 +8,10 @@ import { createResponseError, responseToException } from "@fluidframework/runtim
|
|
|
8
8
|
import { createChildLogger, createChildMonitoringContext, DataProcessingError, PerformanceEvent, } from "@fluidframework/telemetry-utils";
|
|
9
9
|
import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey, } from "../containerRuntime";
|
|
10
10
|
import { ClientSessionExpiredError } from "../error";
|
|
11
|
+
import { ContainerMessageType } from "../messageTypes";
|
|
11
12
|
import { generateGCConfigs } from "./gcConfigs";
|
|
12
|
-
import { GCNodeType, UnreferencedState, } from "./gcDefinitions";
|
|
13
|
-
import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot } from "./gcHelpers";
|
|
13
|
+
import { GCNodeType, UnreferencedState, GarbageCollectionMessageType, } from "./gcDefinitions";
|
|
14
|
+
import { cloneGCData, compatBehaviorAllowsGCMessageType, concatGarbageCollectionData, getGCDataFromSnapshot, } from "./gcHelpers";
|
|
14
15
|
import { runGarbageCollection } from "./gcReferenceGraphAlgorithm";
|
|
15
16
|
import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
|
|
16
17
|
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
@@ -46,7 +47,7 @@ export class GarbageCollector {
|
|
|
46
47
|
}
|
|
47
48
|
/** If false, loading or using a Tombstoned object should merely log, not fail */
|
|
48
49
|
get tombstoneEnforcementAllowed() {
|
|
49
|
-
return this.configs.
|
|
50
|
+
return this.configs.sweepEnabled;
|
|
50
51
|
}
|
|
51
52
|
/** If true, throw an error when a tombstone data store is retrieved */
|
|
52
53
|
get throwOnTombstoneLoad() {
|
|
@@ -80,6 +81,7 @@ export class GarbageCollector {
|
|
|
80
81
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
81
82
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
82
83
|
this.activeConnection = createParams.activeConnection;
|
|
84
|
+
this.submitMessage = createParams.submitMessage;
|
|
83
85
|
const baseSnapshot = createParams.baseSnapshot;
|
|
84
86
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
85
87
|
this.mc = createChildMonitoringContext({
|
|
@@ -141,7 +143,7 @@ export class GarbageCollector {
|
|
|
141
143
|
/**
|
|
142
144
|
* Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
|
|
143
145
|
* connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
|
|
144
|
-
* GC state and updates their inactive or sweep
|
|
146
|
+
* GC state and updates their inactive or sweep-ready state.
|
|
145
147
|
*/
|
|
146
148
|
this.initializeGCStateFromBaseSnapshotP = new LazyPromise(async () => {
|
|
147
149
|
/**
|
|
@@ -197,6 +199,7 @@ export class GarbageCollector {
|
|
|
197
199
|
eventName: "GarbageCollectorLoaded",
|
|
198
200
|
gcConfigs: JSON.stringify(this.configs),
|
|
199
201
|
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
202
|
+
...createParams.createContainerMetadata,
|
|
200
203
|
});
|
|
201
204
|
}
|
|
202
205
|
/**
|
|
@@ -290,7 +293,7 @@ export class GarbageCollector {
|
|
|
290
293
|
const gcNodes = {};
|
|
291
294
|
for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
|
|
292
295
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
293
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
296
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
|
|
294
297
|
}
|
|
295
298
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
296
299
|
}
|
|
@@ -298,7 +301,7 @@ export class GarbageCollector {
|
|
|
298
301
|
}
|
|
299
302
|
/**
|
|
300
303
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
301
|
-
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep
|
|
304
|
+
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep-ready nodes.
|
|
302
305
|
* @param connected - Whether the runtime connected / disconnected.
|
|
303
306
|
* @param clientId - The clientId of this runtime.
|
|
304
307
|
*/
|
|
@@ -314,7 +317,7 @@ export class GarbageCollector {
|
|
|
314
317
|
* the receiving summarizer client.
|
|
315
318
|
*
|
|
316
319
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
317
|
-
* sweep in phases and we want to track when inactive and sweep
|
|
320
|
+
* sweep in phases and we want to track when inactive and sweep-ready objects are used in any client.
|
|
318
321
|
*/
|
|
319
322
|
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
320
323
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
@@ -372,7 +375,11 @@ export class GarbageCollector {
|
|
|
372
375
|
await this.runtime.updateStateBeforeGC();
|
|
373
376
|
/** GC step */
|
|
374
377
|
const gcStats = await this.runGC(fullGC, currentReferenceTimestampMs, logger);
|
|
375
|
-
event.end({
|
|
378
|
+
event.end({
|
|
379
|
+
...gcStats,
|
|
380
|
+
timestamp: currentReferenceTimestampMs,
|
|
381
|
+
sweep: this.configs.shouldRunSweep,
|
|
382
|
+
});
|
|
376
383
|
/** Post-GC steps */
|
|
377
384
|
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
378
385
|
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
@@ -387,10 +394,16 @@ export class GarbageCollector {
|
|
|
387
394
|
}
|
|
388
395
|
/**
|
|
389
396
|
* Runs garbage collection. It does the following:
|
|
397
|
+
*
|
|
390
398
|
* 1. It generates / analyzes the runtime's reference graph.
|
|
391
|
-
*
|
|
399
|
+
*
|
|
400
|
+
* 2. Generates mark phase stats.
|
|
401
|
+
*
|
|
392
402
|
* 3. Runs Mark phase.
|
|
403
|
+
*
|
|
393
404
|
* 4. Runs Sweep phase.
|
|
405
|
+
*
|
|
406
|
+
* 5. Generates sweep phase stats.
|
|
394
407
|
*/
|
|
395
408
|
async runGC(fullGC, currentReferenceTimestampMs, logger) {
|
|
396
409
|
// 1. Generate / analyze the runtime's reference graph.
|
|
@@ -400,17 +413,20 @@ export class GarbageCollector {
|
|
|
400
413
|
// Get all referenced nodes - References in this run + references between the previous and current runs.
|
|
401
414
|
const allReferencedNodeIds = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger) ??
|
|
402
415
|
gcResult.referencedNodeIds;
|
|
403
|
-
// 2.
|
|
404
|
-
//
|
|
405
|
-
const
|
|
416
|
+
// 2. Get the mark phase stats based on the previous / current GC state.
|
|
417
|
+
// This is done before running mark phase because we need the previous GC state before it is updated.
|
|
418
|
+
const markPhaseStats = this.getMarkPhaseStats(gcResult);
|
|
406
419
|
// 3. Run the Mark phase.
|
|
407
|
-
// It will mark nodes as referenced / unreferenced and return
|
|
408
|
-
const sweepReadyNodeIds = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
|
|
420
|
+
// It will mark nodes as referenced / unreferenced and return lists of tombstone-ready and sweep-ready nodes.
|
|
421
|
+
const { tombstoneReadyNodeIds, sweepReadyNodeIds } = this.runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs);
|
|
409
422
|
// 4. Run the Sweep phase.
|
|
410
|
-
// It will
|
|
411
|
-
|
|
412
|
-
this.
|
|
413
|
-
|
|
423
|
+
// It will tombstone any tombstone-ready nodes, and initiate the deletion of sweep-ready nodes by sending a
|
|
424
|
+
// sweep op. All clients, including this one, will delete these nodes once it processes the op.
|
|
425
|
+
this.runSweepPhase(gcResult, tombstoneReadyNodeIds, sweepReadyNodeIds);
|
|
426
|
+
this.gcDataFromLastRun = cloneGCData(gcData);
|
|
427
|
+
// 5. Get the sweep phase stats.
|
|
428
|
+
const sweepPhaseStats = this.getSweepPhaseStats(this.deletedNodes, sweepReadyNodeIds, markPhaseStats);
|
|
429
|
+
return { ...markPhaseStats, ...sweepPhaseStats };
|
|
414
430
|
}
|
|
415
431
|
/**
|
|
416
432
|
* Runs the GC Mark phase. It does the following:
|
|
@@ -424,7 +440,7 @@ export class GarbageCollector {
|
|
|
424
440
|
* @param gcResult - The result of the GC run on the gcData.
|
|
425
441
|
* @param allReferencedNodeIds - Nodes referenced in this GC run + referenced between previous and current GC run.
|
|
426
442
|
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
427
|
-
* @returns
|
|
443
|
+
* @returns The sets of tombstone-ready and sweep-ready nodes, i.e., nodes that ready to be tombstoned or deleted.
|
|
428
444
|
*/
|
|
429
445
|
runMarkPhase(gcResult, allReferencedNodeIds, currentReferenceTimestampMs) {
|
|
430
446
|
// 1. Marks all referenced nodes by clearing their unreferenced tracker, if any.
|
|
@@ -438,83 +454,94 @@ export class GarbageCollector {
|
|
|
438
454
|
}
|
|
439
455
|
}
|
|
440
456
|
// 2. Mark unreferenced nodes in this run by starting unreferenced tracking for them.
|
|
441
|
-
const
|
|
457
|
+
const tombstoneReadyNodeIds = new Set();
|
|
458
|
+
const sweepReadyNodeIds = new Set();
|
|
442
459
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
443
460
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
444
461
|
if (nodeStateTracker === undefined) {
|
|
445
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
462
|
+
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs, this.configs.sweepGracePeriodMs));
|
|
446
463
|
}
|
|
447
464
|
else {
|
|
448
465
|
// If a node was already unreferenced, update its tracking information. Since the current reference time
|
|
449
466
|
// is from the ops seen, this will ensure that we keep updating unreferenced state as time moves forward.
|
|
450
467
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
451
|
-
// If a node is sweep
|
|
468
|
+
// If a node is tombstone or sweep-ready, store it so it can be returned.
|
|
469
|
+
if (nodeStateTracker.state === UnreferencedState.TombstoneReady) {
|
|
470
|
+
tombstoneReadyNodeIds.add(nodeId);
|
|
471
|
+
}
|
|
452
472
|
if (nodeStateTracker.state === UnreferencedState.SweepReady) {
|
|
453
|
-
sweepReadyNodeIds.
|
|
473
|
+
sweepReadyNodeIds.add(nodeId);
|
|
454
474
|
}
|
|
455
475
|
}
|
|
456
476
|
}
|
|
457
477
|
// 3. Call the runtime to update referenced nodes in this run.
|
|
458
478
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
459
|
-
return sweepReadyNodeIds;
|
|
479
|
+
return { tombstoneReadyNodeIds, sweepReadyNodeIds };
|
|
460
480
|
}
|
|
461
481
|
/**
|
|
462
482
|
* Runs the GC Sweep phase. It does the following:
|
|
463
|
-
*
|
|
464
|
-
*
|
|
483
|
+
*
|
|
484
|
+
* 1. Marks tombstone-ready nodes as tombstones.
|
|
485
|
+
*
|
|
486
|
+
* 2. Sends a sweep op to delete nodes that are sweep-ready. Once the op is ack'd, these nodes will be deleted.
|
|
465
487
|
*
|
|
466
488
|
* @param gcResult - The result of the GC run on the gcData.
|
|
467
|
-
* @param
|
|
468
|
-
* @param
|
|
469
|
-
* @param logger - The logger to be used to log any telemetry.
|
|
470
|
-
* @returns A list of nodes that have been deleted.
|
|
489
|
+
* @param tombstoneReadyNodes - List of nodes that are tombstone-ready.
|
|
490
|
+
* @param sweepReadyNodes - List of nodes that are sweep-ready.
|
|
471
491
|
*/
|
|
472
|
-
runSweepPhase(gcResult,
|
|
473
|
-
// Log events for objects that are ready to be deleted by sweep. This will give us data on sweep when
|
|
474
|
-
// its not enabled.
|
|
475
|
-
this.telemetryTracker.logSweepEvents(logger, currentReferenceTimestampMs, this.unreferencedNodesState, this.completedRuns, this.getLastSummaryTimestampMs());
|
|
492
|
+
runSweepPhase(gcResult, tombstoneReadyNodes, sweepReadyNodes) {
|
|
476
493
|
/**
|
|
477
|
-
*
|
|
478
|
-
* Test mode - Unreferenced nodes are immediately deleted without waiting for them to be sweep ready.
|
|
479
|
-
* Tombstone mode - Sweep ready modes are marked as tombstones instead of being deleted.
|
|
480
|
-
* Sweep mode - Sweep ready modes are deleted.
|
|
494
|
+
* Under "Test Mode", unreferenced nodes are immediately deleted without waiting for them to be sweep-ready.
|
|
481
495
|
*
|
|
482
|
-
*
|
|
483
|
-
*
|
|
496
|
+
* Otherwise, depending on how long it's been since the node was unreferenced, it will either be
|
|
497
|
+
* marked as Tombstone, or deleted by Sweep.
|
|
484
498
|
*/
|
|
485
499
|
if (this.configs.testMode) {
|
|
486
500
|
// If we are running in GC test mode, unreferenced nodes (gcResult.deletedNodeIds) are deleted.
|
|
487
501
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
488
|
-
return
|
|
502
|
+
return;
|
|
489
503
|
}
|
|
504
|
+
// If sweep is disabled, we'll tombstone both tombstone-ready and sweep-ready nodes.
|
|
505
|
+
// This is important because a container may never load during a node's Sweep Grace Period,
|
|
506
|
+
// so that node would directly become sweep-ready skipping over tombstone-ready state,
|
|
507
|
+
// but should be Tombstoned since Sweep is disabled.
|
|
508
|
+
const { nodesToTombstone, nodesToDelete } = this.configs.shouldRunSweep
|
|
509
|
+
? {
|
|
510
|
+
nodesToTombstone: [...tombstoneReadyNodes],
|
|
511
|
+
nodesToDelete: [...sweepReadyNodes],
|
|
512
|
+
}
|
|
513
|
+
: {
|
|
514
|
+
nodesToTombstone: [...tombstoneReadyNodes, ...sweepReadyNodes],
|
|
515
|
+
nodesToDelete: [],
|
|
516
|
+
};
|
|
490
517
|
if (this.configs.tombstoneMode) {
|
|
491
|
-
this.tombstones =
|
|
492
|
-
// If we are running in GC tombstone mode, update tombstoned routes.
|
|
493
|
-
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
518
|
+
this.tombstones = nodesToTombstone;
|
|
519
|
+
// If we are running in GC tombstone mode, update tombstoned routes.
|
|
494
520
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
495
|
-
return [];
|
|
496
|
-
}
|
|
497
|
-
if (!this.configs.shouldRunSweep) {
|
|
498
|
-
return [];
|
|
499
521
|
}
|
|
500
|
-
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
507
|
-
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
//
|
|
515
|
-
|
|
522
|
+
if (this.configs.shouldRunSweep && nodesToDelete.length > 0) {
|
|
523
|
+
// Do not send DDS node ids in the GC op. This is an optimization to reduce its size. Since GC applies to
|
|
524
|
+
// to data store only, all its DDSes are deleted along with it. The DDS ids will be retrieved from the
|
|
525
|
+
// local state when processing the op.
|
|
526
|
+
const sweepReadyDSAndBlobs = nodesToDelete.filter((nodeId) => {
|
|
527
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
528
|
+
return nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob;
|
|
529
|
+
});
|
|
530
|
+
const contents = {
|
|
531
|
+
type: GarbageCollectionMessageType.Sweep,
|
|
532
|
+
deletedNodeIds: sweepReadyDSAndBlobs,
|
|
533
|
+
};
|
|
534
|
+
// Its fine for older clients to ignore this op because it doesn't have any functional impact. This op
|
|
535
|
+
// is an optimization to ensure that all clients are in sync when it comes to deleted nodes to prevent their
|
|
536
|
+
// accidental usage. The clients will sync without the delete op too but it may take longer.
|
|
537
|
+
const containerGCMessage = {
|
|
538
|
+
type: ContainerMessageType.GC,
|
|
539
|
+
contents,
|
|
540
|
+
compatDetails: { behavior: "Ignore" },
|
|
541
|
+
};
|
|
542
|
+
this.submitMessage(containerGCMessage);
|
|
543
|
+
return;
|
|
516
544
|
}
|
|
517
|
-
return deletedNodeIds;
|
|
518
545
|
}
|
|
519
546
|
/**
|
|
520
547
|
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
@@ -624,6 +651,65 @@ export class GarbageCollector {
|
|
|
624
651
|
async refreshLatestSummary(result) {
|
|
625
652
|
return this.summaryStateTracker.refreshLatestSummary(result);
|
|
626
653
|
}
|
|
654
|
+
/**
|
|
655
|
+
* Process a GC message.
|
|
656
|
+
* @param message - The GC message from the container runtime.
|
|
657
|
+
* @param local - Whether it was send by this client.
|
|
658
|
+
*/
|
|
659
|
+
processMessage(message, local) {
|
|
660
|
+
switch (message.contents.type) {
|
|
661
|
+
case "Sweep": {
|
|
662
|
+
// Delete the nodes whose ids are present in the contents.
|
|
663
|
+
this.deleteSweepReadyNodes(message.contents.deletedNodeIds);
|
|
664
|
+
break;
|
|
665
|
+
}
|
|
666
|
+
default: {
|
|
667
|
+
if (!compatBehaviorAllowsGCMessageType(message.contents.type, message.compatDetails?.behavior)) {
|
|
668
|
+
const error = DataProcessingError.create(`Garbage collection message of unknown type ${message.contents.type}`, "processMessage");
|
|
669
|
+
throw error;
|
|
670
|
+
}
|
|
671
|
+
break;
|
|
672
|
+
}
|
|
673
|
+
}
|
|
674
|
+
}
|
|
675
|
+
/**
|
|
676
|
+
* Delete nodes that are sweep-ready. Call the runtime to delete these nodes and clear the unreferenced state
|
|
677
|
+
* tracking for nodes that are actually deleted by the runtime.
|
|
678
|
+
* @param sweepReadyNodeIds - The ids of nodes that are ready to be deleted.
|
|
679
|
+
*/
|
|
680
|
+
deleteSweepReadyNodes(sweepReadyNodeIds) {
|
|
681
|
+
// Use a set for lookup because its much faster than array or map.
|
|
682
|
+
const sweepReadyNodesSet = new Set(sweepReadyNodeIds);
|
|
683
|
+
// The ids in the sweep-ready nodes do not contain DDS node ids. This is an optimization to reduce the size
|
|
684
|
+
// of the GC op. Since GC applies to data store only, all its DDSes are deleted along with it. So, get the
|
|
685
|
+
// DDS nodes ID from the unreferenced nodes state.
|
|
686
|
+
const allSweepReadyNodeIds = Array.from(sweepReadyNodeIds);
|
|
687
|
+
for (const [id] of this.unreferencedNodesState) {
|
|
688
|
+
// Ignore data store nodes since they would already be in the list.
|
|
689
|
+
const pathParts = id.split("/");
|
|
690
|
+
if (pathParts.length <= 2) {
|
|
691
|
+
continue;
|
|
692
|
+
}
|
|
693
|
+
// Get the data store id part. Note that this may include blobs but that's okay since the part would just
|
|
694
|
+
// be "_blobs" and it won't be found.
|
|
695
|
+
const dsId = `/${pathParts[1]}`;
|
|
696
|
+
if (sweepReadyNodesSet.has(dsId)) {
|
|
697
|
+
allSweepReadyNodeIds.push(id);
|
|
698
|
+
}
|
|
699
|
+
}
|
|
700
|
+
const deletedNodeIds = this.runtime.deleteSweepReadyNodes(allSweepReadyNodeIds);
|
|
701
|
+
// Clear unreferenced state tracking for deleted nodes.
|
|
702
|
+
for (const nodeId of deletedNodeIds) {
|
|
703
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
704
|
+
if (nodeStateTracker !== undefined) {
|
|
705
|
+
// Stop tracking so as to clear out any running timers.
|
|
706
|
+
nodeStateTracker.stopTracking();
|
|
707
|
+
// Delete the node as we don't need to track it any more.
|
|
708
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
709
|
+
}
|
|
710
|
+
this.deletedNodes.add(nodeId);
|
|
711
|
+
}
|
|
712
|
+
}
|
|
627
713
|
/**
|
|
628
714
|
* Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
|
|
629
715
|
* or throw an error if failing on incorrect usage is configured.
|
|
@@ -712,12 +798,12 @@ export class GarbageCollector {
|
|
|
712
798
|
this.sessionExpiryTimer = undefined;
|
|
713
799
|
}
|
|
714
800
|
/**
|
|
715
|
-
* Generates the stats of a garbage collection
|
|
716
|
-
* @param gcResult - The result of
|
|
717
|
-
* @returns the
|
|
801
|
+
* Generates the stats of a garbage collection mark phase run.
|
|
802
|
+
* @param gcResult - The result of the current GC run.
|
|
803
|
+
* @returns the stats of the mark phase run.
|
|
718
804
|
*/
|
|
719
|
-
|
|
720
|
-
const
|
|
805
|
+
getMarkPhaseStats(gcResult) {
|
|
806
|
+
const markPhaseStats = {
|
|
721
807
|
nodeCount: 0,
|
|
722
808
|
dataStoreCount: 0,
|
|
723
809
|
attachmentBlobCount: 0,
|
|
@@ -729,33 +815,33 @@ export class GarbageCollector {
|
|
|
729
815
|
updatedAttachmentBlobCount: 0,
|
|
730
816
|
};
|
|
731
817
|
const updateNodeStats = (nodeId, referenced) => {
|
|
732
|
-
|
|
818
|
+
markPhaseStats.nodeCount++;
|
|
733
819
|
// If there is no previous GC data, every node's state is generated and is considered as updated.
|
|
734
820
|
// Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
|
|
735
821
|
const stateUpdated = this.gcDataFromLastRun === undefined ||
|
|
736
822
|
this.unreferencedNodesState.has(nodeId) === referenced;
|
|
737
823
|
if (stateUpdated) {
|
|
738
|
-
|
|
824
|
+
markPhaseStats.updatedNodeCount++;
|
|
739
825
|
}
|
|
740
826
|
if (!referenced) {
|
|
741
|
-
|
|
827
|
+
markPhaseStats.unrefNodeCount++;
|
|
742
828
|
}
|
|
743
829
|
if (this.runtime.getNodeType(nodeId) === GCNodeType.DataStore) {
|
|
744
|
-
|
|
830
|
+
markPhaseStats.dataStoreCount++;
|
|
745
831
|
if (stateUpdated) {
|
|
746
|
-
|
|
832
|
+
markPhaseStats.updatedDataStoreCount++;
|
|
747
833
|
}
|
|
748
834
|
if (!referenced) {
|
|
749
|
-
|
|
835
|
+
markPhaseStats.unrefDataStoreCount++;
|
|
750
836
|
}
|
|
751
837
|
}
|
|
752
838
|
if (this.runtime.getNodeType(nodeId) === GCNodeType.Blob) {
|
|
753
|
-
|
|
839
|
+
markPhaseStats.attachmentBlobCount++;
|
|
754
840
|
if (stateUpdated) {
|
|
755
|
-
|
|
841
|
+
markPhaseStats.updatedAttachmentBlobCount++;
|
|
756
842
|
}
|
|
757
843
|
if (!referenced) {
|
|
758
|
-
|
|
844
|
+
markPhaseStats.unrefAttachmentBlobCount++;
|
|
759
845
|
}
|
|
760
846
|
}
|
|
761
847
|
};
|
|
@@ -765,7 +851,58 @@ export class GarbageCollector {
|
|
|
765
851
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
766
852
|
updateNodeStats(nodeId, false /* referenced */);
|
|
767
853
|
}
|
|
768
|
-
return
|
|
854
|
+
return markPhaseStats;
|
|
855
|
+
}
|
|
856
|
+
/**
|
|
857
|
+
* Generates the stats of a garbage collection sweep phase run.
|
|
858
|
+
* @param deletedNodes - The nodes that have been deleted until this run.
|
|
859
|
+
* @param sweepReadyNodes - The nodes that are sweep-ready in this GC run.
|
|
860
|
+
* @param markPhaseStats - The stats of the mark phase run.
|
|
861
|
+
* @returns the stats of the sweep phase run.
|
|
862
|
+
*/
|
|
863
|
+
getSweepPhaseStats(deletedNodes, sweepReadyNodes, markPhaseStats) {
|
|
864
|
+
// Initialize the life time node counts to the mark phase node counts. If sweep is not enabled,
|
|
865
|
+
// these will be the life time node count for this container.
|
|
866
|
+
const sweepPhaseStats = {
|
|
867
|
+
lifetimeNodeCount: markPhaseStats.nodeCount,
|
|
868
|
+
lifetimeDataStoreCount: markPhaseStats.dataStoreCount,
|
|
869
|
+
lifetimeAttachmentBlobCount: markPhaseStats.attachmentBlobCount,
|
|
870
|
+
deletedNodeCount: 0,
|
|
871
|
+
deletedDataStoreCount: 0,
|
|
872
|
+
deletedAttachmentBlobCount: 0,
|
|
873
|
+
};
|
|
874
|
+
for (const nodeId of deletedNodes) {
|
|
875
|
+
sweepPhaseStats.deletedNodeCount++;
|
|
876
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
877
|
+
if (nodeType === GCNodeType.DataStore) {
|
|
878
|
+
sweepPhaseStats.deletedDataStoreCount++;
|
|
879
|
+
}
|
|
880
|
+
else if (nodeType === GCNodeType.Blob) {
|
|
881
|
+
sweepPhaseStats.deletedAttachmentBlobCount++;
|
|
882
|
+
}
|
|
883
|
+
}
|
|
884
|
+
// If sweep is enabled, the counts from the mark phase stats do not include nodes that have been
|
|
885
|
+
// deleted in previous runs. So, add the deleted node counts to life time stats.
|
|
886
|
+
sweepPhaseStats.lifetimeNodeCount += sweepPhaseStats.deletedNodeCount;
|
|
887
|
+
sweepPhaseStats.lifetimeDataStoreCount += sweepPhaseStats.deletedDataStoreCount;
|
|
888
|
+
sweepPhaseStats.lifetimeAttachmentBlobCount += sweepPhaseStats.deletedAttachmentBlobCount;
|
|
889
|
+
if (this.configs.shouldRunSweep) {
|
|
890
|
+
return sweepPhaseStats;
|
|
891
|
+
}
|
|
892
|
+
// If sweep is not enabled, the current sweep-ready node stats should be added to deleted stats since this
|
|
893
|
+
// is the final state the node will be in.
|
|
894
|
+
// If sweep is enabled, this will happen in the run after the GC op round trips back.
|
|
895
|
+
for (const nodeId of sweepReadyNodes) {
|
|
896
|
+
sweepPhaseStats.deletedNodeCount++;
|
|
897
|
+
const nodeType = this.runtime.getNodeType(nodeId);
|
|
898
|
+
if (nodeType === GCNodeType.DataStore) {
|
|
899
|
+
sweepPhaseStats.deletedDataStoreCount++;
|
|
900
|
+
}
|
|
901
|
+
else if (nodeType === GCNodeType.Blob) {
|
|
902
|
+
sweepPhaseStats.deletedAttachmentBlobCount++;
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
return sweepPhaseStats;
|
|
769
906
|
}
|
|
770
907
|
}
|
|
771
908
|
//# sourceMappingURL=garbageCollection.js.map
|