@fluidframework/container-runtime 2.0.0-dev.3.1.0.125672 → 2.0.0-dev.4.2.0.153917
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +58 -0
- package/README.md +69 -0
- package/dist/blobManager.d.ts +29 -24
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +162 -92
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +74 -76
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +328 -264
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +39 -13
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +112 -49
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +28 -4
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +107 -41
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/dist/deltaManagerSummarizerProxy.js +40 -0
- package/dist/deltaManagerSummarizerProxy.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +204 -0
- package/dist/gc/garbageCollection.d.ts.map +1 -0
- package/dist/{garbageCollection.js → gc/garbageCollection.js} +190 -554
- package/dist/gc/garbageCollection.js.map +1 -0
- package/dist/gc/gcConfigs.d.ts +22 -0
- package/dist/gc/gcConfigs.d.ts.map +1 -0
- package/dist/gc/gcConfigs.js +143 -0
- package/dist/gc/gcConfigs.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +320 -0
- package/dist/gc/gcDefinitions.d.ts.map +1 -0
- package/dist/gc/gcDefinitions.js +81 -0
- package/dist/gc/gcDefinitions.js.map +1 -0
- package/dist/gc/gcHelpers.d.ts +86 -0
- package/dist/gc/gcHelpers.d.ts.map +1 -0
- package/dist/gc/gcHelpers.js +268 -0
- package/dist/gc/gcHelpers.js.map +1 -0
- package/dist/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
- package/dist/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
- package/dist/gc/gcReferenceGraphAlgorithm.js +49 -0
- package/dist/gc/gcReferenceGraphAlgorithm.js.map +1 -0
- package/dist/gc/gcSummaryDefinitions.d.ts +52 -0
- package/dist/gc/gcSummaryDefinitions.d.ts.map +1 -0
- package/dist/gc/gcSummaryDefinitions.js +7 -0
- package/dist/gc/gcSummaryDefinitions.js.map +1 -0
- package/dist/gc/gcSummaryStateTracker.d.ts +93 -0
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/dist/gc/gcSummaryStateTracker.js +239 -0
- package/dist/gc/gcSummaryStateTracker.js.map +1 -0
- package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/dist/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +2 -2
- package/dist/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.js +94 -0
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/dist/gc/index.d.ts +13 -0
- package/dist/gc/index.d.ts.map +1 -0
- package/dist/gc/index.js +50 -0
- package/dist/gc/index.js.map +1 -0
- package/dist/index.d.ts +3 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -9
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +11 -13
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +26 -38
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +4 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +2 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +4 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +25 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +4 -0
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +43 -4
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +14 -0
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -0
- package/dist/opLifecycle/opGroupingManager.js +56 -0
- package/dist/opLifecycle/opGroupingManager.js.map +1 -0
- package/dist/opLifecycle/opSplitter.d.ts +16 -4
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +39 -15
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +21 -3
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +90 -51
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +4 -2
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +30 -20
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +3 -3
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +20 -21
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/dist/storageServiceWithAttachBlobs.js +32 -0
- package/dist/storageServiceWithAttachBlobs.js.map +1 -0
- package/dist/summary/index.d.ts +17 -0
- package/dist/summary/index.d.ts.map +1 -0
- package/dist/summary/index.js +48 -0
- package/dist/summary/index.js.map +1 -0
- package/dist/summary/orderedClientElection.d.ts.map +1 -0
- package/dist/summary/orderedClientElection.js.map +1 -0
- package/dist/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +3 -2
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/dist/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{lib → dist/summary}/runningSummarizer.d.ts +23 -20
- package/dist/summary/runningSummarizer.d.ts.map +1 -0
- package/dist/{runningSummarizer.js → summary/runningSummarizer.js} +191 -74
- package/dist/summary/runningSummarizer.js.map +1 -0
- package/dist/{summarizer.d.ts → summary/summarizer.d.ts} +4 -9
- package/dist/summary/summarizer.d.ts.map +1 -0
- package/dist/{summarizer.js → summary/summarizer.js} +10 -79
- package/dist/summary/summarizer.js.map +1 -0
- package/dist/summary/summarizerClientElection.d.ts.map +1 -0
- package/dist/summary/summarizerClientElection.js.map +1 -0
- package/dist/{summarizerHeuristics.d.ts → summary/summarizerHeuristics.d.ts} +2 -1
- package/dist/summary/summarizerHeuristics.d.ts.map +1 -0
- package/dist/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -3
- package/dist/summary/summarizerHeuristics.js.map +1 -0
- package/dist/summary/summarizerNode/index.d.ts +8 -0
- package/dist/summary/summarizerNode/index.d.ts.map +1 -0
- package/dist/summary/summarizerNode/index.js +12 -0
- package/dist/summary/summarizerNode/index.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNode.d.ts +149 -0
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNode.js +531 -0
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +132 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +148 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +424 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
- package/{lib → dist/summary}/summarizerTypes.d.ts +21 -19
- package/dist/summary/summarizerTypes.d.ts.map +1 -0
- package/dist/{summarizerTypes.js → summary/summarizerTypes.js} +0 -5
- package/dist/summary/summarizerTypes.js.map +1 -0
- package/dist/summary/summaryCollection.d.ts.map +1 -0
- package/dist/summary/summaryCollection.js.map +1 -0
- package/{lib → dist/summary}/summaryFormat.d.ts +3 -21
- package/dist/summary/summaryFormat.d.ts.map +1 -0
- package/dist/{summaryFormat.js → summary/summaryFormat.js} +1 -10
- package/dist/summary/summaryFormat.js.map +1 -0
- package/{lib → dist/summary}/summaryGenerator.d.ts +28 -2
- package/dist/summary/summaryGenerator.d.ts.map +1 -0
- package/dist/{summaryGenerator.js → summary/summaryGenerator.js} +23 -20
- package/dist/summary/summaryGenerator.js.map +1 -0
- package/dist/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/dist/summary/summaryManager.d.ts.map +1 -0
- package/dist/summary/summaryManager.js.map +1 -0
- package/lib/blobManager.d.ts +29 -24
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +159 -89
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +74 -76
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +301 -237
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +39 -13
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +101 -38
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +28 -4
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +100 -34
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/lib/deltaManagerSummarizerProxy.js +36 -0
- package/lib/deltaManagerSummarizerProxy.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +204 -0
- package/lib/gc/garbageCollection.d.ts.map +1 -0
- package/lib/{garbageCollection.js → gc/garbageCollection.js} +172 -535
- package/lib/gc/garbageCollection.js.map +1 -0
- package/lib/gc/gcConfigs.d.ts +22 -0
- package/lib/gc/gcConfigs.d.ts.map +1 -0
- package/lib/gc/gcConfigs.js +139 -0
- package/lib/gc/gcConfigs.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +320 -0
- package/lib/gc/gcDefinitions.d.ts.map +1 -0
- package/lib/gc/gcDefinitions.js +78 -0
- package/lib/gc/gcDefinitions.js.map +1 -0
- package/lib/gc/gcHelpers.d.ts +86 -0
- package/lib/gc/gcHelpers.d.ts.map +1 -0
- package/lib/gc/gcHelpers.js +254 -0
- package/lib/gc/gcHelpers.js.map +1 -0
- package/lib/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
- package/lib/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
- package/lib/gc/gcReferenceGraphAlgorithm.js +45 -0
- package/lib/gc/gcReferenceGraphAlgorithm.js.map +1 -0
- package/lib/gc/gcSummaryDefinitions.d.ts +52 -0
- package/lib/gc/gcSummaryDefinitions.d.ts.map +1 -0
- package/lib/gc/gcSummaryDefinitions.js +6 -0
- package/lib/gc/gcSummaryDefinitions.js.map +1 -0
- package/lib/gc/gcSummaryStateTracker.d.ts +93 -0
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/lib/gc/gcSummaryStateTracker.js +235 -0
- package/lib/gc/gcSummaryStateTracker.js.map +1 -0
- package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/lib/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +1 -1
- package/lib/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.js +90 -0
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/lib/gc/index.d.ts +13 -0
- package/lib/gc/index.d.ts.map +1 -0
- package/lib/gc/index.js +12 -0
- package/lib/gc/index.js.map +1 -0
- package/lib/index.d.ts +3 -7
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -4
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +11 -13
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +24 -37
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +4 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +2 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +2 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +26 -11
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +4 -0
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +43 -4
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +14 -0
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -0
- package/lib/opLifecycle/opGroupingManager.js +52 -0
- package/lib/opLifecycle/opGroupingManager.js.map +1 -0
- package/lib/opLifecycle/opSplitter.d.ts +16 -4
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +39 -15
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +21 -3
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +92 -53
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +4 -2
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +30 -20
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +3 -3
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +20 -21
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/lib/storageServiceWithAttachBlobs.js +28 -0
- package/lib/storageServiceWithAttachBlobs.js.map +1 -0
- package/lib/summary/index.d.ts +17 -0
- package/lib/summary/index.d.ts.map +1 -0
- package/lib/summary/index.js +16 -0
- package/lib/summary/index.js.map +1 -0
- package/lib/summary/orderedClientElection.d.ts.map +1 -0
- package/lib/summary/orderedClientElection.js.map +1 -0
- package/lib/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +3 -2
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/lib/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{dist → lib/summary}/runningSummarizer.d.ts +23 -20
- package/lib/summary/runningSummarizer.d.ts.map +1 -0
- package/lib/{runningSummarizer.js → summary/runningSummarizer.js} +192 -75
- package/lib/summary/runningSummarizer.js.map +1 -0
- package/lib/{summarizer.d.ts → summary/summarizer.d.ts} +4 -9
- package/lib/summary/summarizer.d.ts.map +1 -0
- package/lib/{summarizer.js → summary/summarizer.js} +12 -81
- package/lib/summary/summarizer.js.map +1 -0
- package/lib/summary/summarizerClientElection.d.ts.map +1 -0
- package/lib/summary/summarizerClientElection.js.map +1 -0
- package/lib/{summarizerHeuristics.d.ts → summary/summarizerHeuristics.d.ts} +2 -1
- package/lib/summary/summarizerHeuristics.d.ts.map +1 -0
- package/lib/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -3
- package/lib/summary/summarizerHeuristics.js.map +1 -0
- package/lib/summary/summarizerNode/index.d.ts +8 -0
- package/lib/summary/summarizerNode/index.d.ts.map +1 -0
- package/lib/summary/summarizerNode/index.js +7 -0
- package/lib/summary/summarizerNode/index.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNode.d.ts +149 -0
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNode.js +526 -0
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +125 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +148 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +419 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
- package/{dist → lib/summary}/summarizerTypes.d.ts +21 -19
- package/lib/summary/summarizerTypes.d.ts.map +1 -0
- package/lib/summary/summarizerTypes.js +6 -0
- package/lib/summary/summarizerTypes.js.map +1 -0
- package/lib/summary/summaryCollection.d.ts.map +1 -0
- package/lib/summary/summaryCollection.js.map +1 -0
- package/{dist → lib/summary}/summaryFormat.d.ts +3 -21
- package/lib/summary/summaryFormat.d.ts.map +1 -0
- package/lib/{summaryFormat.js → summary/summaryFormat.js} +0 -8
- package/lib/summary/summaryFormat.js.map +1 -0
- package/{dist → lib/summary}/summaryGenerator.d.ts +28 -2
- package/lib/summary/summaryGenerator.d.ts.map +1 -0
- package/lib/{summaryGenerator.js → summary/summaryGenerator.js} +21 -19
- package/lib/summary/summaryGenerator.js.map +1 -0
- package/lib/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/lib/summary/summaryManager.d.ts.map +1 -0
- package/lib/summary/summaryManager.js.map +1 -0
- package/package.json +66 -60
- package/src/blobManager.ts +196 -110
- package/src/containerRuntime.ts +491 -391
- package/src/dataStoreContext.ts +140 -49
- package/src/dataStores.ts +139 -41
- package/src/deltaManagerSummarizerProxy.ts +46 -0
- package/{garbageCollection.md → src/gc/garbageCollection.md} +2 -2
- package/src/{garbageCollection.ts → gc/garbageCollection.ts} +245 -890
- package/src/gc/gcConfigs.ts +193 -0
- package/src/gc/gcDefinitions.ts +387 -0
- package/src/gc/gcHelpers.ts +335 -0
- package/src/gc/gcReferenceGraphAlgorithm.ts +52 -0
- package/src/gc/gcSummaryDefinitions.ts +54 -0
- package/src/gc/gcSummaryStateTracker.ts +329 -0
- package/src/{gcSweepReadyUsageDetection.ts → gc/gcSweepReadyUsageDetection.ts} +1 -1
- package/src/gc/gcUnreferencedStateTracker.ts +114 -0
- package/src/gc/index.ts +65 -0
- package/src/index.ts +10 -22
- package/src/opLifecycle/README.md +263 -0
- package/src/opLifecycle/batchManager.ts +26 -55
- package/src/opLifecycle/definitions.ts +4 -0
- package/src/opLifecycle/index.ts +2 -1
- package/src/opLifecycle/opCompressor.ts +32 -12
- package/src/opLifecycle/opDecompressor.ts +50 -5
- package/src/opLifecycle/opGroupingManager.ts +78 -0
- package/src/opLifecycle/opSplitter.ts +56 -17
- package/src/opLifecycle/outbox.ts +126 -62
- package/src/opLifecycle/remoteMessageProcessor.ts +38 -22
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +34 -27
- package/src/storageServiceWithAttachBlobs.ts +38 -0
- package/src/summary/index.ts +105 -0
- package/src/{runWhileConnectedCoordinator.ts → summary/runWhileConnectedCoordinator.ts} +7 -7
- package/src/{runningSummarizer.ts → summary/runningSummarizer.ts} +318 -156
- package/src/{summarizer.ts → summary/summarizer.ts} +12 -105
- package/src/{summarizerHeuristics.ts → summary/summarizerHeuristics.ts} +13 -4
- package/src/summary/summarizerNode/index.ts +12 -0
- package/src/summary/summarizerNode/summarizerNode.ts +766 -0
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +214 -0
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +644 -0
- package/src/{summarizerTypes.ts → summary/summarizerTypes.ts} +28 -25
- package/src/{summaryFormat.ts → summary/summaryFormat.ts} +3 -29
- package/src/{summaryGenerator.ts → summary/summaryGenerator.ts} +34 -27
- package/src/{summaryManager.ts → summary/summaryManager.ts} +1 -1
- package/dist/garbageCollection.d.ts +0 -411
- package/dist/garbageCollection.d.ts.map +0 -1
- package/dist/garbageCollection.js.map +0 -1
- package/dist/garbageCollectionConstants.d.ts +0 -23
- package/dist/garbageCollectionConstants.d.ts.map +0 -1
- package/dist/garbageCollectionConstants.js +0 -36
- package/dist/garbageCollectionConstants.js.map +0 -1
- package/dist/garbageCollectionHelpers.d.ts +0 -15
- package/dist/garbageCollectionHelpers.d.ts.map +0 -1
- package/dist/garbageCollectionHelpers.js +0 -27
- package/dist/garbageCollectionHelpers.js.map +0 -1
- package/dist/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/dist/gcSweepReadyUsageDetection.js.map +0 -1
- package/dist/orderedClientElection.d.ts.map +0 -1
- package/dist/orderedClientElection.js.map +0 -1
- package/dist/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/dist/runWhileConnectedCoordinator.js.map +0 -1
- package/dist/runningSummarizer.d.ts.map +0 -1
- package/dist/runningSummarizer.js.map +0 -1
- package/dist/serializedSnapshotStorage.d.ts +0 -58
- package/dist/serializedSnapshotStorage.d.ts.map +0 -1
- package/dist/serializedSnapshotStorage.js +0 -110
- package/dist/serializedSnapshotStorage.js.map +0 -1
- package/dist/summarizer.d.ts.map +0 -1
- package/dist/summarizer.js.map +0 -1
- package/dist/summarizerClientElection.d.ts.map +0 -1
- package/dist/summarizerClientElection.js.map +0 -1
- package/dist/summarizerHandle.d.ts +0 -12
- package/dist/summarizerHandle.d.ts.map +0 -1
- package/dist/summarizerHandle.js +0 -22
- package/dist/summarizerHandle.js.map +0 -1
- package/dist/summarizerHeuristics.d.ts.map +0 -1
- package/dist/summarizerHeuristics.js.map +0 -1
- package/dist/summarizerTypes.d.ts.map +0 -1
- package/dist/summarizerTypes.js.map +0 -1
- package/dist/summaryCollection.d.ts.map +0 -1
- package/dist/summaryCollection.js.map +0 -1
- package/dist/summaryFormat.d.ts.map +0 -1
- package/dist/summaryFormat.js.map +0 -1
- package/dist/summaryGenerator.d.ts.map +0 -1
- package/dist/summaryGenerator.js.map +0 -1
- package/dist/summaryManager.d.ts.map +0 -1
- package/dist/summaryManager.js.map +0 -1
- package/lib/garbageCollection.d.ts +0 -411
- package/lib/garbageCollection.d.ts.map +0 -1
- package/lib/garbageCollection.js.map +0 -1
- package/lib/garbageCollectionConstants.d.ts +0 -23
- package/lib/garbageCollectionConstants.d.ts.map +0 -1
- package/lib/garbageCollectionConstants.js +0 -33
- package/lib/garbageCollectionConstants.js.map +0 -1
- package/lib/garbageCollectionHelpers.d.ts +0 -15
- package/lib/garbageCollectionHelpers.d.ts.map +0 -1
- package/lib/garbageCollectionHelpers.js +0 -23
- package/lib/garbageCollectionHelpers.js.map +0 -1
- package/lib/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/lib/gcSweepReadyUsageDetection.js.map +0 -1
- package/lib/orderedClientElection.d.ts.map +0 -1
- package/lib/orderedClientElection.js.map +0 -1
- package/lib/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/lib/runWhileConnectedCoordinator.js.map +0 -1
- package/lib/runningSummarizer.d.ts.map +0 -1
- package/lib/runningSummarizer.js.map +0 -1
- package/lib/serializedSnapshotStorage.d.ts +0 -58
- package/lib/serializedSnapshotStorage.d.ts.map +0 -1
- package/lib/serializedSnapshotStorage.js +0 -106
- package/lib/serializedSnapshotStorage.js.map +0 -1
- package/lib/summarizer.d.ts.map +0 -1
- package/lib/summarizer.js.map +0 -1
- package/lib/summarizerClientElection.d.ts.map +0 -1
- package/lib/summarizerClientElection.js.map +0 -1
- package/lib/summarizerHandle.d.ts +0 -12
- package/lib/summarizerHandle.d.ts.map +0 -1
- package/lib/summarizerHandle.js +0 -18
- package/lib/summarizerHandle.js.map +0 -1
- package/lib/summarizerHeuristics.d.ts.map +0 -1
- package/lib/summarizerHeuristics.js.map +0 -1
- package/lib/summarizerTypes.d.ts.map +0 -1
- package/lib/summarizerTypes.js +0 -9
- package/lib/summarizerTypes.js.map +0 -1
- package/lib/summaryCollection.d.ts.map +0 -1
- package/lib/summaryCollection.js.map +0 -1
- package/lib/summaryFormat.d.ts.map +0 -1
- package/lib/summaryFormat.js.map +0 -1
- package/lib/summaryGenerator.d.ts.map +0 -1
- package/lib/summaryGenerator.js.map +0 -1
- package/lib/summaryManager.d.ts.map +0 -1
- package/lib/summaryManager.js.map +0 -1
- package/src/garbageCollectionConstants.ts +0 -38
- package/src/garbageCollectionHelpers.ts +0 -37
- package/src/serializedSnapshotStorage.ts +0 -151
- package/src/summarizerHandle.ts +0 -23
- /package/dist/{gcSweepReadyUsageDetection.d.ts → gc/gcSweepReadyUsageDetection.d.ts} +0 -0
- /package/dist/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/dist/{orderedClientElection.js → summary/orderedClientElection.js} +0 -0
- /package/dist/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
- /package/dist/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
- /package/dist/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/dist/{summaryCollection.js → summary/summaryCollection.js} +0 -0
- /package/dist/{summaryManager.js → summary/summaryManager.js} +0 -0
- /package/lib/{gcSweepReadyUsageDetection.d.ts → gc/gcSweepReadyUsageDetection.d.ts} +0 -0
- /package/lib/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/lib/{orderedClientElection.js → summary/orderedClientElection.js} +0 -0
- /package/lib/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
- /package/lib/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
- /package/lib/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/lib/{summaryCollection.js → summary/summaryCollection.js} +0 -0
- /package/lib/{summaryManager.js → summary/summaryManager.js} +0 -0
- /package/src/{orderedClientElection.ts → summary/orderedClientElection.ts} +0 -0
- /package/src/{summarizerClientElection.ts → summary/summarizerClientElection.ts} +0 -0
- /package/src/{summaryCollection.ts → summary/summaryCollection.ts} +0 -0
|
@@ -15,108 +15,19 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
15
15
|
return t;
|
|
16
16
|
};
|
|
17
17
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
exports.GarbageCollector =
|
|
18
|
+
exports.GarbageCollector = void 0;
|
|
19
19
|
const common_utils_1 = require("@fluidframework/common-utils");
|
|
20
20
|
const container_utils_1 = require("@fluidframework/container-utils");
|
|
21
|
-
const garbage_collector_1 = require("@fluidframework/garbage-collector");
|
|
22
|
-
const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
23
21
|
const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
24
22
|
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
25
23
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
26
|
-
const containerRuntime_1 = require("
|
|
27
|
-
const
|
|
28
|
-
const
|
|
29
|
-
const
|
|
30
|
-
const
|
|
31
|
-
const
|
|
32
|
-
|
|
33
|
-
exports.GCNodeType = {
|
|
34
|
-
// Nodes that are for data stores.
|
|
35
|
-
DataStore: "DataStore",
|
|
36
|
-
// Nodes that are within a data store. For example, DDS nodes.
|
|
37
|
-
SubDataStore: "SubDataStore",
|
|
38
|
-
// Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
|
|
39
|
-
Blob: "Blob",
|
|
40
|
-
// Nodes that are neither of the above. For example, root node.
|
|
41
|
-
Other: "Other",
|
|
42
|
-
};
|
|
43
|
-
/** The state of node that is unreferenced. */
|
|
44
|
-
exports.UnreferencedState = {
|
|
45
|
-
/** The node is active, i.e., it can become referenced again. */
|
|
46
|
-
Active: "Active",
|
|
47
|
-
/** The node is inactive, i.e., it should not become referenced. */
|
|
48
|
-
Inactive: "Inactive",
|
|
49
|
-
/** The node is ready to be deleted by the sweep phase. */
|
|
50
|
-
SweepReady: "SweepReady",
|
|
51
|
-
};
|
|
52
|
-
/**
|
|
53
|
-
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
|
|
54
|
-
* be deleted by the sweep phase.
|
|
55
|
-
*/
|
|
56
|
-
class UnreferencedStateTracker {
|
|
57
|
-
constructor(unreferencedTimestampMs,
|
|
58
|
-
/** The time after which node transitions to Inactive state. */
|
|
59
|
-
inactiveTimeoutMs,
|
|
60
|
-
/** The current reference timestamp used to track how long this node has been unreferenced for. */
|
|
61
|
-
currentReferenceTimestampMs,
|
|
62
|
-
/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
|
|
63
|
-
sweepTimeoutMs) {
|
|
64
|
-
this.unreferencedTimestampMs = unreferencedTimestampMs;
|
|
65
|
-
this.inactiveTimeoutMs = inactiveTimeoutMs;
|
|
66
|
-
this.sweepTimeoutMs = sweepTimeoutMs;
|
|
67
|
-
this._state = exports.UnreferencedState.Active;
|
|
68
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
69
|
-
(0, common_utils_1.assert)(this.inactiveTimeoutMs <= this.sweepTimeoutMs, 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
|
|
70
|
-
}
|
|
71
|
-
this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
|
|
72
|
-
this._state = exports.UnreferencedState.SweepReady;
|
|
73
|
-
(0, common_utils_1.assert)(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
|
|
74
|
-
});
|
|
75
|
-
this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
|
|
76
|
-
this._state = exports.UnreferencedState.Inactive;
|
|
77
|
-
// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
|
|
78
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
79
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
|
|
80
|
-
}
|
|
81
|
-
});
|
|
82
|
-
this.updateTracking(currentReferenceTimestampMs);
|
|
83
|
-
}
|
|
84
|
-
get state() {
|
|
85
|
-
return this._state;
|
|
86
|
-
}
|
|
87
|
-
/* Updates the unreferenced state based on the provided timestamp. */
|
|
88
|
-
updateTracking(currentReferenceTimestampMs) {
|
|
89
|
-
const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
|
|
90
|
-
// If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
|
|
91
|
-
if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
|
|
92
|
-
this._state = exports.UnreferencedState.SweepReady;
|
|
93
|
-
this.clearTimers();
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
// If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
|
|
97
|
-
// Also, start a timer for the sweep timeout.
|
|
98
|
-
if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
|
|
99
|
-
this._state = exports.UnreferencedState.Inactive;
|
|
100
|
-
this.inactiveTimer.clear();
|
|
101
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
102
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
|
|
103
|
-
}
|
|
104
|
-
return;
|
|
105
|
-
}
|
|
106
|
-
// The node is still active. Ensure the inactive timer is running with the proper remaining duration.
|
|
107
|
-
this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
|
|
108
|
-
}
|
|
109
|
-
clearTimers() {
|
|
110
|
-
this.inactiveTimer.clear();
|
|
111
|
-
this.sweepTimer.clear();
|
|
112
|
-
}
|
|
113
|
-
/** Stop tracking this node. Reset the unreferenced timers and state, if any. */
|
|
114
|
-
stopTracking() {
|
|
115
|
-
this.clearTimers();
|
|
116
|
-
this._state = exports.UnreferencedState.Active;
|
|
117
|
-
}
|
|
118
|
-
}
|
|
119
|
-
exports.UnreferencedStateTracker = UnreferencedStateTracker;
|
|
24
|
+
const containerRuntime_1 = require("../containerRuntime");
|
|
25
|
+
const gcConfigs_1 = require("./gcConfigs");
|
|
26
|
+
const gcDefinitions_1 = require("./gcDefinitions");
|
|
27
|
+
const gcHelpers_1 = require("./gcHelpers");
|
|
28
|
+
const gcReferenceGraphAlgorithm_1 = require("./gcReferenceGraphAlgorithm");
|
|
29
|
+
const gcSummaryStateTracker_1 = require("./gcSummaryStateTracker");
|
|
30
|
+
const gcUnreferencedStateTracker_1 = require("./gcUnreferencedStateTracker");
|
|
120
31
|
/**
|
|
121
32
|
* The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
|
|
122
33
|
* its state across summaries.
|
|
@@ -128,20 +39,19 @@ exports.UnreferencedStateTracker = UnreferencedStateTracker;
|
|
|
128
39
|
* Graph - all nodes with their respective routes
|
|
129
40
|
*
|
|
130
41
|
* ```
|
|
131
|
-
*
|
|
42
|
+
* GC Graph
|
|
132
43
|
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
*
|
|
136
|
-
*
|
|
137
|
-
*
|
|
138
|
-
*
|
|
139
|
-
* NodeId = "dds1"
|
|
44
|
+
* Node
|
|
45
|
+
* NodeId = "datastore1"
|
|
46
|
+
* / \\
|
|
47
|
+
* OutboundRoute OutboundRoute
|
|
48
|
+
* / \\
|
|
49
|
+
* Node Node
|
|
50
|
+
* NodeId = "dds1" NodeId = "dds2"
|
|
140
51
|
* ```
|
|
141
52
|
*/
|
|
142
53
|
class GarbageCollector {
|
|
143
54
|
constructor(createParams) {
|
|
144
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
145
55
|
// Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
|
|
146
56
|
// outbound routes from that node.
|
|
147
57
|
this.newReferencesSinceLastRun = new Map();
|
|
@@ -160,194 +70,61 @@ class GarbageCollector {
|
|
|
160
70
|
this.completedRuns = 0;
|
|
161
71
|
this.runtime = createParams.runtime;
|
|
162
72
|
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
163
|
-
this.gcOptions = createParams.gcOptions;
|
|
164
73
|
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
165
74
|
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
166
75
|
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
167
76
|
this.activeConnection = createParams.activeConnection;
|
|
168
77
|
const baseSnapshot = createParams.baseSnapshot;
|
|
169
|
-
const metadata = createParams.metadata;
|
|
170
78
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
171
79
|
this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
|
|
172
80
|
all: { completedGCRuns: () => this.completedRuns },
|
|
173
81
|
}));
|
|
174
|
-
|
|
175
|
-
this.currentGCVersion =
|
|
176
|
-
this.mc.config.getBoolean(garbageCollectionConstants_1.gcVersionUpgradeToV2Key) === true
|
|
177
|
-
? garbageCollectionConstants_1.currentGCVersion
|
|
178
|
-
: garbageCollectionConstants_1.stableGCVersion;
|
|
179
|
-
this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
|
|
180
|
-
let prevSummaryGCVersion;
|
|
181
|
-
/**
|
|
182
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
183
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
184
|
-
*
|
|
185
|
-
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
186
|
-
* The buffer is added to account for any clock skew or other edge cases.
|
|
187
|
-
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
188
|
-
*/
|
|
189
|
-
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
190
|
-
const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
|
|
191
|
-
const bufferMs = garbageCollectionConstants_1.oneDayMs;
|
|
192
|
-
return (sessionExpiryTimeoutMs &&
|
|
193
|
-
sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
194
|
-
}
|
|
195
|
-
/**
|
|
196
|
-
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
197
|
-
* 1. Whether running GC mark phase is allowed or not.
|
|
198
|
-
* 2. Whether running GC sweep phase is allowed or not.
|
|
199
|
-
* 3. Whether GC session expiry is enabled or not.
|
|
200
|
-
* For existing containers, we get this information from the metadata blob of its summary.
|
|
201
|
-
*/
|
|
202
|
-
if (createParams.existing) {
|
|
203
|
-
prevSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
|
|
204
|
-
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
205
|
-
// other existing documents, GC is enabled.
|
|
206
|
-
this.gcEnabled = prevSummaryGCVersion > 0;
|
|
207
|
-
this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
|
|
208
|
-
this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
|
|
209
|
-
this.sweepTimeoutMs =
|
|
210
|
-
(_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
211
|
-
}
|
|
212
|
-
else {
|
|
213
|
-
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
214
|
-
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
215
|
-
if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
|
|
216
|
-
throw new container_utils_1.UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
|
|
217
|
-
}
|
|
218
|
-
// This Test Override only applies for new containers
|
|
219
|
-
const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
|
|
220
|
-
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
221
|
-
// flag in GC options to false.
|
|
222
|
-
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
223
|
-
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
224
|
-
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
225
|
-
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
226
|
-
if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
|
|
227
|
-
this.sessionExpiryTimeoutMs =
|
|
228
|
-
(_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
|
|
229
|
-
}
|
|
230
|
-
this.sweepTimeoutMs =
|
|
231
|
-
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
232
|
-
}
|
|
82
|
+
this.configs = (0, gcConfigs_1.generateGCConfigs)(this.mc, createParams);
|
|
233
83
|
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
234
|
-
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
84
|
+
if (this.configs.sessionExpiryTimeoutMs !== undefined) {
|
|
235
85
|
// If Test Override config is set, override Session Expiry timeout.
|
|
236
86
|
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
237
|
-
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
|
|
87
|
+
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.configs.sessionExpiryTimeoutMs;
|
|
238
88
|
this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => {
|
|
239
89
|
this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
240
90
|
});
|
|
241
91
|
this.sessionExpiryTimer.start();
|
|
242
92
|
}
|
|
243
|
-
|
|
244
|
-
// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
|
|
245
|
-
this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
|
|
246
|
-
/**
|
|
247
|
-
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
248
|
-
*
|
|
249
|
-
* 1. GC should be enabled for this container.
|
|
250
|
-
*
|
|
251
|
-
* 2. GC should not be disabled via disableGC GC option.
|
|
252
|
-
*
|
|
253
|
-
* These conditions can be overridden via runGCKey feature flag.
|
|
254
|
-
*/
|
|
255
|
-
this.shouldRunGC =
|
|
256
|
-
(_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d :
|
|
257
|
-
// GC must be enabled for the document.
|
|
258
|
-
(this.gcEnabled &&
|
|
259
|
-
// GC must not be disabled via GC options.
|
|
260
|
-
!this.gcOptions.disableGC);
|
|
261
|
-
/**
|
|
262
|
-
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
263
|
-
*
|
|
264
|
-
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
265
|
-
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
266
|
-
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
267
|
-
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
268
|
-
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
269
|
-
* feature flag.
|
|
270
|
-
*/
|
|
271
|
-
this.shouldRunSweep =
|
|
272
|
-
this.shouldRunGC &&
|
|
273
|
-
this.sweepTimeoutMs !== undefined &&
|
|
274
|
-
((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
275
|
-
this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
|
|
276
|
-
// Override inactive timeout if test config or gc options to override it is set.
|
|
277
|
-
this.inactiveTimeoutMs =
|
|
278
|
-
(_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
|
|
279
|
-
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
280
|
-
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
281
|
-
throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
282
|
-
}
|
|
283
|
-
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
284
|
-
this.testMode =
|
|
285
|
-
(_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
286
|
-
// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
|
|
287
|
-
// via feature flags.
|
|
288
|
-
this.tombstoneMode =
|
|
289
|
-
!this.shouldRunSweep && this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
|
|
290
|
-
// If GC ran in the container that generated the base snapshot, it will have a GC tree.
|
|
291
|
-
this.wasGCRunInLatestSummary = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[runtime_definitions_1.gcTreeKey]) !== undefined;
|
|
93
|
+
this.summaryStateTracker = new gcSummaryStateTracker_1.GCSummaryStateTracker(this.configs, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[runtime_definitions_1.gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */);
|
|
292
94
|
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
293
95
|
// it involves fetching blobs from storage which is expensive.
|
|
294
96
|
this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
|
|
295
|
-
var _a;
|
|
296
97
|
if (baseSnapshot === undefined) {
|
|
297
98
|
return undefined;
|
|
298
99
|
}
|
|
299
100
|
try {
|
|
300
101
|
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
301
102
|
const gcSnapshotTree = baseSnapshot.trees[runtime_definitions_1.gcTreeKey];
|
|
302
|
-
if (gcSnapshotTree
|
|
303
|
-
|
|
103
|
+
if (gcSnapshotTree === undefined) {
|
|
104
|
+
// back-compat - Older documents get their gc data reset for simplicity as there are few of them
|
|
105
|
+
// incremental gc summary will not work with older gc data as well
|
|
106
|
+
return undefined;
|
|
304
107
|
}
|
|
305
|
-
|
|
306
|
-
//
|
|
307
|
-
//
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
}
|
|
318
|
-
const gcSummaryDetails = await readAndParseBlob(blobId);
|
|
319
|
-
// If there are no nodes for this data store, skip it.
|
|
320
|
-
if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
|
|
321
|
-
continue;
|
|
322
|
-
}
|
|
323
|
-
const dsRootId = `/${dsId}`;
|
|
324
|
-
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
325
|
-
// Construct that entry by adding root data store ids to its outbound routes.
|
|
326
|
-
const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summaryFormat_1.dataStoreAttributesBlobName]);
|
|
327
|
-
if (initialSnapshotDetails.isRootDataStore) {
|
|
328
|
-
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
329
|
-
}
|
|
330
|
-
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
331
|
-
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
332
|
-
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
333
|
-
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
334
|
-
gcState.gcNodes[rootId] = {
|
|
335
|
-
outboundRoutes: Array.from(outboundRoutes),
|
|
336
|
-
};
|
|
337
|
-
}
|
|
338
|
-
(0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
|
|
339
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
340
|
-
gcSummaryDetails.unrefTimestamp;
|
|
108
|
+
const snapshotData = await (0, gcHelpers_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
|
|
109
|
+
// If the GC version in base snapshot does not match the GC version currently in effect, the GC data
|
|
110
|
+
// in the snapshot cannot be interpreted correctly. Set everything to undefined except for
|
|
111
|
+
// deletedNodes because irrespective of GC versions, these nodes have been deleted and cannot be
|
|
112
|
+
// brought back. The deletedNodes info is needed to identify when these nodes are used.
|
|
113
|
+
if (this.configs.gcVersionInBaseSnapshot !==
|
|
114
|
+
this.summaryStateTracker.currentGCVersion) {
|
|
115
|
+
return {
|
|
116
|
+
gcState: undefined,
|
|
117
|
+
tombstones: undefined,
|
|
118
|
+
deletedNodes: snapshotData.deletedNodes,
|
|
119
|
+
};
|
|
341
120
|
}
|
|
342
|
-
|
|
343
|
-
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
344
|
-
return Object.keys(gcState.gcNodes).length === 1
|
|
345
|
-
? undefined
|
|
346
|
-
: { gcState, tombstones: undefined, deletedNodes: undefined };
|
|
121
|
+
return snapshotData;
|
|
347
122
|
}
|
|
348
123
|
catch (error) {
|
|
349
124
|
const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
|
|
350
|
-
dpe.addTelemetryProperties({
|
|
125
|
+
dpe.addTelemetryProperties({
|
|
126
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
127
|
+
});
|
|
351
128
|
throw dpe;
|
|
352
129
|
}
|
|
353
130
|
});
|
|
@@ -385,12 +162,13 @@ class GarbageCollector {
|
|
|
385
162
|
return;
|
|
386
163
|
}
|
|
387
164
|
this.updateStateFromSnapshotData(baseSnapshotData, currentReferenceTimestampMs);
|
|
165
|
+
this.summaryStateTracker.initializeBaseState(baseSnapshotData);
|
|
388
166
|
});
|
|
389
167
|
// Get the GC details from the GC state in the base summary. This is returned in getBaseGCDetails which is
|
|
390
168
|
// used to initialize the GC state of all the nodes in the container.
|
|
391
169
|
this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
|
|
392
170
|
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
393
|
-
if (baseSnapshotData === undefined) {
|
|
171
|
+
if ((baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.gcState) === undefined) {
|
|
394
172
|
return {};
|
|
395
173
|
}
|
|
396
174
|
const gcNodes = {};
|
|
@@ -400,7 +178,7 @@ class GarbageCollector {
|
|
|
400
178
|
// Run GC on the nodes in the base summary to get the routes used in each node in the container.
|
|
401
179
|
// This is an optimization for space (vs performance) wherein we don't need to store the used routes of
|
|
402
180
|
// each node in the summary.
|
|
403
|
-
const usedRoutes = (0,
|
|
181
|
+
const usedRoutes = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcNodes, ["/"]).referencedNodeIds;
|
|
404
182
|
return { gcData: { gcNodes }, usedRoutes };
|
|
405
183
|
});
|
|
406
184
|
// Log all the GC options and the state determined by the garbage collector. This is interesting only for the
|
|
@@ -409,51 +187,18 @@ class GarbageCollector {
|
|
|
409
187
|
this.mc.logger.sendTelemetryEvent({
|
|
410
188
|
eventName: "GarbageCollectorLoaded",
|
|
411
189
|
gcConfigs: JSON.stringify(this.configs),
|
|
190
|
+
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
412
191
|
});
|
|
413
192
|
}
|
|
414
193
|
}
|
|
415
194
|
static create(createParams) {
|
|
416
195
|
return new GarbageCollector(createParams);
|
|
417
196
|
}
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
*
|
|
421
|
-
* 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
|
|
422
|
-
*
|
|
423
|
-
* 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
|
|
424
|
-
*
|
|
425
|
-
* 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
|
|
426
|
-
*
|
|
427
|
-
* 4. The GC version in the latest summary is different from the current GC version. This can happen if:
|
|
428
|
-
*
|
|
429
|
-
* 4.1. The summary this client loaded with has data from a different GC version.
|
|
430
|
-
*
|
|
431
|
-
* 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
432
|
-
*/
|
|
433
|
-
get summaryStateNeedsReset() {
|
|
434
|
-
return (this.gcStateNeedsReset ||
|
|
435
|
-
(this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion));
|
|
197
|
+
get shouldRunGC() {
|
|
198
|
+
return this.configs.shouldRunGC;
|
|
436
199
|
}
|
|
437
|
-
|
|
438
|
-
|
|
439
|
-
*
|
|
440
|
-
* 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
|
|
441
|
-
* it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
|
|
442
|
-
*
|
|
443
|
-
* 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
|
|
444
|
-
* a document and the first time GC is enabled after is was disabled before.
|
|
445
|
-
*
|
|
446
|
-
* 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
|
|
447
|
-
*
|
|
448
|
-
* Note that the state will be reset only once for the first summary generated after this returns true. After that,
|
|
449
|
-
* this will return false.
|
|
450
|
-
*/
|
|
451
|
-
get gcStateNeedsReset() {
|
|
452
|
-
return this.wasGCRunInLatestSummary !== this.shouldRunGC;
|
|
453
|
-
}
|
|
454
|
-
/** Returns a list of all the configurations for garbage collection. */
|
|
455
|
-
get configs() {
|
|
456
|
-
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
200
|
+
get summaryStateNeedsReset() {
|
|
201
|
+
return this.summaryStateTracker.doesSummaryStateNeedReset;
|
|
457
202
|
}
|
|
458
203
|
/**
|
|
459
204
|
* Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
|
|
@@ -477,7 +222,8 @@ class GarbageCollector {
|
|
|
477
222
|
}
|
|
478
223
|
// If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
|
|
479
224
|
// tombstone routes.
|
|
480
|
-
if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
225
|
+
if (this.configs.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
226
|
+
// Create a copy since we are writing from a source we don't control
|
|
481
227
|
this.tombstones = Array.from(baseSnapshotData.tombstones);
|
|
482
228
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
483
229
|
}
|
|
@@ -511,9 +257,9 @@ class GarbageCollector {
|
|
|
511
257
|
// tombstones.
|
|
512
258
|
// If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
|
|
513
259
|
// in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
|
|
514
|
-
if (this.shouldRunSweep) {
|
|
515
|
-
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.
|
|
516
|
-
? new Set(snapshotData.
|
|
260
|
+
if (this.configs.shouldRunSweep) {
|
|
261
|
+
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
|
|
262
|
+
? new Set(snapshotData.deletedNodes)
|
|
517
263
|
: undefined;
|
|
518
264
|
// If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
|
|
519
265
|
// delete them.
|
|
@@ -529,16 +275,15 @@ class GarbageCollector {
|
|
|
529
275
|
}
|
|
530
276
|
}
|
|
531
277
|
}
|
|
532
|
-
else if (this.tombstoneMode) {
|
|
278
|
+
else if (this.configs.tombstoneMode) {
|
|
533
279
|
// The snapshot may contain more or fewer tombstone nodes than this client. Update tombstone state and
|
|
534
280
|
// notify the runtime to update its state as well.
|
|
535
281
|
this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
|
|
536
282
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
537
283
|
}
|
|
538
284
|
// If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
|
|
539
|
-
if (snapshotData === undefined) {
|
|
285
|
+
if ((snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.gcState) === undefined) {
|
|
540
286
|
this.gcDataFromLastRun = undefined;
|
|
541
|
-
this.latestSummaryData = undefined;
|
|
542
287
|
return;
|
|
543
288
|
}
|
|
544
289
|
// Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
|
|
@@ -546,19 +291,11 @@ class GarbageCollector {
|
|
|
546
291
|
const gcNodes = {};
|
|
547
292
|
for (const [nodeId, nodeData] of Object.entries(snapshotData.gcState.gcNodes)) {
|
|
548
293
|
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
549
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
294
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
550
295
|
}
|
|
551
296
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
552
297
|
}
|
|
553
298
|
this.gcDataFromLastRun = { gcNodes };
|
|
554
|
-
// If tracking state across summaries, update latest summary data from the snapshot's GC data.
|
|
555
|
-
if (this.trackGCState) {
|
|
556
|
-
this.latestSummaryData = {
|
|
557
|
-
serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
|
|
558
|
-
serializedTombstones: JSON.stringify(snapshotData.tombstones),
|
|
559
|
-
serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
|
|
560
|
-
};
|
|
561
|
-
}
|
|
562
299
|
}
|
|
563
300
|
/**
|
|
564
301
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
@@ -580,7 +317,7 @@ class GarbageCollector {
|
|
|
580
317
|
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
581
318
|
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
582
319
|
*/
|
|
583
|
-
if (this.activeConnection() && this.shouldRunGC) {
|
|
320
|
+
if (this.activeConnection() && this.configs.shouldRunGC) {
|
|
584
321
|
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
585
322
|
}
|
|
586
323
|
}
|
|
@@ -588,9 +325,9 @@ class GarbageCollector {
|
|
|
588
325
|
* Runs garbage collection and updates the reference / used state of the nodes in the container.
|
|
589
326
|
* @returns stats of the GC run or undefined if GC did not run.
|
|
590
327
|
*/
|
|
591
|
-
async collectGarbage(options) {
|
|
328
|
+
async collectGarbage(options, telemetryContext) {
|
|
592
329
|
var _a;
|
|
593
|
-
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.
|
|
330
|
+
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.configs.runFullGC === true || this.summaryStateTracker.doesSummaryStateNeedReset);
|
|
594
331
|
const logger = options.logger
|
|
595
332
|
? telemetry_utils_1.ChildLogger.create(options.logger, undefined, {
|
|
596
333
|
all: { completedGCRuns: () => this.completedRuns },
|
|
@@ -613,11 +350,16 @@ class GarbageCollector {
|
|
|
613
350
|
});
|
|
614
351
|
return undefined;
|
|
615
352
|
}
|
|
353
|
+
// Add the options that are used to run GC to the telemetry context.
|
|
354
|
+
telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.setMultiple("fluid_GC", "Options", {
|
|
355
|
+
fullGC,
|
|
356
|
+
runSweep: options.runSweep,
|
|
357
|
+
});
|
|
616
358
|
return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
617
359
|
await this.runPreGCSteps();
|
|
618
360
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
619
361
|
const gcData = await this.runtime.getGCData(fullGC);
|
|
620
|
-
const gcResult = (0,
|
|
362
|
+
const gcResult = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
|
|
621
363
|
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
|
|
622
364
|
event.end(Object.assign(Object.assign({}, gcStats), { timestamp: currentReferenceTimestampMs }));
|
|
623
365
|
this.completedRuns++;
|
|
@@ -634,26 +376,29 @@ class GarbageCollector {
|
|
|
634
376
|
// Generate statistics from the current run. This is done before updating the current state because it
|
|
635
377
|
// generates some of its data based on previous state of the system.
|
|
636
378
|
const gcStats = this.generateStats(gcResult);
|
|
637
|
-
// Update the state
|
|
638
|
-
|
|
639
|
-
this.updateStateSinceLastRun(gcData, logger);
|
|
640
|
-
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
641
|
-
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
379
|
+
// Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
|
|
380
|
+
const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
|
|
642
381
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
643
382
|
// Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
|
|
644
383
|
// delete these objects here instead.
|
|
645
384
|
this.logSweepEvents(logger, currentReferenceTimestampMs);
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
385
|
+
let updatedGCData = gcData;
|
|
386
|
+
if (this.configs.shouldRunSweep) {
|
|
387
|
+
updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
|
|
388
|
+
}
|
|
389
|
+
else if (this.configs.testMode) {
|
|
390
|
+
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
391
|
+
// involving access to deleted data.
|
|
649
392
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
650
393
|
}
|
|
651
|
-
else if (this.tombstoneMode) {
|
|
394
|
+
else if (this.configs.tombstoneMode) {
|
|
395
|
+
this.tombstones = sweepReadyNodes;
|
|
652
396
|
// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
|
|
653
397
|
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
654
398
|
// Note: we will not tombstone in test mode.
|
|
655
399
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
656
400
|
}
|
|
401
|
+
this.gcDataFromLastRun = (0, gcHelpers_1.cloneGCData)(updatedGCData);
|
|
657
402
|
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
658
403
|
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
659
404
|
// reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
|
|
@@ -667,7 +412,7 @@ class GarbageCollector {
|
|
|
667
412
|
*/
|
|
668
413
|
summarize(fullTree, trackState, telemetryContext) {
|
|
669
414
|
var _a;
|
|
670
|
-
if (!this.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
415
|
+
if (!this.configs.shouldRunGC || this.gcDataFromLastRun === undefined) {
|
|
671
416
|
return;
|
|
672
417
|
}
|
|
673
418
|
const gcState = { gcNodes: {} };
|
|
@@ -677,97 +422,7 @@ class GarbageCollector {
|
|
|
677
422
|
unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
|
|
678
423
|
};
|
|
679
424
|
}
|
|
680
|
-
|
|
681
|
-
// Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
|
|
682
|
-
// to identify deleted nodes' usage.
|
|
683
|
-
const serializedDeletedNodes = this.deletedNodes.size > 0
|
|
684
|
-
? JSON.stringify(Array.from(this.deletedNodes).sort())
|
|
685
|
-
: undefined;
|
|
686
|
-
// If running in tombstone mode, serialize and write tombstones, if any.
|
|
687
|
-
const serializedTombstones = this.tombstoneMode
|
|
688
|
-
? this.tombstones.length > 0
|
|
689
|
-
? JSON.stringify(this.tombstones.sort())
|
|
690
|
-
: undefined
|
|
691
|
-
: undefined;
|
|
692
|
-
/**
|
|
693
|
-
* Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
|
|
694
|
-
* write summary handle instead of summary tree for GC.
|
|
695
|
-
* Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
|
|
696
|
-
* for each of these that did not change, write a summary handle.
|
|
697
|
-
*/
|
|
698
|
-
if (this.trackGCState) {
|
|
699
|
-
this.pendingSummaryData = {
|
|
700
|
-
serializedGCState,
|
|
701
|
-
serializedTombstones,
|
|
702
|
-
serializedDeletedNodes,
|
|
703
|
-
};
|
|
704
|
-
if (trackState && !fullTree && this.latestSummaryData !== undefined) {
|
|
705
|
-
// If nothing changed since last summary, send a summary handle for the entire GC data.
|
|
706
|
-
if (this.latestSummaryData.serializedGCState === serializedGCState &&
|
|
707
|
-
this.latestSummaryData.serializedTombstones === serializedTombstones) {
|
|
708
|
-
const stats = (0, runtime_utils_1.mergeStats)();
|
|
709
|
-
stats.handleNodeCount++;
|
|
710
|
-
return {
|
|
711
|
-
summary: {
|
|
712
|
-
type: protocol_definitions_1.SummaryType.Handle,
|
|
713
|
-
handle: `/${runtime_definitions_1.gcTreeKey}`,
|
|
714
|
-
handleType: protocol_definitions_1.SummaryType.Tree,
|
|
715
|
-
},
|
|
716
|
-
stats,
|
|
717
|
-
};
|
|
718
|
-
}
|
|
719
|
-
// If some state changed, build a GC summary tree.
|
|
720
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
|
|
721
|
-
}
|
|
722
|
-
}
|
|
723
|
-
// If not tracking GC state, build a GC summary tree without any summary handles.
|
|
724
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
|
|
725
|
-
}
|
|
726
|
-
/**
|
|
727
|
-
* Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
|
|
728
|
-
* If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
|
|
729
|
-
* If trackState is true, only states that changed are written. Rest are written as handles.
|
|
730
|
-
* @param serializedGCState - The GC state serialized as string.
|
|
731
|
-
* @param serializedTombstones - The tombstone state serialized as string.
|
|
732
|
-
* @param serializedDeletedNodes - Deleted nodes serialized as string.
|
|
733
|
-
* @param trackState - Whether we are tracking GC state across summaries.
|
|
734
|
-
* @returns the GC summary tree.
|
|
735
|
-
*/
|
|
736
|
-
buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, trackState) {
|
|
737
|
-
var _a, _b, _c;
|
|
738
|
-
const gcStateBlobKey = `${runtime_definitions_1.gcBlobPrefix}_root`;
|
|
739
|
-
const builder = new runtime_utils_1.SummaryTreeBuilder();
|
|
740
|
-
// If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
741
|
-
if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
|
|
742
|
-
builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${gcStateBlobKey}`);
|
|
743
|
-
}
|
|
744
|
-
else {
|
|
745
|
-
builder.addBlob(gcStateBlobKey, serializedGCState);
|
|
746
|
-
}
|
|
747
|
-
// If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
|
|
748
|
-
// summary blob.
|
|
749
|
-
if (serializedTombstones !== undefined) {
|
|
750
|
-
if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones &&
|
|
751
|
-
trackState) {
|
|
752
|
-
builder.addHandle(runtime_definitions_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${runtime_definitions_1.gcTombstoneBlobKey}`);
|
|
753
|
-
}
|
|
754
|
-
else {
|
|
755
|
-
builder.addBlob(runtime_definitions_1.gcTombstoneBlobKey, serializedTombstones);
|
|
756
|
-
}
|
|
757
|
-
}
|
|
758
|
-
// If there are no deleted nodes, return the summary tree.
|
|
759
|
-
if (serializedDeletedNodes === undefined) {
|
|
760
|
-
return builder.getSummaryTree();
|
|
761
|
-
}
|
|
762
|
-
// If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
|
|
763
|
-
if (((_c = this.latestSummaryData) === null || _c === void 0 ? void 0 : _c.serializedDeletedNodes) === serializedDeletedNodes &&
|
|
764
|
-
trackState) {
|
|
765
|
-
builder.addHandle(runtime_definitions_1.gcDeletedBlobKey, protocol_definitions_1.SummaryType.Blob, `/${runtime_definitions_1.gcTreeKey}/${runtime_definitions_1.gcDeletedBlobKey}`);
|
|
766
|
-
}
|
|
767
|
-
else {
|
|
768
|
-
builder.addBlob(runtime_definitions_1.gcDeletedBlobKey, serializedDeletedNodes);
|
|
769
|
-
}
|
|
770
|
-
return builder.getSummaryTree();
|
|
425
|
+
return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
|
|
771
426
|
}
|
|
772
427
|
getMetadata() {
|
|
773
428
|
return {
|
|
@@ -775,10 +430,11 @@ class GarbageCollector {
|
|
|
775
430
|
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
776
431
|
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
777
432
|
*/
|
|
778
|
-
gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
433
|
+
gcFeature: this.configs.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
434
|
+
gcFeatureMatrix: this.configs.persistedGcFeatureMatrix,
|
|
435
|
+
sessionExpiryTimeoutMs: this.configs.sessionExpiryTimeoutMs,
|
|
436
|
+
sweepEnabled: false,
|
|
437
|
+
sweepTimeoutMs: this.configs.sweepTimeoutMs,
|
|
782
438
|
};
|
|
783
439
|
}
|
|
784
440
|
/**
|
|
@@ -792,49 +448,23 @@ class GarbageCollector {
|
|
|
792
448
|
* Called to refresh the latest summary state. This happens when either a pending summary is acked or a snapshot
|
|
793
449
|
* is downloaded and should be used to update the state.
|
|
794
450
|
*/
|
|
795
|
-
async refreshLatestSummary(
|
|
796
|
-
|
|
797
|
-
// summary
|
|
798
|
-
//
|
|
799
|
-
|
|
800
|
-
|
|
801
|
-
|
|
802
|
-
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
this.latestSummaryGCVersion = this.currentGCVersion;
|
|
810
|
-
if (this.trackGCState) {
|
|
811
|
-
this.latestSummaryData = this.pendingSummaryData;
|
|
812
|
-
this.pendingSummaryData = undefined;
|
|
451
|
+
async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
|
|
452
|
+
const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
|
|
453
|
+
// If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
|
|
454
|
+
// this snapshot data.
|
|
455
|
+
if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
|
|
456
|
+
// The current reference timestamp should be available if we are refreshing state from a snapshot. There has
|
|
457
|
+
// to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
|
|
458
|
+
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
459
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
460
|
+
throw container_utils_1.DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
|
|
461
|
+
proposalHandle,
|
|
462
|
+
summaryRefSeq: result.summaryRefSeq,
|
|
463
|
+
gcConfigs: JSON.stringify(this.configs),
|
|
464
|
+
});
|
|
813
465
|
}
|
|
814
|
-
|
|
815
|
-
}
|
|
816
|
-
// If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
|
|
817
|
-
const snapshot = result.snapshot;
|
|
818
|
-
const metadataBlobId = snapshot.blobs[summaryFormat_1.metadataBlobName];
|
|
819
|
-
if (metadataBlobId) {
|
|
820
|
-
const metadata = await readAndParseBlob(metadataBlobId);
|
|
821
|
-
this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
|
|
822
|
-
}
|
|
823
|
-
// The current reference timestamp should be available if we are refreshing state from a snapshot. There has
|
|
824
|
-
// to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
|
|
825
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
826
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
827
|
-
throw container_utils_1.DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, { proposalHandle, summaryRefSeq, details: JSON.stringify(this.configs) });
|
|
466
|
+
this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
|
|
828
467
|
}
|
|
829
|
-
const gcSnapshotTree = snapshot.trees[runtime_definitions_1.gcTreeKey];
|
|
830
|
-
// If GC ran in the container that generated this snapshot, it will have a GC tree.
|
|
831
|
-
this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
|
|
832
|
-
let latestGCData;
|
|
833
|
-
if (gcSnapshotTree !== undefined) {
|
|
834
|
-
latestGCData = await (0, garbage_collector_1.getGCDataFromSnapshot)(gcSnapshotTree, readAndParseBlob);
|
|
835
|
-
}
|
|
836
|
-
this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
|
|
837
|
-
this.pendingSummaryData = undefined;
|
|
838
468
|
}
|
|
839
469
|
/**
|
|
840
470
|
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
@@ -845,11 +475,11 @@ class GarbageCollector {
|
|
|
845
475
|
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
846
476
|
*/
|
|
847
477
|
nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
|
|
848
|
-
if (!this.shouldRunGC) {
|
|
478
|
+
if (!this.configs.shouldRunGC) {
|
|
849
479
|
return;
|
|
850
480
|
}
|
|
851
481
|
const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
|
|
852
|
-
if (nodeStateTracker && nodeStateTracker.state !==
|
|
482
|
+
if (nodeStateTracker && nodeStateTracker.state !== gcDefinitions_1.UnreferencedState.Active) {
|
|
853
483
|
this.inactiveNodeUsed(reason, nodePath, nodeStateTracker, undefined /* fromNodeId */, packagePath, timestampMs, requestHeaders);
|
|
854
484
|
}
|
|
855
485
|
}
|
|
@@ -862,31 +492,31 @@ class GarbageCollector {
|
|
|
862
492
|
*/
|
|
863
493
|
addedOutboundReference(fromNodePath, toNodePath) {
|
|
864
494
|
var _a;
|
|
865
|
-
if (!this.shouldRunGC) {
|
|
495
|
+
if (!this.configs.shouldRunGC) {
|
|
866
496
|
return;
|
|
867
497
|
}
|
|
868
498
|
const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
|
|
869
499
|
outboundRoutes.push(toNodePath);
|
|
870
500
|
this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
871
501
|
const nodeStateTracker = this.unreferencedNodesState.get(toNodePath);
|
|
872
|
-
if (nodeStateTracker && nodeStateTracker.state !==
|
|
502
|
+
if (nodeStateTracker && nodeStateTracker.state !== gcDefinitions_1.UnreferencedState.Active) {
|
|
873
503
|
this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
|
|
874
504
|
}
|
|
875
505
|
if (this.tombstones.includes(toNodePath)) {
|
|
876
506
|
const nodeType = this.runtime.getNodeType(toNodePath);
|
|
877
507
|
let eventName = "GC_Tombstone_SubDatastore_Revived";
|
|
878
|
-
if (nodeType ===
|
|
508
|
+
if (nodeType === gcDefinitions_1.GCNodeType.DataStore) {
|
|
879
509
|
eventName = "GC_Tombstone_Datastore_Revived";
|
|
880
510
|
}
|
|
881
|
-
else if (nodeType ===
|
|
511
|
+
else if (nodeType === gcDefinitions_1.GCNodeType.Blob) {
|
|
882
512
|
eventName = "GC_Tombstone_Blob_Revived";
|
|
883
513
|
}
|
|
884
|
-
(0,
|
|
514
|
+
(0, gcHelpers_1.sendGCUnexpectedUsageEvent)(this.mc, {
|
|
885
515
|
eventName,
|
|
886
516
|
category: "generic",
|
|
887
|
-
|
|
888
|
-
url: (0, garbage_collector_1.trimLeadingSlashes)(toNodePath),
|
|
517
|
+
url: toNodePath,
|
|
889
518
|
nodeType,
|
|
519
|
+
gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
|
|
890
520
|
}, undefined /* packagePath */);
|
|
891
521
|
}
|
|
892
522
|
}
|
|
@@ -910,13 +540,16 @@ class GarbageCollector {
|
|
|
910
540
|
* @param gcData - The data representing the reference graph on which GC is run.
|
|
911
541
|
* @param gcResult - The result of the GC run on the gcData.
|
|
912
542
|
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
543
|
+
* @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
|
|
913
544
|
*/
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
545
|
+
updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
|
|
546
|
+
var _a;
|
|
547
|
+
// Get references from the current GC run + references between previous and current run and then update each
|
|
548
|
+
// node's state
|
|
549
|
+
const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
|
|
917
550
|
this.newReferencesSinceLastRun.clear();
|
|
918
551
|
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
919
|
-
for (const nodeId of
|
|
552
|
+
for (const nodeId of allNodesReferencedBetweenGCs) {
|
|
920
553
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
921
554
|
if (nodeStateTracker !== undefined) {
|
|
922
555
|
// Stop tracking so as to clear out any running timers.
|
|
@@ -929,22 +562,62 @@ class GarbageCollector {
|
|
|
929
562
|
* If a node became unreferenced in this run, start tracking it.
|
|
930
563
|
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
931
564
|
* from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
|
|
565
|
+
*
|
|
566
|
+
* If a node is sweep ready, store and then return it.
|
|
932
567
|
*/
|
|
568
|
+
const sweepReadyNodes = [];
|
|
933
569
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
934
570
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
935
571
|
if (nodeStateTracker === undefined) {
|
|
936
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
572
|
+
this.unreferencedNodesState.set(nodeId, new gcUnreferencedStateTracker_1.UnreferencedStateTracker(currentReferenceTimestampMs, this.configs.inactiveTimeoutMs, currentReferenceTimestampMs, this.configs.sweepTimeoutMs));
|
|
937
573
|
}
|
|
938
574
|
else {
|
|
939
575
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
940
|
-
if (
|
|
941
|
-
|
|
942
|
-
if (nodeType === exports.GCNodeType.DataStore || nodeType === exports.GCNodeType.Blob) {
|
|
943
|
-
this.tombstones.push(nodeId);
|
|
944
|
-
}
|
|
576
|
+
if (nodeStateTracker.state === gcDefinitions_1.UnreferencedState.SweepReady) {
|
|
577
|
+
sweepReadyNodes.push(nodeId);
|
|
945
578
|
}
|
|
946
579
|
}
|
|
947
580
|
}
|
|
581
|
+
return sweepReadyNodes;
|
|
582
|
+
}
|
|
583
|
+
/**
|
|
584
|
+
* Deletes nodes from both the runtime and garbage collection
|
|
585
|
+
* @param sweepReadyNodes - nodes that are ready to be deleted
|
|
586
|
+
*/
|
|
587
|
+
runSweepPhase(sweepReadyNodes, gcData) {
|
|
588
|
+
// TODO: GC:Validation - validate that removed routes are not double deleted
|
|
589
|
+
// TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
|
|
590
|
+
const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
|
|
591
|
+
const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
|
|
592
|
+
for (const nodeId of sweptRoutes) {
|
|
593
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
594
|
+
// TODO: GC:Validation - assert that the nodeStateTracker is defined
|
|
595
|
+
if (nodeStateTracker !== undefined) {
|
|
596
|
+
// Stop tracking so as to clear out any running timers.
|
|
597
|
+
nodeStateTracker.stopTracking();
|
|
598
|
+
// Delete the node as we don't need to track it any more.
|
|
599
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
600
|
+
}
|
|
601
|
+
// TODO: GC:Validation - assert that the deleted node is not a duplicate
|
|
602
|
+
this.deletedNodes.add(nodeId);
|
|
603
|
+
}
|
|
604
|
+
return updatedGCData;
|
|
605
|
+
}
|
|
606
|
+
/**
|
|
607
|
+
* @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
|
|
608
|
+
*/
|
|
609
|
+
deleteSweptRoutes(sweptRoutes, gcData) {
|
|
610
|
+
const sweptRoutesSet = new Set(sweptRoutes);
|
|
611
|
+
const gcNodes = {};
|
|
612
|
+
for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
|
|
613
|
+
if (!sweptRoutesSet.has(id)) {
|
|
614
|
+
gcNodes[id] = Array.from(outboundRoutes);
|
|
615
|
+
}
|
|
616
|
+
}
|
|
617
|
+
// TODO: GC:Validation - assert that the nodeId is in gcData
|
|
618
|
+
return {
|
|
619
|
+
gcNodes,
|
|
620
|
+
};
|
|
948
621
|
}
|
|
949
622
|
/**
|
|
950
623
|
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
@@ -956,16 +629,19 @@ class GarbageCollector {
|
|
|
956
629
|
* 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
|
|
957
630
|
* unreferenced, they could have been accessed and in-memory reference to them added.
|
|
958
631
|
*
|
|
959
|
-
* This function identifies nodes that were referenced since last run
|
|
632
|
+
* This function identifies nodes that were referenced since the last run.
|
|
960
633
|
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
634
|
+
*
|
|
635
|
+
* @returns - a list of all nodes referenced from the last local summary until now.
|
|
961
636
|
*/
|
|
962
|
-
|
|
637
|
+
findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
|
|
963
638
|
// If we haven't run GC before there is nothing to do.
|
|
964
|
-
|
|
965
|
-
|
|
639
|
+
// No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
|
|
640
|
+
if (previousGCData === undefined) {
|
|
641
|
+
return undefined;
|
|
966
642
|
}
|
|
967
643
|
// Find any references that haven't been identified correctly.
|
|
968
|
-
const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData,
|
|
644
|
+
const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun);
|
|
969
645
|
if (missingExplicitReferences.length > 0) {
|
|
970
646
|
missingExplicitReferences.forEach((missingExplicitReference) => {
|
|
971
647
|
logger.sendErrorEvent({
|
|
@@ -976,9 +652,9 @@ class GarbageCollector {
|
|
|
976
652
|
});
|
|
977
653
|
}
|
|
978
654
|
// No references were added since the last run so we don't have to update reference states of any unreferenced
|
|
979
|
-
// nodes
|
|
655
|
+
// nodes. There is no in between state at this point.
|
|
980
656
|
if (this.newReferencesSinceLastRun.size === 0) {
|
|
981
|
-
return;
|
|
657
|
+
return undefined;
|
|
982
658
|
}
|
|
983
659
|
/**
|
|
984
660
|
* Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
|
|
@@ -996,7 +672,7 @@ class GarbageCollector {
|
|
|
996
672
|
* - We don't require DDSes handles to be stored in a referenced DDS.
|
|
997
673
|
* - A new data store may have "root" DDSes already created and we don't detect them today.
|
|
998
674
|
*/
|
|
999
|
-
const gcDataSuperSet = (0,
|
|
675
|
+
const gcDataSuperSet = (0, gcHelpers_1.concatGarbageCollectionData)(previousGCData, currentGCData);
|
|
1000
676
|
const newOutboundRoutesSinceLastRun = [];
|
|
1001
677
|
this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
|
|
1002
678
|
if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
|
|
@@ -1014,19 +690,11 @@ class GarbageCollector {
|
|
|
1014
690
|
* Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
|
|
1015
691
|
* unreferenced and add unreferenced state.
|
|
1016
692
|
*/
|
|
1017
|
-
const gcResult = (0,
|
|
693
|
+
const gcResult = (0, gcReferenceGraphAlgorithm_1.runGarbageCollection)(gcDataSuperSet.gcNodes, [
|
|
1018
694
|
"/",
|
|
1019
695
|
...newOutboundRoutesSinceLastRun,
|
|
1020
696
|
]);
|
|
1021
|
-
|
|
1022
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
1023
|
-
if (nodeStateTracker !== undefined) {
|
|
1024
|
-
// Stop tracking so as to clear out any running timers.
|
|
1025
|
-
nodeStateTracker.stopTracking();
|
|
1026
|
-
// Delete the unreferenced state as we don't need to track it any more.
|
|
1027
|
-
this.unreferencedNodesState.delete(nodeId);
|
|
1028
|
-
}
|
|
1029
|
-
}
|
|
697
|
+
return gcResult.referencedNodeIds;
|
|
1030
698
|
}
|
|
1031
699
|
/**
|
|
1032
700
|
* Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
|
|
@@ -1060,7 +728,7 @@ class GarbageCollector {
|
|
|
1060
728
|
*/
|
|
1061
729
|
currentOutboundRoutes.forEach((route) => {
|
|
1062
730
|
const nodeType = this.runtime.getNodeType(route);
|
|
1063
|
-
if ((nodeType ===
|
|
731
|
+
if ((nodeType === gcDefinitions_1.GCNodeType.DataStore || nodeType === gcDefinitions_1.GCNodeType.Blob) &&
|
|
1064
732
|
!nodeId.startsWith(route) &&
|
|
1065
733
|
!previousRoutes.includes(route) &&
|
|
1066
734
|
!explicitRoutes.includes(route)) {
|
|
@@ -1103,7 +771,7 @@ class GarbageCollector {
|
|
|
1103
771
|
if (!referenced) {
|
|
1104
772
|
gcStats.unrefNodeCount++;
|
|
1105
773
|
}
|
|
1106
|
-
if (this.runtime.getNodeType(nodeId) ===
|
|
774
|
+
if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.DataStore) {
|
|
1107
775
|
gcStats.dataStoreCount++;
|
|
1108
776
|
if (stateUpdated) {
|
|
1109
777
|
gcStats.updatedDataStoreCount++;
|
|
@@ -1112,7 +780,7 @@ class GarbageCollector {
|
|
|
1112
780
|
gcStats.unrefDataStoreCount++;
|
|
1113
781
|
}
|
|
1114
782
|
}
|
|
1115
|
-
if (this.runtime.getNodeType(nodeId) ===
|
|
783
|
+
if (this.runtime.getNodeType(nodeId) === gcDefinitions_1.GCNodeType.Blob) {
|
|
1116
784
|
gcStats.attachmentBlobCount++;
|
|
1117
785
|
if (stateUpdated) {
|
|
1118
786
|
gcStats.updatedAttachmentBlobCount++;
|
|
@@ -1135,16 +803,16 @@ class GarbageCollector {
|
|
|
1135
803
|
* this will give us a view into how much deleted content a container has.
|
|
1136
804
|
*/
|
|
1137
805
|
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
1138
|
-
if (this.mc.config.getBoolean(
|
|
1139
|
-
this.sweepTimeoutMs === undefined) {
|
|
806
|
+
if (this.mc.config.getBoolean(gcDefinitions_1.disableSweepLogKey) === true ||
|
|
807
|
+
this.configs.sweepTimeoutMs === undefined) {
|
|
1140
808
|
return;
|
|
1141
809
|
}
|
|
1142
810
|
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
1143
|
-
if (nodeStateTracker.state !==
|
|
811
|
+
if (nodeStateTracker.state !== gcDefinitions_1.UnreferencedState.SweepReady) {
|
|
1144
812
|
return;
|
|
1145
813
|
}
|
|
1146
814
|
const nodeType = this.runtime.getNodeType(nodeId);
|
|
1147
|
-
if (nodeType !==
|
|
815
|
+
if (nodeType !== gcDefinitions_1.GCNodeType.DataStore && nodeType !== gcDefinitions_1.GCNodeType.Blob) {
|
|
1148
816
|
return;
|
|
1149
817
|
}
|
|
1150
818
|
// Log deleted event for each node only once to reduce noise in telemetry.
|
|
@@ -1158,7 +826,7 @@ class GarbageCollector {
|
|
|
1158
826
|
id: nodeId,
|
|
1159
827
|
type: nodeType,
|
|
1160
828
|
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
1161
|
-
timeout: this.sweepTimeoutMs,
|
|
829
|
+
timeout: this.configs.sweepTimeoutMs,
|
|
1162
830
|
completedGCRuns: this.completedRuns,
|
|
1163
831
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1164
832
|
});
|
|
@@ -1172,13 +840,13 @@ class GarbageCollector {
|
|
|
1172
840
|
// logging as nothing interesting would have happened worth logging.
|
|
1173
841
|
// If the node is active, skip logging.
|
|
1174
842
|
if (currentReferenceTimestampMs === undefined ||
|
|
1175
|
-
nodeStateTracker.state ===
|
|
843
|
+
nodeStateTracker.state === gcDefinitions_1.UnreferencedState.Active) {
|
|
1176
844
|
return;
|
|
1177
845
|
}
|
|
1178
846
|
// We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
|
|
1179
847
|
// as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
|
|
1180
848
|
const nodeType = this.runtime.getNodeType(nodeId);
|
|
1181
|
-
if (nodeType !==
|
|
849
|
+
if (nodeType !== gcDefinitions_1.GCNodeType.DataStore && nodeType !== gcDefinitions_1.GCNodeType.Blob) {
|
|
1182
850
|
return;
|
|
1183
851
|
}
|
|
1184
852
|
const state = nodeStateTracker.state;
|
|
@@ -1187,9 +855,9 @@ class GarbageCollector {
|
|
|
1187
855
|
return;
|
|
1188
856
|
}
|
|
1189
857
|
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
1190
|
-
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state ===
|
|
1191
|
-
? this.inactiveTimeoutMs
|
|
1192
|
-
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), {
|
|
858
|
+
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === gcDefinitions_1.UnreferencedState.Inactive
|
|
859
|
+
? this.configs.inactiveTimeoutMs
|
|
860
|
+
: this.configs.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
1193
861
|
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
1194
862
|
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
1195
863
|
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|
|
@@ -1207,19 +875,13 @@ class GarbageCollector {
|
|
|
1207
875
|
const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() });
|
|
1208
876
|
// Do not log the inactive object x events as error events as they are not the best signal for
|
|
1209
877
|
// detecting something wrong with GC either from the partner or from the runtime itself.
|
|
1210
|
-
if (state ===
|
|
878
|
+
if (state === gcDefinitions_1.UnreferencedState.Inactive) {
|
|
1211
879
|
this.mc.logger.sendTelemetryEvent(event);
|
|
1212
880
|
}
|
|
1213
881
|
else {
|
|
1214
882
|
this.mc.logger.sendErrorEvent(event);
|
|
1215
883
|
}
|
|
1216
884
|
}
|
|
1217
|
-
// If SweepReady Usage Detection is enabled, the handler may close the interactive container.
|
|
1218
|
-
// Once Sweep is fully implemented, this will be removed since the objects will be gone
|
|
1219
|
-
// and errors will arise elsewhere in the runtime
|
|
1220
|
-
if (state === exports.UnreferencedState.SweepReady) {
|
|
1221
|
-
this.sweepReadyUsageHandler.usageDetectedInInteractiveClient(Object.assign(Object.assign({}, propsToLog), { usageType }));
|
|
1222
|
-
}
|
|
1223
885
|
}
|
|
1224
886
|
}
|
|
1225
887
|
async logUnreferencedEvents(logger) {
|
|
@@ -1238,7 +900,7 @@ class GarbageCollector {
|
|
|
1238
900
|
*/
|
|
1239
901
|
const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
|
|
1240
902
|
const active = nodeStateTracker === undefined ||
|
|
1241
|
-
nodeStateTracker.state ===
|
|
903
|
+
nodeStateTracker.state === gcDefinitions_1.UnreferencedState.Active;
|
|
1242
904
|
if ((usageType === "Revived") === active) {
|
|
1243
905
|
const pkg = await this.getNodePackagePath(eventProps.id);
|
|
1244
906
|
const fromPkg = eventProps.fromId
|
|
@@ -1249,7 +911,7 @@ class GarbageCollector {
|
|
|
1249
911
|
: undefined, fromPkg: fromPkg
|
|
1250
912
|
? { value: fromPkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact }
|
|
1251
913
|
: undefined });
|
|
1252
|
-
if (state ===
|
|
914
|
+
if (state === gcDefinitions_1.UnreferencedState.Inactive) {
|
|
1253
915
|
logger.sendTelemetryEvent(event);
|
|
1254
916
|
}
|
|
1255
917
|
else {
|
|
@@ -1261,30 +923,4 @@ class GarbageCollector {
|
|
|
1261
923
|
}
|
|
1262
924
|
}
|
|
1263
925
|
exports.GarbageCollector = GarbageCollector;
|
|
1264
|
-
function generateSortedGCState(gcState) {
|
|
1265
|
-
const sortableArray = Object.entries(gcState.gcNodes);
|
|
1266
|
-
sortableArray.sort(([a], [b]) => a.localeCompare(b));
|
|
1267
|
-
const sortedGCState = { gcNodes: {} };
|
|
1268
|
-
for (const [nodeId, nodeData] of sortableArray) {
|
|
1269
|
-
nodeData.outboundRoutes.sort();
|
|
1270
|
-
sortedGCState.gcNodes[nodeId] = nodeData;
|
|
1271
|
-
}
|
|
1272
|
-
return sortedGCState;
|
|
1273
|
-
}
|
|
1274
|
-
/** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
|
|
1275
|
-
class TimerWithNoDefaultTimeout extends common_utils_1.Timer {
|
|
1276
|
-
constructor(callback) {
|
|
1277
|
-
// The default timeout/handlers will never be used since start/restart pass overrides below
|
|
1278
|
-
super(0, () => {
|
|
1279
|
-
throw new Error("DefaultHandler should not be used");
|
|
1280
|
-
});
|
|
1281
|
-
this.callback = callback;
|
|
1282
|
-
}
|
|
1283
|
-
start(timeoutMs) {
|
|
1284
|
-
super.start(timeoutMs, this.callback);
|
|
1285
|
-
}
|
|
1286
|
-
restart(timeoutMs) {
|
|
1287
|
-
super.restart(timeoutMs, this.callback);
|
|
1288
|
-
}
|
|
1289
|
-
}
|
|
1290
926
|
//# sourceMappingURL=garbageCollection.js.map
|