@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.4.1.0.148229
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +21 -10
- package/.mocharc.js +2 -2
- package/api-extractor.json +2 -2
- package/dist/batchTracker.d.ts +1 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +2 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +74 -42
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +321 -152
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +11 -9
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerHandleContext.d.ts.map +1 -1
- package/dist/containerHandleContext.js +3 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +148 -114
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +534 -342
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +11 -9
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +40 -13
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +146 -66
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +7 -3
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts.map +1 -1
- package/dist/dataStoreRegistry.js +3 -1
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +39 -12
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +164 -76
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts +19 -0
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/dist/deltaManagerSummarizerProxy.js +40 -0
- package/dist/deltaManagerSummarizerProxy.js.map +1 -0
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +8 -3
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +204 -0
- package/dist/gc/garbageCollection.d.ts.map +1 -0
- package/dist/gc/garbageCollection.js +926 -0
- package/dist/gc/garbageCollection.js.map +1 -0
- package/dist/gc/gcConfigs.d.ts +22 -0
- package/dist/gc/gcConfigs.d.ts.map +1 -0
- package/dist/gc/gcConfigs.js +143 -0
- package/dist/gc/gcConfigs.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +320 -0
- package/dist/gc/gcDefinitions.d.ts.map +1 -0
- package/dist/gc/gcDefinitions.js +81 -0
- package/dist/gc/gcDefinitions.js.map +1 -0
- package/dist/gc/gcHelpers.d.ts +86 -0
- package/dist/gc/gcHelpers.d.ts.map +1 -0
- package/dist/gc/gcHelpers.js +268 -0
- package/dist/gc/gcHelpers.js.map +1 -0
- package/dist/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
- package/dist/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
- package/dist/gc/gcReferenceGraphAlgorithm.js +49 -0
- package/dist/gc/gcReferenceGraphAlgorithm.js.map +1 -0
- package/dist/gc/gcSummaryDefinitions.d.ts +52 -0
- package/dist/gc/gcSummaryDefinitions.d.ts.map +1 -0
- package/dist/gc/gcSummaryDefinitions.js +7 -0
- package/dist/gc/gcSummaryDefinitions.js.map +1 -0
- package/dist/gc/gcSummaryStateTracker.d.ts +93 -0
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/dist/gc/gcSummaryStateTracker.js +239 -0
- package/dist/gc/gcSummaryStateTracker.js.map +1 -0
- package/{lib → dist/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
- package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/dist/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
- package/dist/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.js +94 -0
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/dist/gc/index.d.ts +13 -0
- package/dist/gc/index.d.ts.map +1 -0
- package/dist/gc/index.js +50 -0
- package/dist/gc/index.js.map +1 -0
- package/dist/index.d.ts +3 -8
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +7 -13
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +10 -0
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +37 -8
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +29 -1
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +3 -3
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +3 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +46 -17
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +6 -1
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +72 -18
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +46 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +142 -5
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +23 -2
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +101 -51
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +17 -2
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.d.ts.map +1 -1
- package/dist/opProperties.js +1 -3
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +6 -15
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +137 -165
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts +0 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +11 -21
- package/dist/scheduleManager.js.map +1 -1
- package/dist/storageServiceWithAttachBlobs.d.ts +17 -0
- package/dist/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/dist/storageServiceWithAttachBlobs.js +32 -0
- package/dist/storageServiceWithAttachBlobs.js.map +1 -0
- package/dist/summary/index.d.ts +17 -0
- package/dist/summary/index.d.ts.map +1 -0
- package/dist/summary/index.js +46 -0
- package/dist/summary/index.js.map +1 -0
- package/dist/summary/orderedClientElection.d.ts.map +1 -0
- package/dist/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
- package/dist/summary/orderedClientElection.js.map +1 -0
- package/{lib → dist/summary}/runWhileConnectedCoordinator.d.ts +3 -2
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/dist/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{lib → dist/summary}/runningSummarizer.d.ts +19 -18
- package/dist/summary/runningSummarizer.d.ts.map +1 -0
- package/dist/{runningSummarizer.js → summary/runningSummarizer.js} +191 -77
- package/dist/summary/runningSummarizer.js.map +1 -0
- package/dist/{summarizer.d.ts → summary/summarizer.d.ts} +6 -12
- package/dist/summary/summarizer.d.ts.map +1 -0
- package/dist/{summarizer.js → summary/summarizer.js} +32 -76
- package/dist/summary/summarizer.js.map +1 -0
- package/dist/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +1 -2
- package/dist/summary/summarizerClientElection.d.ts.map +1 -0
- package/dist/{summarizerClientElection.js → summary/summarizerClientElection.js} +3 -30
- package/dist/summary/summarizerClientElection.js.map +1 -0
- package/dist/{summarizerHeuristics.d.ts → summary/summarizerHeuristics.d.ts} +1 -1
- package/dist/summary/summarizerHeuristics.d.ts.map +1 -0
- package/dist/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +9 -12
- package/dist/summary/summarizerHeuristics.js.map +1 -0
- package/dist/summary/summarizerNode/index.d.ts +8 -0
- package/dist/summary/summarizerNode/index.d.ts.map +1 -0
- package/dist/summary/summarizerNode/index.js +12 -0
- package/dist/summary/summarizerNode/index.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNode.d.ts +149 -0
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNode.js +531 -0
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +132 -0
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +22 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +423 -0
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
- package/{lib → dist/summary}/summarizerTypes.d.ts +29 -42
- package/dist/summary/summarizerTypes.d.ts.map +1 -0
- package/dist/{summarizerTypes.js → summary/summarizerTypes.js} +0 -5
- package/dist/summary/summarizerTypes.js.map +1 -0
- package/dist/summary/summaryCollection.d.ts.map +1 -0
- package/dist/{summaryCollection.js → summary/summaryCollection.js} +18 -8
- package/dist/summary/summaryCollection.js.map +1 -0
- package/{lib → dist/summary}/summaryFormat.d.ts +3 -21
- package/dist/summary/summaryFormat.d.ts.map +1 -0
- package/dist/{summaryFormat.js → summary/summaryFormat.js} +19 -21
- package/dist/summary/summaryFormat.js.map +1 -0
- package/dist/summary/summaryGenerator.d.ts.map +1 -0
- package/dist/{summaryGenerator.js → summary/summaryGenerator.js} +34 -16
- package/dist/summary/summaryGenerator.js.map +1 -0
- package/dist/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/dist/summary/summaryManager.d.ts.map +1 -0
- package/dist/{summaryManager.js → summary/summaryManager.js} +21 -9
- package/dist/summary/summaryManager.js.map +1 -0
- package/dist/throttler.d.ts +2 -2
- package/dist/throttler.d.ts.map +1 -1
- package/dist/throttler.js +4 -4
- package/dist/throttler.js.map +1 -1
- package/lib/batchTracker.d.ts +1 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +2 -1
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +74 -42
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +322 -153
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +11 -9
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerHandleContext.d.ts.map +1 -1
- package/lib/containerHandleContext.js +3 -1
- package/lib/containerHandleContext.js.map +1 -1
- package/lib/containerRuntime.d.ts +148 -114
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +506 -314
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +11 -9
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +40 -13
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +136 -56
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +7 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts.map +1 -1
- package/lib/dataStoreRegistry.js +3 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +39 -12
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +162 -74
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts +19 -0
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -0
- package/lib/deltaManagerSummarizerProxy.js +36 -0
- package/lib/deltaManagerSummarizerProxy.js.map +1 -0
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +9 -4
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +204 -0
- package/lib/gc/garbageCollection.d.ts.map +1 -0
- package/lib/gc/garbageCollection.js +922 -0
- package/lib/gc/garbageCollection.js.map +1 -0
- package/lib/gc/gcConfigs.d.ts +22 -0
- package/lib/gc/gcConfigs.d.ts.map +1 -0
- package/lib/gc/gcConfigs.js +139 -0
- package/lib/gc/gcConfigs.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +320 -0
- package/lib/gc/gcDefinitions.d.ts.map +1 -0
- package/lib/gc/gcDefinitions.js +78 -0
- package/lib/gc/gcDefinitions.js.map +1 -0
- package/lib/gc/gcHelpers.d.ts +86 -0
- package/lib/gc/gcHelpers.d.ts.map +1 -0
- package/lib/gc/gcHelpers.js +254 -0
- package/lib/gc/gcHelpers.js.map +1 -0
- package/lib/gc/gcReferenceGraphAlgorithm.d.ts +16 -0
- package/lib/gc/gcReferenceGraphAlgorithm.d.ts.map +1 -0
- package/lib/gc/gcReferenceGraphAlgorithm.js +45 -0
- package/lib/gc/gcReferenceGraphAlgorithm.js.map +1 -0
- package/lib/gc/gcSummaryDefinitions.d.ts +52 -0
- package/lib/gc/gcSummaryDefinitions.d.ts.map +1 -0
- package/lib/gc/gcSummaryDefinitions.js +6 -0
- package/lib/gc/gcSummaryDefinitions.js.map +1 -0
- package/lib/gc/gcSummaryStateTracker.d.ts +93 -0
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/lib/gc/gcSummaryStateTracker.js +235 -0
- package/lib/gc/gcSummaryStateTracker.js.map +1 -0
- package/{dist → lib/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
- package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/lib/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
- package/lib/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.js +90 -0
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/lib/gc/index.d.ts +13 -0
- package/lib/gc/index.d.ts.map +1 -0
- package/lib/gc/index.js +12 -0
- package/lib/gc/index.js.map +1 -0
- package/lib/index.d.ts +3 -8
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -6
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +10 -0
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +35 -7
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +29 -1
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +3 -3
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +2 -2
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +47 -18
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +6 -1
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +72 -18
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +46 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +141 -5
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +23 -2
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +103 -53
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +17 -2
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.d.ts.map +1 -1
- package/lib/opProperties.js +1 -3
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +6 -15
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +137 -165
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts +0 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +11 -21
- package/lib/scheduleManager.js.map +1 -1
- package/lib/storageServiceWithAttachBlobs.d.ts +17 -0
- package/lib/storageServiceWithAttachBlobs.d.ts.map +1 -0
- package/lib/storageServiceWithAttachBlobs.js +28 -0
- package/lib/storageServiceWithAttachBlobs.js.map +1 -0
- package/lib/summary/index.d.ts +17 -0
- package/lib/summary/index.d.ts.map +1 -0
- package/lib/summary/index.js +15 -0
- package/lib/summary/index.js.map +1 -0
- package/lib/summary/orderedClientElection.d.ts.map +1 -0
- package/lib/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
- package/lib/summary/orderedClientElection.js.map +1 -0
- package/{dist → lib/summary}/runWhileConnectedCoordinator.d.ts +3 -2
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/lib/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +5 -4
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{dist → lib/summary}/runningSummarizer.d.ts +19 -18
- package/lib/summary/runningSummarizer.d.ts.map +1 -0
- package/lib/{runningSummarizer.js → summary/runningSummarizer.js} +193 -79
- package/lib/summary/runningSummarizer.js.map +1 -0
- package/lib/{summarizer.d.ts → summary/summarizer.d.ts} +6 -12
- package/lib/summary/summarizer.d.ts.map +1 -0
- package/lib/{summarizer.js → summary/summarizer.js} +34 -78
- package/lib/summary/summarizer.js.map +1 -0
- package/lib/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +1 -2
- package/lib/summary/summarizerClientElection.d.ts.map +1 -0
- package/lib/{summarizerClientElection.js → summary/summarizerClientElection.js} +3 -30
- package/lib/summary/summarizerClientElection.js.map +1 -0
- package/lib/{summarizerHeuristics.d.ts → summary/summarizerHeuristics.d.ts} +1 -1
- package/lib/summary/summarizerHeuristics.d.ts.map +1 -0
- package/lib/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +9 -12
- package/lib/summary/summarizerHeuristics.js.map +1 -0
- package/lib/summary/summarizerNode/index.d.ts +8 -0
- package/lib/summary/summarizerNode/index.d.ts.map +1 -0
- package/lib/summary/summarizerNode/index.js +7 -0
- package/lib/summary/summarizerNode/index.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNode.d.ts +149 -0
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNode.js +526 -0
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +125 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +125 -0
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +22 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +419 -0
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -0
- package/{dist → lib/summary}/summarizerTypes.d.ts +29 -42
- package/lib/summary/summarizerTypes.d.ts.map +1 -0
- package/lib/summary/summarizerTypes.js +6 -0
- package/lib/summary/summarizerTypes.js.map +1 -0
- package/lib/summary/summaryCollection.d.ts.map +1 -0
- package/lib/{summaryCollection.js → summary/summaryCollection.js} +18 -8
- package/lib/summary/summaryCollection.js.map +1 -0
- package/{dist → lib/summary}/summaryFormat.d.ts +3 -21
- package/lib/summary/summaryFormat.d.ts.map +1 -0
- package/lib/{summaryFormat.js → summary/summaryFormat.js} +20 -21
- package/lib/summary/summaryFormat.js.map +1 -0
- package/lib/summary/summaryGenerator.d.ts.map +1 -0
- package/lib/{summaryGenerator.js → summary/summaryGenerator.js} +34 -16
- package/lib/summary/summaryGenerator.js.map +1 -0
- package/lib/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/lib/summary/summaryManager.d.ts.map +1 -0
- package/lib/{summaryManager.js → summary/summaryManager.js} +21 -9
- package/lib/summary/summaryManager.js.map +1 -0
- package/lib/throttler.d.ts +2 -2
- package/lib/throttler.d.ts.map +1 -1
- package/lib/throttler.js +4 -4
- package/lib/throttler.js.map +1 -1
- package/package.json +67 -61
- package/prettier.config.cjs +1 -1
- package/src/batchTracker.ts +55 -50
- package/src/blobManager.ts +863 -594
- package/src/connectionTelemetry.ts +280 -249
- package/src/containerHandleContext.ts +27 -29
- package/src/containerRuntime.ts +3174 -2805
- package/src/dataStore.ts +172 -159
- package/src/dataStoreContext.ts +1141 -993
- package/src/dataStoreContexts.ts +178 -161
- package/src/dataStoreRegistry.ts +25 -20
- package/src/dataStores.ts +887 -716
- package/src/deltaManagerSummarizerProxy.ts +46 -0
- package/src/deltaScheduler.ts +158 -150
- package/{garbageCollection.md → src/gc/garbageCollection.md} +16 -3
- package/src/gc/garbageCollection.ts +1250 -0
- package/src/gc/gcConfigs.ts +193 -0
- package/src/gc/gcDefinitions.ts +387 -0
- package/src/gc/gcHelpers.ts +332 -0
- package/src/gc/gcReferenceGraphAlgorithm.ts +52 -0
- package/src/gc/gcSummaryDefinitions.ts +54 -0
- package/src/gc/gcSummaryStateTracker.ts +329 -0
- package/src/gc/gcSweepReadyUsageDetection.ts +145 -0
- package/src/gc/gcUnreferencedStateTracker.ts +114 -0
- package/src/gc/index.ts +65 -0
- package/src/index.ts +61 -75
- package/src/opLifecycle/README.md +157 -0
- package/src/opLifecycle/batchManager.ts +119 -86
- package/src/opLifecycle/definitions.ts +49 -19
- package/src/opLifecycle/index.ts +7 -6
- package/src/opLifecycle/opCompressor.ts +78 -40
- package/src/opLifecycle/opDecompressor.ts +148 -64
- package/src/opLifecycle/opSplitter.ts +269 -66
- package/src/opLifecycle/outbox.ts +268 -184
- package/src/opLifecycle/remoteMessageProcessor.ts +63 -47
- package/src/opProperties.ts +11 -9
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +386 -381
- package/src/scheduleManager.ts +299 -280
- package/src/storageServiceWithAttachBlobs.ts +38 -0
- package/src/summary/index.ts +105 -0
- package/src/summary/orderedClientElection.ts +564 -0
- package/src/summary/runWhileConnectedCoordinator.ts +113 -0
- package/src/summary/runningSummarizer.ts +788 -0
- package/src/summary/summarizer.ts +372 -0
- package/src/summary/summarizerClientElection.ts +139 -0
- package/src/summary/summarizerHeuristics.ts +224 -0
- package/src/summary/summarizerNode/index.ts +12 -0
- package/src/summary/summarizerNode/summarizerNode.ts +766 -0
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +214 -0
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +644 -0
- package/src/summary/summarizerTypes.ts +507 -0
- package/src/summary/summaryCollection.ts +450 -0
- package/src/summary/summaryFormat.ts +228 -0
- package/src/summary/summaryGenerator.ts +505 -0
- package/src/summary/summaryManager.ts +423 -0
- package/src/throttler.ts +131 -122
- package/tsconfig.esnext.json +6 -6
- package/tsconfig.json +9 -13
- package/dist/garbageCollection.d.ts +0 -387
- package/dist/garbageCollection.d.ts.map +0 -1
- package/dist/garbageCollection.js +0 -1138
- package/dist/garbageCollection.js.map +0 -1
- package/dist/garbageCollectionConstants.d.ts +0 -19
- package/dist/garbageCollectionConstants.d.ts.map +0 -1
- package/dist/garbageCollectionConstants.js +0 -34
- package/dist/garbageCollectionConstants.js.map +0 -1
- package/dist/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/dist/gcSweepReadyUsageDetection.js.map +0 -1
- package/dist/orderedClientElection.d.ts.map +0 -1
- package/dist/orderedClientElection.js.map +0 -1
- package/dist/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/dist/runWhileConnectedCoordinator.js.map +0 -1
- package/dist/runningSummarizer.d.ts.map +0 -1
- package/dist/runningSummarizer.js.map +0 -1
- package/dist/serializedSnapshotStorage.d.ts +0 -58
- package/dist/serializedSnapshotStorage.d.ts.map +0 -1
- package/dist/serializedSnapshotStorage.js +0 -108
- package/dist/serializedSnapshotStorage.js.map +0 -1
- package/dist/summarizer.d.ts.map +0 -1
- package/dist/summarizer.js.map +0 -1
- package/dist/summarizerClientElection.d.ts.map +0 -1
- package/dist/summarizerClientElection.js.map +0 -1
- package/dist/summarizerHandle.d.ts +0 -12
- package/dist/summarizerHandle.d.ts.map +0 -1
- package/dist/summarizerHandle.js +0 -22
- package/dist/summarizerHandle.js.map +0 -1
- package/dist/summarizerHeuristics.d.ts.map +0 -1
- package/dist/summarizerHeuristics.js.map +0 -1
- package/dist/summarizerTypes.d.ts.map +0 -1
- package/dist/summarizerTypes.js.map +0 -1
- package/dist/summaryCollection.d.ts.map +0 -1
- package/dist/summaryCollection.js.map +0 -1
- package/dist/summaryFormat.d.ts.map +0 -1
- package/dist/summaryFormat.js.map +0 -1
- package/dist/summaryGenerator.d.ts.map +0 -1
- package/dist/summaryGenerator.js.map +0 -1
- package/dist/summaryManager.d.ts.map +0 -1
- package/dist/summaryManager.js.map +0 -1
- package/lib/garbageCollection.d.ts +0 -387
- package/lib/garbageCollection.d.ts.map +0 -1
- package/lib/garbageCollection.js +0 -1133
- package/lib/garbageCollection.js.map +0 -1
- package/lib/garbageCollectionConstants.d.ts +0 -19
- package/lib/garbageCollectionConstants.d.ts.map +0 -1
- package/lib/garbageCollectionConstants.js +0 -31
- package/lib/garbageCollectionConstants.js.map +0 -1
- package/lib/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/lib/gcSweepReadyUsageDetection.js.map +0 -1
- package/lib/orderedClientElection.d.ts.map +0 -1
- package/lib/orderedClientElection.js.map +0 -1
- package/lib/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/lib/runWhileConnectedCoordinator.js.map +0 -1
- package/lib/runningSummarizer.d.ts.map +0 -1
- package/lib/runningSummarizer.js.map +0 -1
- package/lib/serializedSnapshotStorage.d.ts +0 -58
- package/lib/serializedSnapshotStorage.d.ts.map +0 -1
- package/lib/serializedSnapshotStorage.js +0 -104
- package/lib/serializedSnapshotStorage.js.map +0 -1
- package/lib/summarizer.d.ts.map +0 -1
- package/lib/summarizer.js.map +0 -1
- package/lib/summarizerClientElection.d.ts.map +0 -1
- package/lib/summarizerClientElection.js.map +0 -1
- package/lib/summarizerHandle.d.ts +0 -12
- package/lib/summarizerHandle.d.ts.map +0 -1
- package/lib/summarizerHandle.js +0 -18
- package/lib/summarizerHandle.js.map +0 -1
- package/lib/summarizerHeuristics.d.ts.map +0 -1
- package/lib/summarizerHeuristics.js.map +0 -1
- package/lib/summarizerTypes.d.ts.map +0 -1
- package/lib/summarizerTypes.js +0 -9
- package/lib/summarizerTypes.js.map +0 -1
- package/lib/summaryCollection.d.ts.map +0 -1
- package/lib/summaryCollection.js.map +0 -1
- package/lib/summaryFormat.d.ts.map +0 -1
- package/lib/summaryFormat.js.map +0 -1
- package/lib/summaryGenerator.d.ts.map +0 -1
- package/lib/summaryGenerator.js.map +0 -1
- package/lib/summaryManager.d.ts.map +0 -1
- package/lib/summaryManager.js.map +0 -1
- package/src/garbageCollection.ts +0 -1646
- package/src/garbageCollectionConstants.ts +0 -35
- package/src/gcSweepReadyUsageDetection.ts +0 -139
- package/src/orderedClientElection.ts +0 -532
- package/src/runWhileConnectedCoordinator.ts +0 -106
- package/src/runningSummarizer.ts +0 -611
- package/src/serializedSnapshotStorage.ts +0 -146
- package/src/summarizer.ts +0 -421
- package/src/summarizerClientElection.ts +0 -161
- package/src/summarizerHandle.ts +0 -21
- package/src/summarizerHeuristics.ts +0 -222
- package/src/summarizerTypes.ts +0 -510
- package/src/summaryCollection.ts +0 -421
- package/src/summaryFormat.ts +0 -235
- package/src/summaryGenerator.ts +0 -446
- package/src/summaryManager.ts +0 -394
- /package/dist/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/dist/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/dist/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
- /package/lib/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/lib/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/lib/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
|
@@ -1,1138 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
/*!
|
|
3
|
-
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
4
|
-
* Licensed under the MIT License.
|
|
5
|
-
*/
|
|
6
|
-
var __rest = (this && this.__rest) || function (s, e) {
|
|
7
|
-
var t = {};
|
|
8
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
9
|
-
t[p] = s[p];
|
|
10
|
-
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
11
|
-
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
12
|
-
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
13
|
-
t[p[i]] = s[p[i]];
|
|
14
|
-
}
|
|
15
|
-
return t;
|
|
16
|
-
};
|
|
17
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
18
|
-
exports.GarbageCollector = exports.UnreferencedStateTracker = exports.UnreferencedState = exports.GCNodeType = void 0;
|
|
19
|
-
const common_utils_1 = require("@fluidframework/common-utils");
|
|
20
|
-
const container_utils_1 = require("@fluidframework/container-utils");
|
|
21
|
-
const garbage_collector_1 = require("@fluidframework/garbage-collector");
|
|
22
|
-
const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
23
|
-
const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
24
|
-
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
25
|
-
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
26
|
-
const containerRuntime_1 = require("./containerRuntime");
|
|
27
|
-
const dataStores_1 = require("./dataStores");
|
|
28
|
-
const garbageCollectionConstants_1 = require("./garbageCollectionConstants");
|
|
29
|
-
const gcSweepReadyUsageDetection_1 = require("./gcSweepReadyUsageDetection");
|
|
30
|
-
const summaryFormat_1 = require("./summaryFormat");
|
|
31
|
-
/** This is the current version of garbage collection. */
|
|
32
|
-
const GCVersion = 1;
|
|
33
|
-
/** The types of GC nodes in the GC reference graph. */
|
|
34
|
-
exports.GCNodeType = {
|
|
35
|
-
// Nodes that are for data stores.
|
|
36
|
-
DataStore: "DataStore",
|
|
37
|
-
// Nodes that are within a data store. For example, DDS nodes.
|
|
38
|
-
SubDataStore: "SubDataStore",
|
|
39
|
-
// Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
|
|
40
|
-
Blob: "Blob",
|
|
41
|
-
// Nodes that are neither of the above. For example, root node.
|
|
42
|
-
Other: "Other",
|
|
43
|
-
};
|
|
44
|
-
/** The state of node that is unreferenced. */
|
|
45
|
-
exports.UnreferencedState = {
|
|
46
|
-
/** The node is active, i.e., it can become referenced again. */
|
|
47
|
-
Active: "Active",
|
|
48
|
-
/** The node is inactive, i.e., it should not become referenced. */
|
|
49
|
-
Inactive: "Inactive",
|
|
50
|
-
/** The node is ready to be deleted by the sweep phase. */
|
|
51
|
-
SweepReady: "SweepReady",
|
|
52
|
-
};
|
|
53
|
-
/**
|
|
54
|
-
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
|
|
55
|
-
* be deleted by the sweep phase.
|
|
56
|
-
*/
|
|
57
|
-
class UnreferencedStateTracker {
|
|
58
|
-
constructor(unreferencedTimestampMs,
|
|
59
|
-
/** The time after which node transitions to Inactive state. */
|
|
60
|
-
inactiveTimeoutMs,
|
|
61
|
-
/** The current reference timestamp used to track how long this node has been unreferenced for. */
|
|
62
|
-
currentReferenceTimestampMs,
|
|
63
|
-
/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
|
|
64
|
-
sweepTimeoutMs) {
|
|
65
|
-
this.unreferencedTimestampMs = unreferencedTimestampMs;
|
|
66
|
-
this.inactiveTimeoutMs = inactiveTimeoutMs;
|
|
67
|
-
this.sweepTimeoutMs = sweepTimeoutMs;
|
|
68
|
-
this._state = exports.UnreferencedState.Active;
|
|
69
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
70
|
-
(0, common_utils_1.assert)(this.inactiveTimeoutMs <= this.sweepTimeoutMs, 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
|
|
71
|
-
}
|
|
72
|
-
this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
|
|
73
|
-
this._state = exports.UnreferencedState.SweepReady;
|
|
74
|
-
(0, common_utils_1.assert)(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
|
|
75
|
-
});
|
|
76
|
-
this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
|
|
77
|
-
this._state = exports.UnreferencedState.Inactive;
|
|
78
|
-
// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
|
|
79
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
80
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
|
|
81
|
-
}
|
|
82
|
-
});
|
|
83
|
-
this.updateTracking(currentReferenceTimestampMs);
|
|
84
|
-
}
|
|
85
|
-
get state() {
|
|
86
|
-
return this._state;
|
|
87
|
-
}
|
|
88
|
-
/* Updates the unreferenced state based on the provided timestamp. */
|
|
89
|
-
updateTracking(currentReferenceTimestampMs) {
|
|
90
|
-
const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
|
|
91
|
-
// If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
|
|
92
|
-
if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
|
|
93
|
-
this._state = exports.UnreferencedState.SweepReady;
|
|
94
|
-
this.clearTimers();
|
|
95
|
-
return;
|
|
96
|
-
}
|
|
97
|
-
// If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
|
|
98
|
-
// Also, start a timer for the sweep timeout.
|
|
99
|
-
if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
|
|
100
|
-
this._state = exports.UnreferencedState.Inactive;
|
|
101
|
-
this.inactiveTimer.clear();
|
|
102
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
103
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
|
|
104
|
-
}
|
|
105
|
-
return;
|
|
106
|
-
}
|
|
107
|
-
// The node is still active. Ensure the inactive timer is running with the proper remaining duration.
|
|
108
|
-
this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
|
|
109
|
-
}
|
|
110
|
-
clearTimers() {
|
|
111
|
-
this.inactiveTimer.clear();
|
|
112
|
-
this.sweepTimer.clear();
|
|
113
|
-
}
|
|
114
|
-
/** Stop tracking this node. Reset the unreferenced timers and state, if any. */
|
|
115
|
-
stopTracking() {
|
|
116
|
-
this.clearTimers();
|
|
117
|
-
this._state = exports.UnreferencedState.Active;
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
exports.UnreferencedStateTracker = UnreferencedStateTracker;
|
|
121
|
-
/**
|
|
122
|
-
* The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
|
|
123
|
-
* its state across summaries.
|
|
124
|
-
*
|
|
125
|
-
* Node - represented as nodeId, it's a node on the GC graph
|
|
126
|
-
*
|
|
127
|
-
* Outbound Route - a path from one node to another node, think `nodeA` -\> `nodeB`
|
|
128
|
-
*
|
|
129
|
-
* Graph - all nodes with their respective routes
|
|
130
|
-
*
|
|
131
|
-
* ```
|
|
132
|
-
* GC Graph
|
|
133
|
-
*
|
|
134
|
-
* Node
|
|
135
|
-
* NodeId = "datastore1"
|
|
136
|
-
* / \\
|
|
137
|
-
* OutboundRoute OutboundRoute
|
|
138
|
-
* / \\
|
|
139
|
-
* Node Node
|
|
140
|
-
* NodeId = "dds1" NodeId = "dds2"
|
|
141
|
-
* ```
|
|
142
|
-
*/
|
|
143
|
-
class GarbageCollector {
|
|
144
|
-
constructor(createParams) {
|
|
145
|
-
var _a, _b, _c, _d, _e, _f, _g, _h;
|
|
146
|
-
/**
|
|
147
|
-
* Tells whether the initial GC state needs to be reset. This can happen under 2 conditions:
|
|
148
|
-
*
|
|
149
|
-
* 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
|
|
150
|
-
* it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
|
|
151
|
-
*
|
|
152
|
-
* 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
|
|
153
|
-
* a document and the first time GC is enabled after is was disabled before.
|
|
154
|
-
*
|
|
155
|
-
* Note that the state needs reset only for the very first time summary is generated by this client. After that, the
|
|
156
|
-
* state will be up-to-date and this flag will be reset.
|
|
157
|
-
*/
|
|
158
|
-
this.initialStateNeedsReset = false;
|
|
159
|
-
// The current GC version that this container is running.
|
|
160
|
-
this.currentGCVersion = GCVersion;
|
|
161
|
-
// Keeps a list of references (edges in the GC graph) between GC runs. Each entry has a node id and a list of
|
|
162
|
-
// outbound routes from that node.
|
|
163
|
-
this.newReferencesSinceLastRun = new Map();
|
|
164
|
-
this.tombstones = [];
|
|
165
|
-
// Map of node ids to their unreferenced state tracker.
|
|
166
|
-
this.unreferencedNodesState = new Map();
|
|
167
|
-
// Keeps track of unreferenced events that are logged for a node. This is used to limit the log generation to one
|
|
168
|
-
// per event per node.
|
|
169
|
-
this.loggedUnreferencedEvents = new Set();
|
|
170
|
-
// Queue for unreferenced events that should be logged the next time GC runs.
|
|
171
|
-
this.pendingEventsQueue = [];
|
|
172
|
-
// The number of times GC has successfully completed on this instance of GarbageCollector.
|
|
173
|
-
this.completedRuns = 0;
|
|
174
|
-
this.runtime = createParams.runtime;
|
|
175
|
-
this.isSummarizerClient = createParams.isSummarizerClient;
|
|
176
|
-
this.gcOptions = createParams.gcOptions;
|
|
177
|
-
this.createContainerMetadata = createParams.createContainerMetadata;
|
|
178
|
-
this.getNodePackagePath = createParams.getNodePackagePath;
|
|
179
|
-
this.getLastSummaryTimestampMs = createParams.getLastSummaryTimestampMs;
|
|
180
|
-
this.activeConnection = createParams.activeConnection;
|
|
181
|
-
const baseSnapshot = createParams.baseSnapshot;
|
|
182
|
-
const metadata = createParams.metadata;
|
|
183
|
-
const readAndParseBlob = createParams.readAndParseBlob;
|
|
184
|
-
this.mc = (0, telemetry_utils_1.loggerToMonitoringContext)(telemetry_utils_1.ChildLogger.create(createParams.baseLogger, "GarbageCollector", { all: { completedGCRuns: () => this.completedRuns } }));
|
|
185
|
-
this.sweepReadyUsageHandler = new gcSweepReadyUsageDetection_1.SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
|
|
186
|
-
let prevSummaryGCVersion;
|
|
187
|
-
/**
|
|
188
|
-
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
189
|
-
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
190
|
-
*
|
|
191
|
-
* The snapshot cache expiry timeout cannot be known precisely but the upper bound is 5 days.
|
|
192
|
-
* The buffer is added to account for any clock skew or other edge cases.
|
|
193
|
-
* We use server timestamps throughout so the skew should be minimal but make it 1 day to be safe.
|
|
194
|
-
*/
|
|
195
|
-
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
196
|
-
const maxSnapshotCacheExpiryMs = 5 * garbageCollectionConstants_1.oneDayMs;
|
|
197
|
-
const bufferMs = garbageCollectionConstants_1.oneDayMs;
|
|
198
|
-
return sessionExpiryTimeoutMs &&
|
|
199
|
-
(sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
200
|
-
}
|
|
201
|
-
/**
|
|
202
|
-
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
203
|
-
* 1. Whether running GC mark phase is allowed or not.
|
|
204
|
-
* 2. Whether running GC sweep phase is allowed or not.
|
|
205
|
-
* 3. Whether GC session expiry is enabled or not.
|
|
206
|
-
* For existing containers, we get this information from the metadata blob of its summary.
|
|
207
|
-
*/
|
|
208
|
-
if (createParams.existing) {
|
|
209
|
-
prevSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
|
|
210
|
-
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
211
|
-
// other existing documents, GC is enabled.
|
|
212
|
-
this.gcEnabled = prevSummaryGCVersion > 0;
|
|
213
|
-
this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
|
|
214
|
-
this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
|
|
215
|
-
this.sweepTimeoutMs =
|
|
216
|
-
(_b = metadata === null || metadata === void 0 ? void 0 : metadata.sweepTimeoutMs) !== null && _b !== void 0 ? _b : computeSweepTimeout(this.sessionExpiryTimeoutMs); // Backfill old documents that didn't persist this
|
|
217
|
-
}
|
|
218
|
-
else {
|
|
219
|
-
// Sweep should not be enabled without enabling GC mark phase. We could silently disable sweep in this
|
|
220
|
-
// scenario but explicitly failing makes it clearer and promotes correct usage.
|
|
221
|
-
if (this.gcOptions.sweepAllowed && this.gcOptions.gcAllowed === false) {
|
|
222
|
-
throw new container_utils_1.UsageError("GC sweep phase cannot be enabled without enabling GC mark phase");
|
|
223
|
-
}
|
|
224
|
-
// This Test Override only applies for new containers
|
|
225
|
-
const testOverrideSweepTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SweepTimeoutMs");
|
|
226
|
-
// For new documents, GC is enabled by default. It can be explicitly disabled by setting the gcAllowed
|
|
227
|
-
// flag in GC options to false.
|
|
228
|
-
this.gcEnabled = this.gcOptions.gcAllowed !== false;
|
|
229
|
-
// The sweep phase has to be explicitly enabled by setting the sweepAllowed flag in GC options to true.
|
|
230
|
-
// ...unless we're using the TestOverride
|
|
231
|
-
this.sweepEnabled = this.gcOptions.sweepAllowed === true || testOverrideSweepTimeoutMs !== undefined;
|
|
232
|
-
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
233
|
-
if (this.mc.config.getBoolean(garbageCollectionConstants_1.runSessionExpiryKey) && this.gcEnabled) {
|
|
234
|
-
this.sessionExpiryTimeoutMs = (_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : garbageCollectionConstants_1.defaultSessionExpiryDurationMs;
|
|
235
|
-
}
|
|
236
|
-
this.sweepTimeoutMs =
|
|
237
|
-
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
238
|
-
}
|
|
239
|
-
// If session expiry is enabled, we need to close the container when the session expiry timeout expires.
|
|
240
|
-
if (this.sessionExpiryTimeoutMs !== undefined) {
|
|
241
|
-
// If Test Override config is set, override Session Expiry timeout.
|
|
242
|
-
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
243
|
-
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
|
|
244
|
-
this.sessionExpiryTimer = new common_utils_1.Timer(timeoutMs, () => { this.runtime.closeFn(new container_utils_1.ClientSessionExpiredError(`Client session expired.`, timeoutMs)); });
|
|
245
|
-
this.sessionExpiryTimer.start();
|
|
246
|
-
}
|
|
247
|
-
// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
|
|
248
|
-
// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
|
|
249
|
-
this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
|
|
250
|
-
/**
|
|
251
|
-
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
252
|
-
*
|
|
253
|
-
* 1. GC should be enabled for this container.
|
|
254
|
-
*
|
|
255
|
-
* 2. GC should not be disabled via disableGC GC option.
|
|
256
|
-
*
|
|
257
|
-
* These conditions can be overridden via runGCKey feature flag.
|
|
258
|
-
*/
|
|
259
|
-
this.shouldRunGC = (_d = this.mc.config.getBoolean(garbageCollectionConstants_1.runGCKey)) !== null && _d !== void 0 ? _d : (
|
|
260
|
-
// GC must be enabled for the document.
|
|
261
|
-
this.gcEnabled
|
|
262
|
-
// GC must not be disabled via GC options.
|
|
263
|
-
&& !this.gcOptions.disableGC);
|
|
264
|
-
/**
|
|
265
|
-
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
266
|
-
*
|
|
267
|
-
* 1. Overall GC or mark phase must be enabled (this.shouldRunGC).
|
|
268
|
-
* 2. Sweep timeout should be available. Without this, we wouldn't know when an object should be deleted.
|
|
269
|
-
* 3. The driver must implement the policy limiting the age of snapshots used for loading. Otherwise
|
|
270
|
-
* the Sweep Timeout calculation is not valid. We use the persisted value to ensure consistency over time.
|
|
271
|
-
* 4. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
|
|
272
|
-
* feature flag.
|
|
273
|
-
*/
|
|
274
|
-
this.shouldRunSweep =
|
|
275
|
-
this.shouldRunGC
|
|
276
|
-
&& this.sweepTimeoutMs !== undefined
|
|
277
|
-
&& ((_e = this.mc.config.getBoolean(garbageCollectionConstants_1.runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
278
|
-
this.trackGCState = this.mc.config.getBoolean(garbageCollectionConstants_1.trackGCStateKey) === true;
|
|
279
|
-
// Override inactive timeout if test config or gc options to override it is set.
|
|
280
|
-
this.inactiveTimeoutMs = (_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : garbageCollectionConstants_1.defaultInactiveTimeoutMs;
|
|
281
|
-
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
282
|
-
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
283
|
-
throw new container_utils_1.UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
284
|
-
}
|
|
285
|
-
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
286
|
-
this.testMode = (_h = this.mc.config.getBoolean(garbageCollectionConstants_1.gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
287
|
-
// Whether we are running in tombstone mode. This is true by default unless disabled via feature flags.
|
|
288
|
-
this.tombstoneMode = this.mc.config.getBoolean(garbageCollectionConstants_1.disableTombstoneKey) !== true;
|
|
289
|
-
// The GC state needs to be reset if the base snapshot contains GC tree and GC is disabled or it doesn't
|
|
290
|
-
// contain GC tree and GC is enabled.
|
|
291
|
-
const gcTreePresent = (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey]) !== undefined;
|
|
292
|
-
this.initialStateNeedsReset = gcTreePresent !== this.shouldRunGC;
|
|
293
|
-
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
294
|
-
// it involves fetching blobs from storage which is expensive.
|
|
295
|
-
this.baseSnapshotDataP = new common_utils_1.LazyPromise(async () => {
|
|
296
|
-
var _a;
|
|
297
|
-
if (baseSnapshot === undefined) {
|
|
298
|
-
return undefined;
|
|
299
|
-
}
|
|
300
|
-
try {
|
|
301
|
-
// For newer documents, GC data should be present in the GC tree in the root of the snapshot.
|
|
302
|
-
const gcSnapshotTree = baseSnapshot.trees[garbageCollectionConstants_1.gcTreeKey];
|
|
303
|
-
if (gcSnapshotTree !== undefined) {
|
|
304
|
-
return getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
305
|
-
}
|
|
306
|
-
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
307
|
-
// consolidate into IGarbageCollectionState format.
|
|
308
|
-
// Add a node for the root node that is not present in older snapshot format.
|
|
309
|
-
const gcState = { gcNodes: { "/": { outboundRoutes: [] } } };
|
|
310
|
-
const dataStoreSnapshotTree = (0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata);
|
|
311
|
-
(0, common_utils_1.assert)(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
312
|
-
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
313
|
-
const blobId = dsSnapshotTree.blobs[runtime_definitions_1.gcBlobKey];
|
|
314
|
-
if (blobId === undefined) {
|
|
315
|
-
continue;
|
|
316
|
-
}
|
|
317
|
-
const gcSummaryDetails = await readAndParseBlob(blobId);
|
|
318
|
-
// If there are no nodes for this data store, skip it.
|
|
319
|
-
if (((_a = gcSummaryDetails.gcData) === null || _a === void 0 ? void 0 : _a.gcNodes) === undefined) {
|
|
320
|
-
continue;
|
|
321
|
-
}
|
|
322
|
-
const dsRootId = `/${dsId}`;
|
|
323
|
-
// Since we used to write GC data at data store level, we won't have an entry for the root ("/").
|
|
324
|
-
// Construct that entry by adding root data store ids to its outbound routes.
|
|
325
|
-
const initialSnapshotDetails = await readAndParseBlob(dsSnapshotTree.blobs[summaryFormat_1.dataStoreAttributesBlobName]);
|
|
326
|
-
if (initialSnapshotDetails.isRootDataStore) {
|
|
327
|
-
gcState.gcNodes["/"].outboundRoutes.push(dsRootId);
|
|
328
|
-
}
|
|
329
|
-
for (const [id, outboundRoutes] of Object.entries(gcSummaryDetails.gcData.gcNodes)) {
|
|
330
|
-
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
331
|
-
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
332
|
-
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
333
|
-
gcState.gcNodes[rootId] = { outboundRoutes: Array.from(outboundRoutes) };
|
|
334
|
-
}
|
|
335
|
-
(0, common_utils_1.assert)(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
|
|
336
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs = gcSummaryDetails.unrefTimestamp;
|
|
337
|
-
}
|
|
338
|
-
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
339
|
-
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
340
|
-
return Object.keys(gcState.gcNodes).length === 1 ? undefined : { gcState, tombstones: undefined };
|
|
341
|
-
}
|
|
342
|
-
catch (error) {
|
|
343
|
-
const dpe = container_utils_1.DataProcessingError.wrapIfUnrecognized(error, "FailedToInitializeGC");
|
|
344
|
-
dpe.addTelemetryProperties({ gcConfigs: JSON.stringify(this.configs) });
|
|
345
|
-
throw dpe;
|
|
346
|
-
}
|
|
347
|
-
});
|
|
348
|
-
/**
|
|
349
|
-
* Set up the initializer which initializes the GC state from the data in base snapshot. This is done when
|
|
350
|
-
* connected in write mode or when GC runs the first time. It sets up all unreferenced nodes from the base
|
|
351
|
-
* GC state and updates their inactive or sweep ready state.
|
|
352
|
-
*/
|
|
353
|
-
this.initializeGCStateFromBaseSnapshotP = new common_utils_1.LazyPromise(async () => {
|
|
354
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
355
|
-
/**
|
|
356
|
-
* If there is no current reference timestamp, skip initialization. We need the current timestamp to track
|
|
357
|
-
* how long objects have been unreferenced and if they can be deleted.
|
|
358
|
-
*
|
|
359
|
-
* Note that the only scenario where there is no reference timestamp is when no ops have ever been processed
|
|
360
|
-
* for this container and it is in read mode. In this scenario, there is no point in running GC anyway
|
|
361
|
-
* because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
362
|
-
*/
|
|
363
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
364
|
-
// Log an event so we can evaluate how often we run into this scenario.
|
|
365
|
-
this.mc.logger.sendErrorEvent({
|
|
366
|
-
eventName: "GarbageCollectorInitializedWithoutTimestamp",
|
|
367
|
-
gcConfigs: JSON.stringify(this.configs),
|
|
368
|
-
});
|
|
369
|
-
return;
|
|
370
|
-
}
|
|
371
|
-
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
372
|
-
/**
|
|
373
|
-
* The base snapshot data will not be present if the container is loaded from:
|
|
374
|
-
* 1. The first summary created by the detached container.
|
|
375
|
-
* 2. A summary that was generated with GC disabled.
|
|
376
|
-
* 3. A summary that was generated before GC even existed.
|
|
377
|
-
*/
|
|
378
|
-
if (baseSnapshotData === undefined) {
|
|
379
|
-
return;
|
|
380
|
-
}
|
|
381
|
-
const gcNodes = {};
|
|
382
|
-
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
383
|
-
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
384
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(nodeData.unreferencedTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
385
|
-
}
|
|
386
|
-
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
387
|
-
}
|
|
388
|
-
this.previousGCDataFromLastRun = { gcNodes };
|
|
389
|
-
// If tracking state across summaries, update latest summary data from the base snapshot's GC data.
|
|
390
|
-
if (this.trackGCState) {
|
|
391
|
-
this.latestSummaryData = {
|
|
392
|
-
serializedGCState: JSON.stringify(generateSortedGCState(baseSnapshotData.gcState)),
|
|
393
|
-
serializedTombstones: JSON.stringify(baseSnapshotData.tombstones),
|
|
394
|
-
};
|
|
395
|
-
}
|
|
396
|
-
});
|
|
397
|
-
// Get the GC details for each node from the GC state in the base summary. This is returned in getBaseGCDetails
|
|
398
|
-
// which the caller uses to initialize each node's GC state.
|
|
399
|
-
this.baseGCDetailsP = new common_utils_1.LazyPromise(async () => {
|
|
400
|
-
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
401
|
-
if (baseSnapshotData === undefined) {
|
|
402
|
-
return new Map();
|
|
403
|
-
}
|
|
404
|
-
const gcNodes = {};
|
|
405
|
-
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
406
|
-
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
407
|
-
}
|
|
408
|
-
// Run GC on the nodes in the base summary to get the routes used in each node in the container.
|
|
409
|
-
// This is an optimization for space (vs performance) wherein we don't need to store the used routes of
|
|
410
|
-
// each node in the summary.
|
|
411
|
-
const usedRoutes = (0, garbage_collector_1.runGarbageCollection)(gcNodes, ["/"]).referencedNodeIds;
|
|
412
|
-
const baseGCDetailsMap = (0, garbage_collector_1.unpackChildNodesGCDetails)({ gcData: { gcNodes }, usedRoutes });
|
|
413
|
-
// Currently, the nodes may write the GC data. So, we need to update its base GC details with the
|
|
414
|
-
// unreferenced timestamp. Once we start writing the GC data here, we won't need to do this anymore.
|
|
415
|
-
for (const [nodeId, nodeData] of Object.entries(baseSnapshotData.gcState.gcNodes)) {
|
|
416
|
-
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
417
|
-
const dataStoreGCDetails = baseGCDetailsMap.get(nodeId.slice(1));
|
|
418
|
-
if (dataStoreGCDetails !== undefined) {
|
|
419
|
-
dataStoreGCDetails.unrefTimestamp = nodeData.unreferencedTimestampMs;
|
|
420
|
-
}
|
|
421
|
-
}
|
|
422
|
-
}
|
|
423
|
-
return baseGCDetailsMap;
|
|
424
|
-
});
|
|
425
|
-
// Log all the GC options and the state determined by the garbage collector. This is interesting only for the
|
|
426
|
-
// summarizer client since it is the only one that runs GC. It also helps keep the telemetry less noisy.
|
|
427
|
-
if (this.isSummarizerClient) {
|
|
428
|
-
this.mc.logger.sendTelemetryEvent({
|
|
429
|
-
eventName: "GarbageCollectorLoaded",
|
|
430
|
-
gcConfigs: JSON.stringify(this.configs),
|
|
431
|
-
});
|
|
432
|
-
}
|
|
433
|
-
}
|
|
434
|
-
static create(createParams) {
|
|
435
|
-
return new GarbageCollector(createParams);
|
|
436
|
-
}
|
|
437
|
-
/**
|
|
438
|
-
* Tells whether the GC state needs to be reset in the next summary. We need to do this if:
|
|
439
|
-
*
|
|
440
|
-
* 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
|
|
441
|
-
*
|
|
442
|
-
* 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
|
|
443
|
-
*
|
|
444
|
-
* 3. The GC version in the latest summary is different from the current GC version. This can happen if:
|
|
445
|
-
*
|
|
446
|
-
* 3.1. The summary this client loaded with has data from a different GC version.
|
|
447
|
-
*
|
|
448
|
-
* 3.2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
449
|
-
*/
|
|
450
|
-
get summaryStateNeedsReset() {
|
|
451
|
-
return this.initialStateNeedsReset ||
|
|
452
|
-
(this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
|
|
453
|
-
}
|
|
454
|
-
/** Returns a list of all the configurations for garbage collection. */
|
|
455
|
-
get configs() {
|
|
456
|
-
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
457
|
-
}
|
|
458
|
-
/**
|
|
459
|
-
* Called during container initialization. Initialize the tombstone state so that object are marked as tombstones
|
|
460
|
-
* before they are loaded or used. This is important to get accurate information of whether tombstoned object are
|
|
461
|
-
* in use or not.
|
|
462
|
-
*/
|
|
463
|
-
async initializeBaseState() {
|
|
464
|
-
const baseSnapshotData = await this.baseSnapshotDataP;
|
|
465
|
-
/**
|
|
466
|
-
* The base snapshot data or tombstone state will not be present if the container is loaded from:
|
|
467
|
-
* 1. The first summary created by the detached container.
|
|
468
|
-
* 2. A summary that was generated with GC disabled.
|
|
469
|
-
* 3. A summary that was generated before GC even existed.
|
|
470
|
-
* 4. A summary that was generated with tombstone feature disabled.
|
|
471
|
-
*/
|
|
472
|
-
if (!this.tombstoneMode || (baseSnapshotData === null || baseSnapshotData === void 0 ? void 0 : baseSnapshotData.tombstones) === undefined) {
|
|
473
|
-
return;
|
|
474
|
-
}
|
|
475
|
-
this.tombstones = baseSnapshotData.tombstones;
|
|
476
|
-
this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
|
|
477
|
-
}
|
|
478
|
-
/**
|
|
479
|
-
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
480
|
-
* to initialize the base state for non-summarizer clients so that they can track inactive / sweep ready nodes.
|
|
481
|
-
* @param connected - Whether the runtime connected / disconnected.
|
|
482
|
-
* @param clientId - The clientId of this runtime.
|
|
483
|
-
*/
|
|
484
|
-
setConnectionState(connected, clientId) {
|
|
485
|
-
/**
|
|
486
|
-
* For all clients, initialize the base state when the container becomes active, i.e., it transitions
|
|
487
|
-
* to "write" mode. This will ensure that the container's own join op is processed and there is a recent
|
|
488
|
-
* reference timestamp that will be used to update the state of unreferenced nodes. Also, all trailing ops which
|
|
489
|
-
* could affect the GC state will have been processed.
|
|
490
|
-
*
|
|
491
|
-
* If GC is up-to-date for the client and the summarizing client, there will be an doubling of both
|
|
492
|
-
* InactiveObject_Loaded and SweepReady_Loaded errors, as there will be one from the sending client and one from
|
|
493
|
-
* the receiving summarizer client.
|
|
494
|
-
*
|
|
495
|
-
* Ideally, this initialization should only be done for summarizer client. However, we are currently rolling out
|
|
496
|
-
* sweep in phases and we want to track when inactive and sweep ready objects are used in any client.
|
|
497
|
-
*/
|
|
498
|
-
if (this.activeConnection() && this.shouldRunGC) {
|
|
499
|
-
this.initializeGCStateFromBaseSnapshotP.catch((error) => { });
|
|
500
|
-
}
|
|
501
|
-
}
|
|
502
|
-
/**
|
|
503
|
-
* Runs garbage collection and updates the reference / used state of the nodes in the container.
|
|
504
|
-
* @returns stats of the GC run or undefined if GC did not run.
|
|
505
|
-
*/
|
|
506
|
-
async collectGarbage(options) {
|
|
507
|
-
var _a;
|
|
508
|
-
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true || this.summaryStateNeedsReset);
|
|
509
|
-
const logger = options.logger
|
|
510
|
-
? telemetry_utils_1.ChildLogger.create(options.logger, undefined, { all: { completedGCRuns: () => this.completedRuns } })
|
|
511
|
-
: this.mc.logger;
|
|
512
|
-
/**
|
|
513
|
-
* If there is no current reference timestamp, skip running GC. We need the current timestamp to track
|
|
514
|
-
* how long objects have been unreferenced and if they should be deleted.
|
|
515
|
-
*
|
|
516
|
-
* Note that the only scenario where GC is called and there is no reference timestamp is when no ops have ever
|
|
517
|
-
* been processed for this container and it is in read mode. In this scenario, there is no point in running GC
|
|
518
|
-
* anyway because references in the container do not change without any ops, i.e., there is nothing to collect.
|
|
519
|
-
*/
|
|
520
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
521
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
522
|
-
// Log an event so we can evaluate how often we run into this scenario.
|
|
523
|
-
logger.sendErrorEvent({
|
|
524
|
-
eventName: "CollectGarbageCalledWithoutTimestamp",
|
|
525
|
-
gcConfigs: JSON.stringify(this.configs),
|
|
526
|
-
});
|
|
527
|
-
return undefined;
|
|
528
|
-
}
|
|
529
|
-
return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
530
|
-
await this.runPreGCSteps();
|
|
531
|
-
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
532
|
-
const gcData = await this.runtime.getGCData(fullGC);
|
|
533
|
-
const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcData.gcNodes, ["/"]);
|
|
534
|
-
const gcStats = await this.runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs);
|
|
535
|
-
event.end(Object.assign(Object.assign({}, gcStats), { timestamp: currentReferenceTimestampMs }));
|
|
536
|
-
this.completedRuns++;
|
|
537
|
-
return gcStats;
|
|
538
|
-
}, { end: true, cancel: "error" });
|
|
539
|
-
}
|
|
540
|
-
async runPreGCSteps() {
|
|
541
|
-
// Ensure that state has been initialized from the base snapshot data.
|
|
542
|
-
await this.initializeGCStateFromBaseSnapshotP;
|
|
543
|
-
// Let the runtime update its pending state before GC runs.
|
|
544
|
-
await this.runtime.updateStateBeforeGC();
|
|
545
|
-
}
|
|
546
|
-
async runPostGCSteps(gcData, gcResult, logger, currentReferenceTimestampMs) {
|
|
547
|
-
// Generate statistics from the current run. This is done before updating the current state because it
|
|
548
|
-
// generates some of its data based on previous state of the system.
|
|
549
|
-
const gcStats = this.generateStats(gcResult);
|
|
550
|
-
// Update the state since the last GC run. There can be nodes that were referenced between the last and
|
|
551
|
-
// the current run. We need to identify than and update their unreferenced state if needed.
|
|
552
|
-
this.updateStateSinceLastRun(gcData, logger);
|
|
553
|
-
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
554
|
-
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
555
|
-
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
556
|
-
// Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
|
|
557
|
-
// delete these objects here instead.
|
|
558
|
-
this.logSweepEvents(logger, currentReferenceTimestampMs);
|
|
559
|
-
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
560
|
-
// involving access to deleted data.
|
|
561
|
-
if (this.testMode) {
|
|
562
|
-
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds, false /* tombstone */);
|
|
563
|
-
}
|
|
564
|
-
else if (this.tombstoneMode) {
|
|
565
|
-
// If we are running in GC tombstone mode, tombstone objects for unused routes. This enables testing
|
|
566
|
-
// scenarios involving access to "deleted" data without actually deleting the data from summaries.
|
|
567
|
-
// Note: we will not tombstone in test mode
|
|
568
|
-
this.runtime.updateUnusedRoutes(this.tombstones, true /* tombstone */);
|
|
569
|
-
}
|
|
570
|
-
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
571
|
-
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
572
|
-
// reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
|
|
573
|
-
await this.logUnreferencedEvents(logger);
|
|
574
|
-
return gcStats;
|
|
575
|
-
}
|
|
576
|
-
/**
|
|
577
|
-
* Summarizes the GC data and returns it as a summary tree.
|
|
578
|
-
* We current write the entire GC state in a single blob. This can be modified later to write multiple
|
|
579
|
-
* blobs. All the blob keys should start with `gcBlobPrefix`.
|
|
580
|
-
*/
|
|
581
|
-
summarize(fullTree, trackState, telemetryContext) {
|
|
582
|
-
var _a;
|
|
583
|
-
if (!this.shouldRunGC || this.previousGCDataFromLastRun === undefined) {
|
|
584
|
-
return;
|
|
585
|
-
}
|
|
586
|
-
const gcState = { gcNodes: {} };
|
|
587
|
-
for (const [nodeId, outboundRoutes] of Object.entries(this.previousGCDataFromLastRun.gcNodes)) {
|
|
588
|
-
gcState.gcNodes[nodeId] = {
|
|
589
|
-
outboundRoutes,
|
|
590
|
-
unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
|
|
591
|
-
};
|
|
592
|
-
}
|
|
593
|
-
const serializedGCState = JSON.stringify(generateSortedGCState(gcState));
|
|
594
|
-
const serializedTombstones = this.tombstoneMode
|
|
595
|
-
? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
|
|
596
|
-
: undefined;
|
|
597
|
-
/**
|
|
598
|
-
* Incremental summary of GC data - If any of the GC state or tombstone state hasn't changed since the last
|
|
599
|
-
* summary, send summary handles for them. Otherwise, send the data in summary blobs.
|
|
600
|
-
*/
|
|
601
|
-
if (this.trackGCState) {
|
|
602
|
-
this.pendingSummaryData = { serializedGCState, serializedTombstones };
|
|
603
|
-
if (trackState && !fullTree && this.latestSummaryData !== undefined) {
|
|
604
|
-
// If neither GC state or tombstone state changed, send a summary handle for the entire GC data.
|
|
605
|
-
if (this.latestSummaryData.serializedGCState === serializedGCState
|
|
606
|
-
&& this.latestSummaryData.serializedTombstones === serializedTombstones) {
|
|
607
|
-
const stats = (0, runtime_utils_1.mergeStats)();
|
|
608
|
-
stats.handleNodeCount++;
|
|
609
|
-
return {
|
|
610
|
-
summary: {
|
|
611
|
-
type: protocol_definitions_1.SummaryType.Handle,
|
|
612
|
-
handle: `/${garbageCollectionConstants_1.gcTreeKey}`,
|
|
613
|
-
handleType: protocol_definitions_1.SummaryType.Tree,
|
|
614
|
-
},
|
|
615
|
-
stats,
|
|
616
|
-
};
|
|
617
|
-
}
|
|
618
|
-
// If either or both of GC state or tombstone state changed, build a GC summary tree.
|
|
619
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, true /* trackState */);
|
|
620
|
-
}
|
|
621
|
-
}
|
|
622
|
-
// If not tracking GC state, build a GC summary tree without any summary handles.
|
|
623
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, false /* trackState */);
|
|
624
|
-
}
|
|
625
|
-
/**
|
|
626
|
-
* Builds the GC summary tree which contains GC state and tombstone state.
|
|
627
|
-
* If trackState is false, both GC state and tombstone state are written as summary blobs.
|
|
628
|
-
* If trackState is true, summary blob is written for GC state or tombstone state if they changed.
|
|
629
|
-
* @param serializedGCState - The GC state serialized as string.
|
|
630
|
-
* @param serializedTombstones - THe tombstone state serialized as string.
|
|
631
|
-
* @param trackState - Whether we are tracking GC state across summaries.
|
|
632
|
-
* @returns the GC summary tree.
|
|
633
|
-
*/
|
|
634
|
-
buildGCSummaryTree(serializedGCState, serializedTombstones, trackState) {
|
|
635
|
-
var _a, _b;
|
|
636
|
-
const gcStateBlobKey = `${garbageCollectionConstants_1.gcBlobPrefix}_root`;
|
|
637
|
-
const builder = new runtime_utils_1.SummaryTreeBuilder();
|
|
638
|
-
// If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
639
|
-
if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
|
|
640
|
-
builder.addHandle(gcStateBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${gcStateBlobKey}`);
|
|
641
|
-
}
|
|
642
|
-
else {
|
|
643
|
-
builder.addBlob(gcStateBlobKey, serializedGCState);
|
|
644
|
-
}
|
|
645
|
-
// If there is no tombstone data, return only the GC state.
|
|
646
|
-
if (serializedTombstones === undefined) {
|
|
647
|
-
return builder.getSummaryTree();
|
|
648
|
-
}
|
|
649
|
-
// If the tombstone state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
650
|
-
if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
|
|
651
|
-
builder.addHandle(garbageCollectionConstants_1.gcTombstoneBlobKey, protocol_definitions_1.SummaryType.Blob, `/${garbageCollectionConstants_1.gcTreeKey}/${garbageCollectionConstants_1.gcTombstoneBlobKey}`);
|
|
652
|
-
}
|
|
653
|
-
else {
|
|
654
|
-
builder.addBlob(garbageCollectionConstants_1.gcTombstoneBlobKey, serializedTombstones);
|
|
655
|
-
}
|
|
656
|
-
return builder.getSummaryTree();
|
|
657
|
-
}
|
|
658
|
-
getMetadata() {
|
|
659
|
-
return {
|
|
660
|
-
/**
|
|
661
|
-
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
662
|
-
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
663
|
-
*/
|
|
664
|
-
gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
|
|
665
|
-
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
666
|
-
sweepEnabled: this.sweepEnabled,
|
|
667
|
-
sweepTimeoutMs: this.sweepTimeoutMs,
|
|
668
|
-
};
|
|
669
|
-
}
|
|
670
|
-
/**
|
|
671
|
-
* Returns a map of node ids to their base GC details generated from the base summary. This is used by the caller
|
|
672
|
-
* to initialize the GC state of the nodes.
|
|
673
|
-
*/
|
|
674
|
-
async getBaseGCDetails() {
|
|
675
|
-
return this.baseGCDetailsP;
|
|
676
|
-
}
|
|
677
|
-
/**
|
|
678
|
-
* Called when the latest summary of the system has been refreshed. This will be used to update the state of the
|
|
679
|
-
* latest summary tracked.
|
|
680
|
-
*/
|
|
681
|
-
async latestSummaryStateRefreshed(result, readAndParseBlob) {
|
|
682
|
-
if (!this.shouldRunGC || !result.latestSummaryUpdated) {
|
|
683
|
-
return;
|
|
684
|
-
}
|
|
685
|
-
// If the summary was tracked by this client, it was the one that generated the summary in the first place.
|
|
686
|
-
// Basically, it was written in the current GC version.
|
|
687
|
-
if (result.wasSummaryTracked) {
|
|
688
|
-
this.latestSummaryGCVersion = this.currentGCVersion;
|
|
689
|
-
this.initialStateNeedsReset = false;
|
|
690
|
-
if (this.trackGCState) {
|
|
691
|
-
this.latestSummaryData = this.pendingSummaryData;
|
|
692
|
-
this.pendingSummaryData = undefined;
|
|
693
|
-
}
|
|
694
|
-
return;
|
|
695
|
-
}
|
|
696
|
-
// If the summary was not tracked by this client, update latest GC version and blob from the snapshot in the
|
|
697
|
-
// result as that is now the latest summary.
|
|
698
|
-
const snapshot = result.snapshot;
|
|
699
|
-
const metadataBlobId = snapshot.blobs[summaryFormat_1.metadataBlobName];
|
|
700
|
-
if (metadataBlobId) {
|
|
701
|
-
const metadata = await readAndParseBlob(metadataBlobId);
|
|
702
|
-
this.latestSummaryGCVersion = (0, summaryFormat_1.getGCVersion)(metadata);
|
|
703
|
-
}
|
|
704
|
-
const gcSnapshotTree = snapshot.trees[garbageCollectionConstants_1.gcTreeKey];
|
|
705
|
-
if (gcSnapshotTree !== undefined && this.trackGCState) {
|
|
706
|
-
const latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
707
|
-
this.latestSummaryData = {
|
|
708
|
-
serializedGCState: JSON.stringify(generateSortedGCState(latestGCData.gcState)),
|
|
709
|
-
serializedTombstones: JSON.stringify(latestGCData.tombstones),
|
|
710
|
-
};
|
|
711
|
-
}
|
|
712
|
-
else {
|
|
713
|
-
this.latestSummaryData = undefined;
|
|
714
|
-
}
|
|
715
|
-
this.pendingSummaryData = undefined;
|
|
716
|
-
}
|
|
717
|
-
/**
|
|
718
|
-
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
719
|
-
* @param nodePath - The id of the node that changed.
|
|
720
|
-
* @param reason - Whether the node was loaded or changed.
|
|
721
|
-
* @param timestampMs - The timestamp when the node changed.
|
|
722
|
-
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
723
|
-
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
724
|
-
*/
|
|
725
|
-
nodeUpdated(nodePath, reason, timestampMs, packagePath, requestHeaders) {
|
|
726
|
-
if (!this.shouldRunGC) {
|
|
727
|
-
return;
|
|
728
|
-
}
|
|
729
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodePath);
|
|
730
|
-
if (nodeStateTracker && nodeStateTracker.state !== exports.UnreferencedState.Active) {
|
|
731
|
-
this.inactiveNodeUsed(reason, nodePath, nodeStateTracker, undefined /* fromNodeId */, packagePath, timestampMs, requestHeaders);
|
|
732
|
-
}
|
|
733
|
-
}
|
|
734
|
-
/**
|
|
735
|
-
* Called when an outbound reference is added to a node. This is used to identify all nodes that have been
|
|
736
|
-
* referenced between summaries so that their unreferenced timestamp can be reset.
|
|
737
|
-
*
|
|
738
|
-
* @param fromNodePath - The node from which the reference is added.
|
|
739
|
-
* @param toNodePath - The node to which the reference is added.
|
|
740
|
-
*/
|
|
741
|
-
addedOutboundReference(fromNodePath, toNodePath) {
|
|
742
|
-
var _a;
|
|
743
|
-
if (!this.shouldRunGC) {
|
|
744
|
-
return;
|
|
745
|
-
}
|
|
746
|
-
const outboundRoutes = (_a = this.newReferencesSinceLastRun.get(fromNodePath)) !== null && _a !== void 0 ? _a : [];
|
|
747
|
-
outboundRoutes.push(toNodePath);
|
|
748
|
-
this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
749
|
-
const nodeStateTracker = this.unreferencedNodesState.get(toNodePath);
|
|
750
|
-
if (nodeStateTracker && nodeStateTracker.state !== exports.UnreferencedState.Active) {
|
|
751
|
-
this.inactiveNodeUsed("Revived", toNodePath, nodeStateTracker, fromNodePath);
|
|
752
|
-
}
|
|
753
|
-
}
|
|
754
|
-
dispose() {
|
|
755
|
-
var _a;
|
|
756
|
-
(_a = this.sessionExpiryTimer) === null || _a === void 0 ? void 0 : _a.clear();
|
|
757
|
-
this.sessionExpiryTimer = undefined;
|
|
758
|
-
}
|
|
759
|
-
/**
|
|
760
|
-
* Updates the state of the system as per the current GC run. It does the following:
|
|
761
|
-
* 1. Sets up the current GC state as per the gcData.
|
|
762
|
-
* 2. Starts tracking for nodes that have become unreferenced in this run.
|
|
763
|
-
* 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
|
|
764
|
-
* @param gcData - The data representing the reference graph on which GC is run.
|
|
765
|
-
* @param gcResult - The result of the GC run on the gcData.
|
|
766
|
-
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
767
|
-
*/
|
|
768
|
-
updateCurrentState(gcData, gcResult, currentReferenceTimestampMs) {
|
|
769
|
-
this.previousGCDataFromLastRun = (0, garbage_collector_1.cloneGCData)(gcData);
|
|
770
|
-
this.tombstones = [];
|
|
771
|
-
this.newReferencesSinceLastRun.clear();
|
|
772
|
-
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
773
|
-
for (const nodeId of gcResult.referencedNodeIds) {
|
|
774
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
775
|
-
if (nodeStateTracker !== undefined) {
|
|
776
|
-
// Stop tracking so as to clear out any running timers.
|
|
777
|
-
nodeStateTracker.stopTracking();
|
|
778
|
-
// Delete the node as we don't need to track it any more.
|
|
779
|
-
this.unreferencedNodesState.delete(nodeId);
|
|
780
|
-
}
|
|
781
|
-
}
|
|
782
|
-
/**
|
|
783
|
-
* If a node became unreferenced in this run, start tracking it.
|
|
784
|
-
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
785
|
-
* from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
|
|
786
|
-
*/
|
|
787
|
-
for (const nodeId of gcResult.deletedNodeIds) {
|
|
788
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
789
|
-
if (nodeStateTracker === undefined) {
|
|
790
|
-
this.unreferencedNodesState.set(nodeId, new UnreferencedStateTracker(currentReferenceTimestampMs, this.inactiveTimeoutMs, currentReferenceTimestampMs, this.sweepTimeoutMs));
|
|
791
|
-
}
|
|
792
|
-
else {
|
|
793
|
-
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
794
|
-
if (this.tombstoneMode && nodeStateTracker.state === exports.UnreferencedState.SweepReady) {
|
|
795
|
-
const nodeType = this.runtime.getNodeType(nodeId);
|
|
796
|
-
if (nodeType === exports.GCNodeType.DataStore || nodeType === exports.GCNodeType.Blob) {
|
|
797
|
-
this.tombstones.push(nodeId);
|
|
798
|
-
}
|
|
799
|
-
}
|
|
800
|
-
}
|
|
801
|
-
}
|
|
802
|
-
}
|
|
803
|
-
/**
|
|
804
|
-
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
805
|
-
* time. There can be nodes that were referenced in between two runs and their unreferenced state needs to be
|
|
806
|
-
* updated. For example, in the following scenarios not updating the unreferenced timestamp can lead to deletion of
|
|
807
|
-
* these objects while there can be in-memory referenced to it:
|
|
808
|
-
* 1. A node transitions from `unreferenced -> referenced -> unreferenced` between two runs. When the reference is
|
|
809
|
-
* added, the object may have been accessed and in-memory reference to it added.
|
|
810
|
-
* 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
|
|
811
|
-
* unreferenced, they could have been accessed and in-memory reference to them added.
|
|
812
|
-
*
|
|
813
|
-
* This function identifies nodes that were referenced since last run and removes their unreferenced state, if any.
|
|
814
|
-
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
815
|
-
*/
|
|
816
|
-
updateStateSinceLastRun(currentGCData, logger) {
|
|
817
|
-
// If we haven't run GC before there is nothing to do.
|
|
818
|
-
if (this.previousGCDataFromLastRun === undefined) {
|
|
819
|
-
return;
|
|
820
|
-
}
|
|
821
|
-
// Find any references that haven't been identified correctly.
|
|
822
|
-
const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, this.previousGCDataFromLastRun, this.newReferencesSinceLastRun);
|
|
823
|
-
if (missingExplicitReferences.length > 0) {
|
|
824
|
-
missingExplicitReferences.forEach((missingExplicitReference) => {
|
|
825
|
-
const event = {
|
|
826
|
-
eventName: "gcUnknownOutboundReferences",
|
|
827
|
-
gcNodeId: missingExplicitReference[0],
|
|
828
|
-
gcRoutes: JSON.stringify(missingExplicitReference[1]),
|
|
829
|
-
};
|
|
830
|
-
logger.sendPerformanceEvent(event);
|
|
831
|
-
});
|
|
832
|
-
}
|
|
833
|
-
// No references were added since the last run so we don't have to update reference states of any unreferenced
|
|
834
|
-
// nodes
|
|
835
|
-
if (this.newReferencesSinceLastRun.size === 0) {
|
|
836
|
-
return;
|
|
837
|
-
}
|
|
838
|
-
/**
|
|
839
|
-
* Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
|
|
840
|
-
* edges that have been added since then. To do this, combine the GC data from the last run and the current
|
|
841
|
-
* run, and then add the references since last run.
|
|
842
|
-
*
|
|
843
|
-
* Note on why we need to combine the data from previous run, current run and all references in between -
|
|
844
|
-
* 1. We need data from last run because some of its references may have been deleted since then. If those
|
|
845
|
-
* references added new outbound references before they were deleted, we need to detect them.
|
|
846
|
-
*
|
|
847
|
-
* 2. We need new outbound references since last run because some of them may have been deleted later. If those
|
|
848
|
-
* references added new outbound references before they were deleted, we need to detect them.
|
|
849
|
-
*
|
|
850
|
-
* 3. We need data from the current run because currently we may not detect when DDSes are referenced:
|
|
851
|
-
* - We don't require DDSes handles to be stored in a referenced DDS.
|
|
852
|
-
* - A new data store may have "root" DDSes already created and we don't detect them today.
|
|
853
|
-
*/
|
|
854
|
-
const gcDataSuperSet = (0, garbage_collector_1.concatGarbageCollectionData)(this.previousGCDataFromLastRun, currentGCData);
|
|
855
|
-
const newOutboundRoutesSinceLastRun = [];
|
|
856
|
-
this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
|
|
857
|
-
if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
|
|
858
|
-
gcDataSuperSet.gcNodes[sourceNodeId] = outboundRoutes;
|
|
859
|
-
}
|
|
860
|
-
else {
|
|
861
|
-
gcDataSuperSet.gcNodes[sourceNodeId].push(...outboundRoutes);
|
|
862
|
-
}
|
|
863
|
-
newOutboundRoutesSinceLastRun.push(...outboundRoutes);
|
|
864
|
-
});
|
|
865
|
-
/**
|
|
866
|
-
* Run GC on the above reference graph starting with root and all new outbound routes. This will generate a
|
|
867
|
-
* list of all nodes that could have been referenced since the last run. If any of these nodes are unreferenced,
|
|
868
|
-
* unreferenced, stop tracking them and remove from unreferenced list.
|
|
869
|
-
* Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
|
|
870
|
-
* unreferenced and add unreferenced state.
|
|
871
|
-
*/
|
|
872
|
-
const gcResult = (0, garbage_collector_1.runGarbageCollection)(gcDataSuperSet.gcNodes, ["/", ...newOutboundRoutesSinceLastRun]);
|
|
873
|
-
for (const nodeId of gcResult.referencedNodeIds) {
|
|
874
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
875
|
-
if (nodeStateTracker !== undefined) {
|
|
876
|
-
// Stop tracking so as to clear out any running timers.
|
|
877
|
-
nodeStateTracker.stopTracking();
|
|
878
|
-
// Delete the unreferenced state as we don't need to track it any more.
|
|
879
|
-
this.unreferencedNodesState.delete(nodeId);
|
|
880
|
-
}
|
|
881
|
-
}
|
|
882
|
-
}
|
|
883
|
-
/**
|
|
884
|
-
* Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
|
|
885
|
-
* The principle is that every new reference or outbound route must be notified to GC via the
|
|
886
|
-
* addedOutboundReference method. It it hasn't, its a bug and we want to identify these scenarios.
|
|
887
|
-
*
|
|
888
|
-
* In more simple terms:
|
|
889
|
-
* Missing Explicit References = Current References - Previous References - Explicitly Added References;
|
|
890
|
-
*
|
|
891
|
-
* @param currentGCData - The GC data (reference graph) from the current GC run.
|
|
892
|
-
* @param previousGCData - The GC data (reference graph) from the previous GC run.
|
|
893
|
-
* @param explicitReferences - New references added explicity between the previous and the current run.
|
|
894
|
-
* @returns - a list of missing explicit references
|
|
895
|
-
*/
|
|
896
|
-
findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
|
|
897
|
-
(0, common_utils_1.assert)(previousGCData !== undefined, 0x2b7);
|
|
898
|
-
const currentGraph = Object.entries(currentGCData.gcNodes);
|
|
899
|
-
const missingExplicitReferences = [];
|
|
900
|
-
currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
|
|
901
|
-
var _a, _b;
|
|
902
|
-
const previousRoutes = (_a = previousGCData.gcNodes[nodeId]) !== null && _a !== void 0 ? _a : [];
|
|
903
|
-
const explicitRoutes = (_b = explicitReferences.get(nodeId)) !== null && _b !== void 0 ? _b : [];
|
|
904
|
-
const missingExplicitRoutes = [];
|
|
905
|
-
currentOutboundRoutes.forEach((route) => {
|
|
906
|
-
const isBlobOrDataStoreRoute = this.runtime.getNodeType(route) === exports.GCNodeType.Blob ||
|
|
907
|
-
this.runtime.getNodeType(route) === exports.GCNodeType.DataStore;
|
|
908
|
-
// Ignore implicitly added DDS routes to their parent datastores
|
|
909
|
-
const notRouteFromDDSToParentDataStore = !nodeId.startsWith(route);
|
|
910
|
-
if (isBlobOrDataStoreRoute &&
|
|
911
|
-
notRouteFromDDSToParentDataStore &&
|
|
912
|
-
(!previousRoutes.includes(route) && !explicitRoutes.includes(route))) {
|
|
913
|
-
missingExplicitRoutes.push(route);
|
|
914
|
-
}
|
|
915
|
-
});
|
|
916
|
-
if (missingExplicitRoutes.length > 0) {
|
|
917
|
-
missingExplicitReferences.push([nodeId, missingExplicitRoutes]);
|
|
918
|
-
}
|
|
919
|
-
});
|
|
920
|
-
// Ideally missingExplicitReferences should always have a size 0
|
|
921
|
-
return missingExplicitReferences;
|
|
922
|
-
}
|
|
923
|
-
/**
|
|
924
|
-
* Generates the stats of a garbage collection run from the given results of the run.
|
|
925
|
-
* @param gcResult - The result of a GC run.
|
|
926
|
-
* @returns the GC stats of the GC run.
|
|
927
|
-
*/
|
|
928
|
-
generateStats(gcResult) {
|
|
929
|
-
const gcStats = {
|
|
930
|
-
nodeCount: 0,
|
|
931
|
-
dataStoreCount: 0,
|
|
932
|
-
attachmentBlobCount: 0,
|
|
933
|
-
unrefNodeCount: 0,
|
|
934
|
-
unrefDataStoreCount: 0,
|
|
935
|
-
unrefAttachmentBlobCount: 0,
|
|
936
|
-
updatedNodeCount: 0,
|
|
937
|
-
updatedDataStoreCount: 0,
|
|
938
|
-
updatedAttachmentBlobCount: 0,
|
|
939
|
-
};
|
|
940
|
-
const updateNodeStats = (nodeId, referenced) => {
|
|
941
|
-
gcStats.nodeCount++;
|
|
942
|
-
// If there is no previous GC data, every node's state is generated and is considered as updated.
|
|
943
|
-
// Otherwise, find out if any node went from referenced to unreferenced or vice-versa.
|
|
944
|
-
const stateUpdated = this.previousGCDataFromLastRun === undefined ||
|
|
945
|
-
this.unreferencedNodesState.has(nodeId) === referenced;
|
|
946
|
-
if (stateUpdated) {
|
|
947
|
-
gcStats.updatedNodeCount++;
|
|
948
|
-
}
|
|
949
|
-
if (!referenced) {
|
|
950
|
-
gcStats.unrefNodeCount++;
|
|
951
|
-
}
|
|
952
|
-
if (this.runtime.getNodeType(nodeId) === exports.GCNodeType.DataStore) {
|
|
953
|
-
gcStats.dataStoreCount++;
|
|
954
|
-
if (stateUpdated) {
|
|
955
|
-
gcStats.updatedDataStoreCount++;
|
|
956
|
-
}
|
|
957
|
-
if (!referenced) {
|
|
958
|
-
gcStats.unrefDataStoreCount++;
|
|
959
|
-
}
|
|
960
|
-
}
|
|
961
|
-
if (this.runtime.getNodeType(nodeId) === exports.GCNodeType.Blob) {
|
|
962
|
-
gcStats.attachmentBlobCount++;
|
|
963
|
-
if (stateUpdated) {
|
|
964
|
-
gcStats.updatedAttachmentBlobCount++;
|
|
965
|
-
}
|
|
966
|
-
if (!referenced) {
|
|
967
|
-
gcStats.unrefAttachmentBlobCount++;
|
|
968
|
-
}
|
|
969
|
-
}
|
|
970
|
-
};
|
|
971
|
-
for (const nodeId of gcResult.referencedNodeIds) {
|
|
972
|
-
updateNodeStats(nodeId, true /* referenced */);
|
|
973
|
-
}
|
|
974
|
-
for (const nodeId of gcResult.deletedNodeIds) {
|
|
975
|
-
updateNodeStats(nodeId, false /* referenced */);
|
|
976
|
-
}
|
|
977
|
-
return gcStats;
|
|
978
|
-
}
|
|
979
|
-
/**
|
|
980
|
-
* For nodes that are ready to sweep, log an event for now. Until we start running sweep which deletes objects,
|
|
981
|
-
* this will give us a view into how much deleted content a container has.
|
|
982
|
-
*/
|
|
983
|
-
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
984
|
-
if (this.mc.config.getBoolean(garbageCollectionConstants_1.disableSweepLogKey) === true || this.sweepTimeoutMs === undefined) {
|
|
985
|
-
return;
|
|
986
|
-
}
|
|
987
|
-
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
988
|
-
if (nodeStateTracker.state !== exports.UnreferencedState.SweepReady) {
|
|
989
|
-
return;
|
|
990
|
-
}
|
|
991
|
-
const nodeType = this.runtime.getNodeType(nodeId);
|
|
992
|
-
if (nodeType !== exports.GCNodeType.DataStore && nodeType !== exports.GCNodeType.Blob) {
|
|
993
|
-
return;
|
|
994
|
-
}
|
|
995
|
-
// Log deleted event for each node only once to reduce noise in telemetry.
|
|
996
|
-
const uniqueEventId = `Deleted-${nodeId}`;
|
|
997
|
-
if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
998
|
-
return;
|
|
999
|
-
}
|
|
1000
|
-
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
1001
|
-
logger.sendTelemetryEvent({
|
|
1002
|
-
eventName: "GCObjectDeleted",
|
|
1003
|
-
id: nodeId,
|
|
1004
|
-
type: nodeType,
|
|
1005
|
-
age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs,
|
|
1006
|
-
timeout: this.sweepTimeoutMs,
|
|
1007
|
-
completedGCRuns: this.completedRuns,
|
|
1008
|
-
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1009
|
-
});
|
|
1010
|
-
});
|
|
1011
|
-
}
|
|
1012
|
-
/**
|
|
1013
|
-
* Called when an inactive node is used after. Queue up an event that will be logged next time GC runs.
|
|
1014
|
-
*/
|
|
1015
|
-
inactiveNodeUsed(usageType, nodeId, nodeStateTracker, fromNodeId, packagePath, currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs(), requestHeaders) {
|
|
1016
|
-
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
1017
|
-
// logging as nothing interesting would have happened worth logging.
|
|
1018
|
-
// If the node is active, skip logging.
|
|
1019
|
-
if (currentReferenceTimestampMs === undefined || nodeStateTracker.state === exports.UnreferencedState.Active) {
|
|
1020
|
-
return;
|
|
1021
|
-
}
|
|
1022
|
-
// We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
|
|
1023
|
-
// as unreferenced. Also, if an inactive DDS is used, the corresponding data store store will also be used.
|
|
1024
|
-
const nodeType = this.runtime.getNodeType(nodeId);
|
|
1025
|
-
if (nodeType !== exports.GCNodeType.DataStore && nodeType !== exports.GCNodeType.Blob) {
|
|
1026
|
-
return;
|
|
1027
|
-
}
|
|
1028
|
-
const state = nodeStateTracker.state;
|
|
1029
|
-
const uniqueEventId = `${state}-${nodeId}-${usageType}`;
|
|
1030
|
-
if (this.loggedUnreferencedEvents.has(uniqueEventId)) {
|
|
1031
|
-
return;
|
|
1032
|
-
}
|
|
1033
|
-
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
1034
|
-
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === exports.UnreferencedState.Inactive
|
|
1035
|
-
? this.inactiveTimeoutMs
|
|
1036
|
-
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { externalRequest: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.externalRequest], viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[containerRuntime_1.RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
1037
|
-
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
1038
|
-
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
1039
|
-
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|
|
1040
|
-
// Inactive errors are usages of Objects that are unreferenced for at least a period of 7 days.
|
|
1041
|
-
// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
|
|
1042
|
-
if (this.isSummarizerClient) {
|
|
1043
|
-
this.pendingEventsQueue.push(Object.assign(Object.assign({}, propsToLog), { usageType, state }));
|
|
1044
|
-
}
|
|
1045
|
-
else {
|
|
1046
|
-
// For non-summarizer clients, only log "Loaded" type events since these objects may not be loaded in the
|
|
1047
|
-
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
1048
|
-
// Events generated:
|
|
1049
|
-
// InactiveObject_Loaded, SweepReadyObject_Loaded
|
|
1050
|
-
if (usageType === "Loaded") {
|
|
1051
|
-
this.mc.logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: (0, runtime_utils_1.packagePathToTelemetryProperty)(packagePath), stack: (0, telemetry_utils_1.generateStack)() }));
|
|
1052
|
-
}
|
|
1053
|
-
// If SweepReady Usage Detection is enabed, the handler may close the interactive container.
|
|
1054
|
-
// Once Sweep is fully implemented, this will be removed since the objects will be gone
|
|
1055
|
-
// and errors will arise elsewhere in the runtime
|
|
1056
|
-
if (state === exports.UnreferencedState.SweepReady) {
|
|
1057
|
-
this.sweepReadyUsageHandler.usageDetectedInInteractiveClient(Object.assign(Object.assign({}, propsToLog), { usageType }));
|
|
1058
|
-
}
|
|
1059
|
-
}
|
|
1060
|
-
}
|
|
1061
|
-
async logUnreferencedEvents(logger) {
|
|
1062
|
-
// Events sent come only from the summarizer client. In between summaries, events are pushed to a queue and at
|
|
1063
|
-
// summary time they are then logged.
|
|
1064
|
-
// Events generated:
|
|
1065
|
-
// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
|
|
1066
|
-
// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
|
|
1067
|
-
for (const eventProps of this.pendingEventsQueue) {
|
|
1068
|
-
const { usageType, state } = eventProps, propsToLog = __rest(eventProps, ["usageType", "state"]);
|
|
1069
|
-
/**
|
|
1070
|
-
* Revived event is logged only if the node is active. If the node is not active, the reference to it was
|
|
1071
|
-
* from another unreferenced node and this scenario is not interesting to log.
|
|
1072
|
-
* Loaded and Changed events are logged only if the node is not active. If the node is active, it was
|
|
1073
|
-
* revived and a Revived event will be logged for it.
|
|
1074
|
-
*/
|
|
1075
|
-
const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
|
|
1076
|
-
const active = nodeStateTracker === undefined || nodeStateTracker.state === exports.UnreferencedState.Active;
|
|
1077
|
-
if ((usageType === "Revived") === active) {
|
|
1078
|
-
const pkg = await this.getNodePackagePath(eventProps.id);
|
|
1079
|
-
const fromPkg = eventProps.fromId ? await this.getNodePackagePath(eventProps.fromId) : undefined;
|
|
1080
|
-
logger.sendErrorEvent(Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg ? { value: pkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined, fromPkg: fromPkg ? { value: fromPkg.join("/"), tag: telemetry_utils_1.TelemetryDataTag.CodeArtifact } : undefined }));
|
|
1081
|
-
}
|
|
1082
|
-
}
|
|
1083
|
-
this.pendingEventsQueue = [];
|
|
1084
|
-
}
|
|
1085
|
-
}
|
|
1086
|
-
exports.GarbageCollector = GarbageCollector;
|
|
1087
|
-
/**
|
|
1088
|
-
* Gets the base garbage collection state from the given snapshot tree. It contains GC state and tombstone state.
|
|
1089
|
-
* The GC state may be written into multiple blobs. Merge the GC state from all such blobs into one.
|
|
1090
|
-
*/
|
|
1091
|
-
async function getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob) {
|
|
1092
|
-
let rootGCState = { gcNodes: {} };
|
|
1093
|
-
let tombstones;
|
|
1094
|
-
for (const key of Object.keys(gcSnapshotTree.blobs)) {
|
|
1095
|
-
if (key === garbageCollectionConstants_1.gcTombstoneBlobKey) {
|
|
1096
|
-
tombstones = await readAndParseBlob(gcSnapshotTree.blobs[key]);
|
|
1097
|
-
continue;
|
|
1098
|
-
}
|
|
1099
|
-
// Skip blobs that do not start with the GC prefix.
|
|
1100
|
-
if (!key.startsWith(garbageCollectionConstants_1.gcBlobPrefix)) {
|
|
1101
|
-
continue;
|
|
1102
|
-
}
|
|
1103
|
-
const blobId = gcSnapshotTree.blobs[key];
|
|
1104
|
-
if (blobId === undefined) {
|
|
1105
|
-
continue;
|
|
1106
|
-
}
|
|
1107
|
-
const gcState = await readAndParseBlob(blobId);
|
|
1108
|
-
(0, common_utils_1.assert)(gcState !== undefined, 0x2ad /* "GC blob missing from snapshot" */);
|
|
1109
|
-
// Merge the GC state of this blob into the root GC state.
|
|
1110
|
-
rootGCState = (0, garbage_collector_1.concatGarbageCollectionStates)(rootGCState, gcState);
|
|
1111
|
-
}
|
|
1112
|
-
return { gcState: rootGCState, tombstones };
|
|
1113
|
-
}
|
|
1114
|
-
function generateSortedGCState(gcState) {
|
|
1115
|
-
const sortableArray = Object.entries(gcState.gcNodes);
|
|
1116
|
-
sortableArray.sort(([a], [b]) => a.localeCompare(b));
|
|
1117
|
-
const sortedGCState = { gcNodes: {} };
|
|
1118
|
-
for (const [nodeId, nodeData] of sortableArray) {
|
|
1119
|
-
nodeData.outboundRoutes.sort();
|
|
1120
|
-
sortedGCState.gcNodes[nodeId] = nodeData;
|
|
1121
|
-
}
|
|
1122
|
-
return sortedGCState;
|
|
1123
|
-
}
|
|
1124
|
-
/** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
|
|
1125
|
-
class TimerWithNoDefaultTimeout extends common_utils_1.Timer {
|
|
1126
|
-
constructor(callback) {
|
|
1127
|
-
// The default timeout/handlers will never be used since start/restart pass overrides below
|
|
1128
|
-
super(0, () => { throw new Error("DefaultHandler should not be used"); });
|
|
1129
|
-
this.callback = callback;
|
|
1130
|
-
}
|
|
1131
|
-
start(timeoutMs) {
|
|
1132
|
-
super.start(timeoutMs, this.callback);
|
|
1133
|
-
}
|
|
1134
|
-
restart(timeoutMs) {
|
|
1135
|
-
super.restart(timeoutMs, this.callback);
|
|
1136
|
-
}
|
|
1137
|
-
}
|
|
1138
|
-
//# sourceMappingURL=garbageCollection.js.map
|