@fluidframework/container-runtime 2.0.0-internal.3.0.2 → 2.0.0-internal.3.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +19 -19
- package/.mocharc.js +2 -2
- package/api-extractor.json +2 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +2 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +15 -2
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +109 -37
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +11 -9
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerHandleContext.d.ts.map +1 -1
- package/dist/containerHandleContext.js +3 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +23 -11
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +225 -132
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +11 -9
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +27 -13
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +95 -56
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +7 -3
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts.map +1 -1
- package/dist/dataStoreRegistry.js +3 -1
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +28 -4
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +122 -44
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +8 -3
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/{garbageCollection.d.ts → gc/garbageCollection.d.ts} +27 -203
- package/dist/gc/garbageCollection.d.ts.map +1 -0
- package/dist/{garbageCollection.js → gc/garbageCollection.js} +210 -400
- package/dist/gc/garbageCollection.js.map +1 -0
- package/dist/gc/gcDefinitions.d.ts +189 -0
- package/dist/gc/gcDefinitions.d.ts.map +1 -0
- package/dist/{garbageCollectionConstants.js → gc/gcDefinitions.js} +27 -2
- package/dist/gc/gcDefinitions.js.map +1 -0
- package/dist/gc/gcHelpers.d.ts +30 -0
- package/dist/gc/gcHelpers.d.ts.map +1 -0
- package/dist/gc/gcHelpers.js +65 -0
- package/dist/gc/gcHelpers.js.map +1 -0
- package/dist/gc/gcSummaryStateTracker.d.ts +86 -0
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/dist/gc/gcSummaryStateTracker.js +246 -0
- package/dist/gc/gcSummaryStateTracker.js.map +1 -0
- package/{lib → dist/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
- package/dist/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/dist/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
- package/dist/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/dist/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/dist/gc/gcUnreferencedStateTracker.js +94 -0
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/dist/gc/index.d.ts +11 -0
- package/dist/gc/index.d.ts.map +1 -0
- package/dist/gc/index.js +40 -0
- package/dist/gc/index.js.map +1 -0
- package/dist/index.d.ts +2 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -9
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +2 -13
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +19 -41
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +4 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +1 -0
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +5 -2
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +1 -1
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +24 -13
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +19 -3
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +78 -45
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.d.ts.map +1 -1
- package/dist/opProperties.js +1 -3
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +8 -2
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +21 -13
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +3 -2
- package/dist/scheduleManager.js.map +1 -1
- package/dist/serializedSnapshotStorage.d.ts +2 -2
- package/dist/serializedSnapshotStorage.d.ts.map +1 -1
- package/dist/serializedSnapshotStorage.js +5 -3
- package/dist/serializedSnapshotStorage.js.map +1 -1
- package/dist/summary/index.d.ts +17 -0
- package/dist/summary/index.d.ts.map +1 -0
- package/dist/summary/index.js +47 -0
- package/dist/summary/index.js.map +1 -0
- package/dist/summary/orderedClientElection.d.ts.map +1 -0
- package/dist/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
- package/dist/summary/orderedClientElection.js.map +1 -0
- package/dist/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{lib → dist/summary}/runningSummarizer.d.ts +19 -18
- package/dist/summary/runningSummarizer.d.ts.map +1 -0
- package/dist/{runningSummarizer.js → summary/runningSummarizer.js} +191 -76
- package/dist/summary/runningSummarizer.js.map +1 -0
- package/dist/{summarizer.d.ts → summary/summarizer.d.ts} +4 -6
- package/dist/summary/summarizer.d.ts.map +1 -0
- package/dist/{summarizer.js → summary/summarizer.js} +31 -71
- package/dist/summary/summarizer.js.map +1 -0
- package/dist/summary/summarizerClientElection.d.ts.map +1 -0
- package/dist/summary/summarizerClientElection.js.map +1 -0
- package/dist/summary/summarizerHandle.d.ts.map +1 -0
- package/dist/summary/summarizerHandle.js.map +1 -0
- package/{lib → dist/summary}/summarizerHeuristics.d.ts +1 -1
- package/dist/summary/summarizerHeuristics.d.ts.map +1 -0
- package/dist/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -9
- package/dist/summary/summarizerHeuristics.js.map +1 -0
- package/{lib → dist/summary}/summarizerTypes.d.ts +22 -22
- package/dist/summary/summarizerTypes.d.ts.map +1 -0
- package/dist/summary/summarizerTypes.js.map +1 -0
- package/dist/summary/summaryCollection.d.ts.map +1 -0
- package/dist/{summaryCollection.js → summary/summaryCollection.js} +18 -8
- package/dist/summary/summaryCollection.js.map +1 -0
- package/{lib → dist/summary}/summaryFormat.d.ts +1 -40
- package/dist/summary/summaryFormat.d.ts.map +1 -0
- package/dist/{summaryFormat.js → summary/summaryFormat.js} +19 -20
- package/dist/summary/summaryFormat.js.map +1 -0
- package/dist/summary/summaryGenerator.d.ts.map +1 -0
- package/dist/{summaryGenerator.js → summary/summaryGenerator.js} +33 -15
- package/dist/summary/summaryGenerator.js.map +1 -0
- package/dist/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/dist/summary/summaryManager.d.ts.map +1 -0
- package/dist/{summaryManager.js → summary/summaryManager.js} +21 -9
- package/dist/summary/summaryManager.js.map +1 -0
- package/dist/throttler.d.ts +2 -2
- package/dist/throttler.d.ts.map +1 -1
- package/dist/throttler.js +4 -4
- package/dist/throttler.js.map +1 -1
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +2 -1
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +15 -2
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +109 -37
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +11 -9
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerHandleContext.d.ts.map +1 -1
- package/lib/containerHandleContext.js +3 -1
- package/lib/containerHandleContext.js.map +1 -1
- package/lib/containerRuntime.d.ts +23 -11
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +201 -108
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +11 -9
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +27 -13
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +84 -45
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +7 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts.map +1 -1
- package/lib/dataStoreRegistry.js +3 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +28 -4
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +120 -42
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +9 -4
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/{garbageCollection.d.ts → gc/garbageCollection.d.ts} +27 -203
- package/lib/gc/garbageCollection.d.ts.map +1 -0
- package/lib/{garbageCollection.js → gc/garbageCollection.js} +190 -379
- package/lib/gc/garbageCollection.js.map +1 -0
- package/lib/gc/gcDefinitions.d.ts +189 -0
- package/lib/gc/gcDefinitions.d.ts.map +1 -0
- package/lib/{garbageCollectionConstants.js → gc/gcDefinitions.js} +26 -1
- package/lib/gc/gcDefinitions.js.map +1 -0
- package/lib/gc/gcHelpers.d.ts +30 -0
- package/lib/gc/gcHelpers.d.ts.map +1 -0
- package/lib/gc/gcHelpers.js +58 -0
- package/lib/gc/gcHelpers.js.map +1 -0
- package/lib/gc/gcSummaryStateTracker.d.ts +86 -0
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -0
- package/lib/gc/gcSummaryStateTracker.js +242 -0
- package/lib/gc/gcSummaryStateTracker.js.map +1 -0
- package/{dist → lib/gc}/gcSweepReadyUsageDetection.d.ts +5 -5
- package/lib/gc/gcSweepReadyUsageDetection.d.ts.map +1 -0
- package/lib/{gcSweepReadyUsageDetection.js → gc/gcSweepReadyUsageDetection.js} +15 -11
- package/lib/gc/gcSweepReadyUsageDetection.js.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts +34 -0
- package/lib/gc/gcUnreferencedStateTracker.d.ts.map +1 -0
- package/lib/gc/gcUnreferencedStateTracker.js +90 -0
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -0
- package/lib/gc/index.d.ts +11 -0
- package/lib/gc/index.d.ts.map +1 -0
- package/lib/gc/index.js +11 -0
- package/lib/gc/index.js.map +1 -0
- package/lib/index.d.ts +2 -5
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -4
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +2 -13
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +19 -41
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +4 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +1 -0
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +5 -2
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +1 -1
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +25 -14
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +19 -3
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +79 -46
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.d.ts.map +1 -1
- package/lib/opProperties.js +1 -3
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +8 -2
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +21 -13
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +3 -2
- package/lib/scheduleManager.js.map +1 -1
- package/lib/serializedSnapshotStorage.d.ts +2 -2
- package/lib/serializedSnapshotStorage.d.ts.map +1 -1
- package/lib/serializedSnapshotStorage.js +5 -3
- package/lib/serializedSnapshotStorage.js.map +1 -1
- package/lib/summary/index.d.ts +17 -0
- package/lib/summary/index.d.ts.map +1 -0
- package/lib/summary/index.js +16 -0
- package/lib/summary/index.js.map +1 -0
- package/lib/summary/orderedClientElection.d.ts.map +1 -0
- package/lib/{orderedClientElection.js → summary/orderedClientElection.js} +10 -4
- package/lib/summary/orderedClientElection.js.map +1 -0
- package/lib/summary/runWhileConnectedCoordinator.d.ts.map +1 -0
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -0
- package/{dist → lib/summary}/runningSummarizer.d.ts +19 -18
- package/lib/summary/runningSummarizer.d.ts.map +1 -0
- package/lib/{runningSummarizer.js → summary/runningSummarizer.js} +193 -78
- package/lib/summary/runningSummarizer.js.map +1 -0
- package/lib/{summarizer.d.ts → summary/summarizer.d.ts} +4 -6
- package/lib/summary/summarizer.d.ts.map +1 -0
- package/lib/{summarizer.js → summary/summarizer.js} +33 -73
- package/lib/summary/summarizer.js.map +1 -0
- package/lib/summary/summarizerClientElection.d.ts.map +1 -0
- package/lib/summary/summarizerClientElection.js.map +1 -0
- package/lib/summary/summarizerHandle.d.ts.map +1 -0
- package/lib/summary/summarizerHandle.js.map +1 -0
- package/{dist → lib/summary}/summarizerHeuristics.d.ts +1 -1
- package/lib/summary/summarizerHeuristics.d.ts.map +1 -0
- package/lib/{summarizerHeuristics.js → summary/summarizerHeuristics.js} +6 -9
- package/lib/summary/summarizerHeuristics.js.map +1 -0
- package/{dist → lib/summary}/summarizerTypes.d.ts +22 -22
- package/lib/summary/summarizerTypes.d.ts.map +1 -0
- package/lib/summary/summarizerTypes.js.map +1 -0
- package/lib/summary/summaryCollection.d.ts.map +1 -0
- package/lib/{summaryCollection.js → summary/summaryCollection.js} +18 -8
- package/lib/summary/summaryCollection.js.map +1 -0
- package/{dist → lib/summary}/summaryFormat.d.ts +1 -40
- package/lib/summary/summaryFormat.d.ts.map +1 -0
- package/lib/{summaryFormat.js → summary/summaryFormat.js} +20 -20
- package/lib/summary/summaryFormat.js.map +1 -0
- package/lib/summary/summaryGenerator.d.ts.map +1 -0
- package/lib/{summaryGenerator.js → summary/summaryGenerator.js} +33 -15
- package/lib/summary/summaryGenerator.js.map +1 -0
- package/lib/{summaryManager.d.ts → summary/summaryManager.d.ts} +1 -1
- package/lib/summary/summaryManager.d.ts.map +1 -0
- package/lib/{summaryManager.js → summary/summaryManager.js} +21 -9
- package/lib/summary/summaryManager.js.map +1 -0
- package/lib/throttler.d.ts +2 -2
- package/lib/throttler.d.ts.map +1 -1
- package/lib/throttler.js +4 -4
- package/lib/throttler.js.map +1 -1
- package/package.json +60 -51
- package/prettier.config.cjs +1 -1
- package/src/batchTracker.ts +54 -49
- package/src/blobManager.ts +825 -674
- package/src/connectionTelemetry.ts +280 -249
- package/src/containerHandleContext.ts +27 -29
- package/src/containerRuntime.ts +3249 -2978
- package/src/dataStore.ts +172 -159
- package/src/dataStoreContext.ts +1141 -1057
- package/src/dataStoreContexts.ts +178 -161
- package/src/dataStoreRegistry.ts +25 -20
- package/src/dataStores.ts +880 -731
- package/src/deltaScheduler.ts +158 -150
- package/{garbageCollection.md → src/gc/garbageCollection.md} +16 -3
- package/src/gc/garbageCollection.ts +1506 -0
- package/src/gc/gcDefinitions.ts +244 -0
- package/src/gc/gcHelpers.ts +86 -0
- package/src/gc/gcSummaryStateTracker.ts +339 -0
- package/src/gc/gcSweepReadyUsageDetection.ts +145 -0
- package/src/gc/gcUnreferencedStateTracker.ts +114 -0
- package/src/gc/index.ts +40 -0
- package/src/index.ts +67 -70
- package/src/opLifecycle/README.md +152 -0
- package/src/opLifecycle/batchManager.ts +101 -144
- package/src/opLifecycle/definitions.ts +33 -29
- package/src/opLifecycle/index.ts +5 -5
- package/src/opLifecycle/opCompressor.ts +55 -53
- package/src/opLifecycle/opDecompressor.ts +100 -81
- package/src/opLifecycle/opSplitter.ts +233 -188
- package/src/opLifecycle/outbox.ts +251 -195
- package/src/opLifecycle/remoteMessageProcessor.ts +62 -62
- package/src/opProperties.ts +11 -9
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +396 -338
- package/src/scheduleManager.ts +299 -269
- package/src/serializedSnapshotStorage.ts +126 -112
- package/src/summary/index.ts +99 -0
- package/src/summary/orderedClientElection.ts +564 -0
- package/src/summary/runWhileConnectedCoordinator.ts +113 -0
- package/src/summary/runningSummarizer.ts +788 -0
- package/src/summary/summarizer.ts +386 -0
- package/src/summary/summarizerClientElection.ts +139 -0
- package/src/{summarizerHandle.ts → summary/summarizerHandle.ts} +11 -9
- package/src/summary/summarizerHeuristics.ts +219 -0
- package/src/summary/summarizerTypes.ts +521 -0
- package/src/summary/summaryCollection.ts +450 -0
- package/src/summary/summaryFormat.ts +226 -0
- package/src/summary/summaryGenerator.ts +505 -0
- package/src/summary/summaryManager.ts +423 -0
- package/src/throttler.ts +131 -122
- package/tsconfig.esnext.json +6 -6
- package/tsconfig.json +9 -13
- package/dist/garbageCollection.d.ts.map +0 -1
- package/dist/garbageCollection.js.map +0 -1
- package/dist/garbageCollectionConstants.d.ts +0 -25
- package/dist/garbageCollectionConstants.d.ts.map +0 -1
- package/dist/garbageCollectionConstants.js.map +0 -1
- package/dist/garbageCollectionTombstoneUtils.d.ts +0 -14
- package/dist/garbageCollectionTombstoneUtils.d.ts.map +0 -1
- package/dist/garbageCollectionTombstoneUtils.js +0 -23
- package/dist/garbageCollectionTombstoneUtils.js.map +0 -1
- package/dist/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/dist/gcSweepReadyUsageDetection.js.map +0 -1
- package/dist/orderedClientElection.d.ts.map +0 -1
- package/dist/orderedClientElection.js.map +0 -1
- package/dist/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/dist/runWhileConnectedCoordinator.js.map +0 -1
- package/dist/runningSummarizer.d.ts.map +0 -1
- package/dist/runningSummarizer.js.map +0 -1
- package/dist/summarizer.d.ts.map +0 -1
- package/dist/summarizer.js.map +0 -1
- package/dist/summarizerClientElection.d.ts.map +0 -1
- package/dist/summarizerClientElection.js.map +0 -1
- package/dist/summarizerHandle.d.ts.map +0 -1
- package/dist/summarizerHandle.js.map +0 -1
- package/dist/summarizerHeuristics.d.ts.map +0 -1
- package/dist/summarizerHeuristics.js.map +0 -1
- package/dist/summarizerTypes.d.ts.map +0 -1
- package/dist/summarizerTypes.js.map +0 -1
- package/dist/summaryCollection.d.ts.map +0 -1
- package/dist/summaryCollection.js.map +0 -1
- package/dist/summaryFormat.d.ts.map +0 -1
- package/dist/summaryFormat.js.map +0 -1
- package/dist/summaryGenerator.d.ts.map +0 -1
- package/dist/summaryGenerator.js.map +0 -1
- package/dist/summaryManager.d.ts.map +0 -1
- package/dist/summaryManager.js.map +0 -1
- package/lib/garbageCollection.d.ts.map +0 -1
- package/lib/garbageCollection.js.map +0 -1
- package/lib/garbageCollectionConstants.d.ts +0 -25
- package/lib/garbageCollectionConstants.d.ts.map +0 -1
- package/lib/garbageCollectionConstants.js.map +0 -1
- package/lib/garbageCollectionTombstoneUtils.d.ts +0 -14
- package/lib/garbageCollectionTombstoneUtils.d.ts.map +0 -1
- package/lib/garbageCollectionTombstoneUtils.js +0 -19
- package/lib/garbageCollectionTombstoneUtils.js.map +0 -1
- package/lib/gcSweepReadyUsageDetection.d.ts.map +0 -1
- package/lib/gcSweepReadyUsageDetection.js.map +0 -1
- package/lib/orderedClientElection.d.ts.map +0 -1
- package/lib/orderedClientElection.js.map +0 -1
- package/lib/runWhileConnectedCoordinator.d.ts.map +0 -1
- package/lib/runWhileConnectedCoordinator.js.map +0 -1
- package/lib/runningSummarizer.d.ts.map +0 -1
- package/lib/runningSummarizer.js.map +0 -1
- package/lib/summarizer.d.ts.map +0 -1
- package/lib/summarizer.js.map +0 -1
- package/lib/summarizerClientElection.d.ts.map +0 -1
- package/lib/summarizerClientElection.js.map +0 -1
- package/lib/summarizerHandle.d.ts.map +0 -1
- package/lib/summarizerHandle.js.map +0 -1
- package/lib/summarizerHeuristics.d.ts.map +0 -1
- package/lib/summarizerHeuristics.js.map +0 -1
- package/lib/summarizerTypes.d.ts.map +0 -1
- package/lib/summarizerTypes.js.map +0 -1
- package/lib/summaryCollection.d.ts.map +0 -1
- package/lib/summaryCollection.js.map +0 -1
- package/lib/summaryFormat.d.ts.map +0 -1
- package/lib/summaryFormat.js.map +0 -1
- package/lib/summaryGenerator.d.ts.map +0 -1
- package/lib/summaryGenerator.js.map +0 -1
- package/lib/summaryManager.d.ts.map +0 -1
- package/lib/summaryManager.js.map +0 -1
- package/src/garbageCollection.ts +0 -1800
- package/src/garbageCollectionConstants.ts +0 -41
- package/src/garbageCollectionTombstoneUtils.ts +0 -28
- package/src/gcSweepReadyUsageDetection.ts +0 -139
- package/src/orderedClientElection.ts +0 -532
- package/src/runWhileConnectedCoordinator.ts +0 -106
- package/src/runningSummarizer.ts +0 -610
- package/src/summarizer.ts +0 -421
- package/src/summarizerClientElection.ts +0 -132
- package/src/summarizerHeuristics.ts +0 -222
- package/src/summarizerTypes.ts +0 -507
- package/src/summaryCollection.ts +0 -421
- package/src/summaryFormat.ts +0 -256
- package/src/summaryGenerator.ts +0 -450
- package/src/summaryManager.ts +0 -394
- /package/dist/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/dist/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +0 -0
- /package/dist/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +0 -0
- /package/dist/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
- /package/dist/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
- /package/dist/{summarizerHandle.d.ts → summary/summarizerHandle.d.ts} +0 -0
- /package/dist/{summarizerHandle.js → summary/summarizerHandle.js} +0 -0
- /package/dist/{summarizerTypes.js → summary/summarizerTypes.js} +0 -0
- /package/dist/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/dist/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
- /package/lib/{orderedClientElection.d.ts → summary/orderedClientElection.d.ts} +0 -0
- /package/lib/{runWhileConnectedCoordinator.d.ts → summary/runWhileConnectedCoordinator.d.ts} +0 -0
- /package/lib/{runWhileConnectedCoordinator.js → summary/runWhileConnectedCoordinator.js} +0 -0
- /package/lib/{summarizerClientElection.d.ts → summary/summarizerClientElection.d.ts} +0 -0
- /package/lib/{summarizerClientElection.js → summary/summarizerClientElection.js} +0 -0
- /package/lib/{summarizerHandle.d.ts → summary/summarizerHandle.d.ts} +0 -0
- /package/lib/{summarizerHandle.js → summary/summarizerHandle.js} +0 -0
- /package/lib/{summarizerTypes.js → summary/summarizerTypes.js} +0 -0
- /package/lib/{summaryCollection.d.ts → summary/summaryCollection.d.ts} +0 -0
- /package/lib/{summaryGenerator.d.ts → summary/summaryGenerator.d.ts} +0 -0
|
@@ -14,105 +14,19 @@ var __rest = (this && this.__rest) || function (s, e) {
|
|
|
14
14
|
return t;
|
|
15
15
|
};
|
|
16
16
|
import { assert, LazyPromise, Timer } from "@fluidframework/common-utils";
|
|
17
|
-
import { ClientSessionExpiredError, DataProcessingError, UsageError } from "@fluidframework/container-utils";
|
|
17
|
+
import { ClientSessionExpiredError, DataProcessingError, UsageError, } from "@fluidframework/container-utils";
|
|
18
18
|
import { cloneGCData, concatGarbageCollectionData, getGCDataFromSnapshot, runGarbageCollection, trimLeadingSlashes, } from "@fluidframework/garbage-collector";
|
|
19
|
-
import {
|
|
20
|
-
import {
|
|
21
|
-
import { mergeStats, packagePathToTelemetryProperty, SummaryTreeBuilder, } from "@fluidframework/runtime-utils";
|
|
19
|
+
import { gcTreeKey, } from "@fluidframework/runtime-definitions";
|
|
20
|
+
import { packagePathToTelemetryProperty, } from "@fluidframework/runtime-utils";
|
|
22
21
|
import { ChildLogger, generateStack, loggerToMonitoringContext, PerformanceEvent, TelemetryDataTag, } from "@fluidframework/telemetry-utils";
|
|
23
|
-
import { RuntimeHeaders } from "
|
|
24
|
-
import { getSummaryForDatastores } from "
|
|
25
|
-
import {
|
|
26
|
-
import {
|
|
22
|
+
import { RuntimeHeaders } from "../containerRuntime";
|
|
23
|
+
import { getSummaryForDatastores } from "../dataStores";
|
|
24
|
+
import { dataStoreAttributesBlobName, } from "../summary";
|
|
25
|
+
import { defaultInactiveTimeoutMs, defaultSessionExpiryDurationMs, disableSweepLogKey, disableTombstoneKey, GCNodeType, gcTestModeKey, oneDayMs, runGCKey, runSessionExpiryKey, runSweepKey, trackGCStateKey, gcTombstoneGenerationOptionName, UnreferencedState, } from "./gcDefinitions";
|
|
26
|
+
import { getGCVersion, sendGCUnexpectedUsageEvent } from "./gcHelpers";
|
|
27
|
+
import { GCSummaryStateTracker } from "./gcSummaryStateTracker";
|
|
27
28
|
import { SweepReadyUsageDetectionHandler } from "./gcSweepReadyUsageDetection";
|
|
28
|
-
import {
|
|
29
|
-
/** The types of GC nodes in the GC reference graph. */
|
|
30
|
-
export const GCNodeType = {
|
|
31
|
-
// Nodes that are for data stores.
|
|
32
|
-
DataStore: "DataStore",
|
|
33
|
-
// Nodes that are within a data store. For example, DDS nodes.
|
|
34
|
-
SubDataStore: "SubDataStore",
|
|
35
|
-
// Nodes that are for attachment blobs, i.e., blobs uploaded via BlobManager.
|
|
36
|
-
Blob: "Blob",
|
|
37
|
-
// Nodes that are neither of the above. For example, root node.
|
|
38
|
-
Other: "Other",
|
|
39
|
-
};
|
|
40
|
-
/** The state of node that is unreferenced. */
|
|
41
|
-
export const UnreferencedState = {
|
|
42
|
-
/** The node is active, i.e., it can become referenced again. */
|
|
43
|
-
Active: "Active",
|
|
44
|
-
/** The node is inactive, i.e., it should not become referenced. */
|
|
45
|
-
Inactive: "Inactive",
|
|
46
|
-
/** The node is ready to be deleted by the sweep phase. */
|
|
47
|
-
SweepReady: "SweepReady",
|
|
48
|
-
};
|
|
49
|
-
/**
|
|
50
|
-
* Helper class that tracks the state of an unreferenced node such as the time it was unreferenced and if it can
|
|
51
|
-
* be deleted by the sweep phase.
|
|
52
|
-
*/
|
|
53
|
-
export class UnreferencedStateTracker {
|
|
54
|
-
constructor(unreferencedTimestampMs,
|
|
55
|
-
/** The time after which node transitions to Inactive state. */
|
|
56
|
-
inactiveTimeoutMs,
|
|
57
|
-
/** The current reference timestamp used to track how long this node has been unreferenced for. */
|
|
58
|
-
currentReferenceTimestampMs,
|
|
59
|
-
/** The time after which node transitions to SweepReady state; undefined if session expiry is disabled. */
|
|
60
|
-
sweepTimeoutMs) {
|
|
61
|
-
this.unreferencedTimestampMs = unreferencedTimestampMs;
|
|
62
|
-
this.inactiveTimeoutMs = inactiveTimeoutMs;
|
|
63
|
-
this.sweepTimeoutMs = sweepTimeoutMs;
|
|
64
|
-
this._state = UnreferencedState.Active;
|
|
65
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
66
|
-
assert(this.inactiveTimeoutMs <= this.sweepTimeoutMs, 0x3b0 /* inactive timeout must not be greater than the sweep timeout */);
|
|
67
|
-
}
|
|
68
|
-
this.sweepTimer = new TimerWithNoDefaultTimeout(() => {
|
|
69
|
-
this._state = UnreferencedState.SweepReady;
|
|
70
|
-
assert(!this.inactiveTimer.hasTimer, 0x3b1 /* inactiveTimer still running after sweepTimer fired! */);
|
|
71
|
-
});
|
|
72
|
-
this.inactiveTimer = new TimerWithNoDefaultTimeout(() => {
|
|
73
|
-
this._state = UnreferencedState.Inactive;
|
|
74
|
-
// After the node becomes inactive, start the sweep timer after which the node will be ready for sweep.
|
|
75
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
76
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - this.inactiveTimeoutMs);
|
|
77
|
-
}
|
|
78
|
-
});
|
|
79
|
-
this.updateTracking(currentReferenceTimestampMs);
|
|
80
|
-
}
|
|
81
|
-
get state() {
|
|
82
|
-
return this._state;
|
|
83
|
-
}
|
|
84
|
-
/* Updates the unreferenced state based on the provided timestamp. */
|
|
85
|
-
updateTracking(currentReferenceTimestampMs) {
|
|
86
|
-
const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
|
|
87
|
-
// If the node has been unreferenced for sweep timeout amount of time, update the state to SweepReady.
|
|
88
|
-
if (this.sweepTimeoutMs !== undefined && unreferencedDurationMs >= this.sweepTimeoutMs) {
|
|
89
|
-
this._state = UnreferencedState.SweepReady;
|
|
90
|
-
this.clearTimers();
|
|
91
|
-
return;
|
|
92
|
-
}
|
|
93
|
-
// If the node has been unreferenced for inactive timeoutMs amount of time, update the state to inactive.
|
|
94
|
-
// Also, start a timer for the sweep timeout.
|
|
95
|
-
if (unreferencedDurationMs >= this.inactiveTimeoutMs) {
|
|
96
|
-
this._state = UnreferencedState.Inactive;
|
|
97
|
-
this.inactiveTimer.clear();
|
|
98
|
-
if (this.sweepTimeoutMs !== undefined) {
|
|
99
|
-
this.sweepTimer.restart(this.sweepTimeoutMs - unreferencedDurationMs);
|
|
100
|
-
}
|
|
101
|
-
return;
|
|
102
|
-
}
|
|
103
|
-
// The node is still active. Ensure the inactive timer is running with the proper remaining duration.
|
|
104
|
-
this.inactiveTimer.restart(this.inactiveTimeoutMs - unreferencedDurationMs);
|
|
105
|
-
}
|
|
106
|
-
clearTimers() {
|
|
107
|
-
this.inactiveTimer.clear();
|
|
108
|
-
this.sweepTimer.clear();
|
|
109
|
-
}
|
|
110
|
-
/** Stop tracking this node. Reset the unreferenced timers and state, if any. */
|
|
111
|
-
stopTracking() {
|
|
112
|
-
this.clearTimers();
|
|
113
|
-
this._state = UnreferencedState.Active;
|
|
114
|
-
}
|
|
115
|
-
}
|
|
29
|
+
import { UnreferencedStateTracker } from "./gcUnreferencedStateTracker";
|
|
116
30
|
/**
|
|
117
31
|
* The garbage collector for the container runtime. It consolidates the garbage collection functionality and maintains
|
|
118
32
|
* its state across summaries.
|
|
@@ -124,15 +38,15 @@ export class UnreferencedStateTracker {
|
|
|
124
38
|
* Graph - all nodes with their respective routes
|
|
125
39
|
*
|
|
126
40
|
* ```
|
|
127
|
-
*
|
|
41
|
+
* GC Graph
|
|
128
42
|
*
|
|
129
|
-
*
|
|
130
|
-
*
|
|
131
|
-
*
|
|
132
|
-
*
|
|
133
|
-
*
|
|
134
|
-
*
|
|
135
|
-
* NodeId = "dds1"
|
|
43
|
+
* Node
|
|
44
|
+
* NodeId = "datastore1"
|
|
45
|
+
* / \\
|
|
46
|
+
* OutboundRoute OutboundRoute
|
|
47
|
+
* / \\
|
|
48
|
+
* Node Node
|
|
49
|
+
* NodeId = "dds1" NodeId = "dds2"
|
|
136
50
|
* ```
|
|
137
51
|
*/
|
|
138
52
|
export class GarbageCollector {
|
|
@@ -164,12 +78,11 @@ export class GarbageCollector {
|
|
|
164
78
|
const baseSnapshot = createParams.baseSnapshot;
|
|
165
79
|
const metadata = createParams.metadata;
|
|
166
80
|
const readAndParseBlob = createParams.readAndParseBlob;
|
|
167
|
-
this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
this.mc.config.getBoolean(gcVersionUpgradeToV2Key) === true ? currentGCVersion : stableGCVersion;
|
|
81
|
+
this.mc = loggerToMonitoringContext(ChildLogger.create(createParams.baseLogger, "GarbageCollector", {
|
|
82
|
+
all: { completedGCRuns: () => this.completedRuns },
|
|
83
|
+
}));
|
|
171
84
|
this.sweepReadyUsageHandler = new SweepReadyUsageDetectionHandler(createParams.getContainerDiagnosticId(), this.mc, this.runtime.closeFn);
|
|
172
|
-
let
|
|
85
|
+
let gcVersionInBaseSnapshot;
|
|
173
86
|
/**
|
|
174
87
|
* Sweep timeout is the time after which unreferenced content can be swept.
|
|
175
88
|
* Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer.
|
|
@@ -181,8 +94,8 @@ export class GarbageCollector {
|
|
|
181
94
|
function computeSweepTimeout(sessionExpiryTimeoutMs) {
|
|
182
95
|
const maxSnapshotCacheExpiryMs = 5 * oneDayMs;
|
|
183
96
|
const bufferMs = oneDayMs;
|
|
184
|
-
return sessionExpiryTimeoutMs &&
|
|
185
|
-
|
|
97
|
+
return (sessionExpiryTimeoutMs &&
|
|
98
|
+
sessionExpiryTimeoutMs + maxSnapshotCacheExpiryMs + bufferMs);
|
|
186
99
|
}
|
|
187
100
|
/**
|
|
188
101
|
* The following GC state is enabled during container creation and cannot be changed throughout its lifetime:
|
|
@@ -192,10 +105,10 @@ export class GarbageCollector {
|
|
|
192
105
|
* For existing containers, we get this information from the metadata blob of its summary.
|
|
193
106
|
*/
|
|
194
107
|
if (createParams.existing) {
|
|
195
|
-
|
|
108
|
+
gcVersionInBaseSnapshot = getGCVersion(metadata);
|
|
196
109
|
// Existing documents which did not have metadata blob or had GC disabled have version as 0. For all
|
|
197
110
|
// other existing documents, GC is enabled.
|
|
198
|
-
this.gcEnabled =
|
|
111
|
+
this.gcEnabled = gcVersionInBaseSnapshot > 0;
|
|
199
112
|
this.sweepEnabled = (_a = metadata === null || metadata === void 0 ? void 0 : metadata.sweepEnabled) !== null && _a !== void 0 ? _a : false;
|
|
200
113
|
this.sessionExpiryTimeoutMs = metadata === null || metadata === void 0 ? void 0 : metadata.sessionExpiryTimeoutMs;
|
|
201
114
|
this.sweepTimeoutMs =
|
|
@@ -217,7 +130,8 @@ export class GarbageCollector {
|
|
|
217
130
|
this.sweepEnabled = this.gcOptions.sweepAllowed === true;
|
|
218
131
|
// Set the Session Expiry only if the flag is enabled and GC is enabled.
|
|
219
132
|
if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
|
|
220
|
-
this.sessionExpiryTimeoutMs =
|
|
133
|
+
this.sessionExpiryTimeoutMs =
|
|
134
|
+
(_c = this.gcOptions.sessionExpiryTimeoutMs) !== null && _c !== void 0 ? _c : defaultSessionExpiryDurationMs;
|
|
221
135
|
}
|
|
222
136
|
this.sweepTimeoutMs =
|
|
223
137
|
testOverrideSweepTimeoutMs !== null && testOverrideSweepTimeoutMs !== void 0 ? testOverrideSweepTimeoutMs : computeSweepTimeout(this.sessionExpiryTimeoutMs);
|
|
@@ -232,12 +146,11 @@ export class GarbageCollector {
|
|
|
232
146
|
// If Test Override config is set, override Session Expiry timeout.
|
|
233
147
|
const overrideSessionExpiryTimeoutMs = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.SessionExpiryMs");
|
|
234
148
|
const timeoutMs = overrideSessionExpiryTimeoutMs !== null && overrideSessionExpiryTimeoutMs !== void 0 ? overrideSessionExpiryTimeoutMs : this.sessionExpiryTimeoutMs;
|
|
235
|
-
this.sessionExpiryTimer = new Timer(timeoutMs, () => {
|
|
149
|
+
this.sessionExpiryTimer = new Timer(timeoutMs, () => {
|
|
150
|
+
this.runtime.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
151
|
+
});
|
|
236
152
|
this.sessionExpiryTimer.start();
|
|
237
153
|
}
|
|
238
|
-
// For existing document, the latest summary is the one that we loaded from. So, use its GC version as the
|
|
239
|
-
// latest tracked GC version. For new documents, we will be writing the first summary with the current version.
|
|
240
|
-
this.latestSummaryGCVersion = prevSummaryGCVersion !== null && prevSummaryGCVersion !== void 0 ? prevSummaryGCVersion : this.currentGCVersion;
|
|
241
154
|
/**
|
|
242
155
|
* Whether GC should run or not. The following conditions have to be met to run sweep:
|
|
243
156
|
*
|
|
@@ -247,11 +160,12 @@ export class GarbageCollector {
|
|
|
247
160
|
*
|
|
248
161
|
* These conditions can be overridden via runGCKey feature flag.
|
|
249
162
|
*/
|
|
250
|
-
this.shouldRunGC =
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
163
|
+
this.shouldRunGC =
|
|
164
|
+
(_d = this.mc.config.getBoolean(runGCKey)) !== null && _d !== void 0 ? _d :
|
|
165
|
+
// GC must be enabled for the document.
|
|
166
|
+
(this.gcEnabled &&
|
|
167
|
+
// GC must not be disabled via GC options.
|
|
168
|
+
!this.gcOptions.disableGC);
|
|
255
169
|
/**
|
|
256
170
|
* Whether sweep should run or not. The following conditions have to be met to run sweep:
|
|
257
171
|
*
|
|
@@ -263,23 +177,25 @@ export class GarbageCollector {
|
|
|
263
177
|
* feature flag.
|
|
264
178
|
*/
|
|
265
179
|
this.shouldRunSweep =
|
|
266
|
-
this.shouldRunGC
|
|
267
|
-
|
|
268
|
-
|
|
180
|
+
this.shouldRunGC &&
|
|
181
|
+
this.sweepTimeoutMs !== undefined &&
|
|
182
|
+
((_e = this.mc.config.getBoolean(runSweepKey)) !== null && _e !== void 0 ? _e : this.sweepEnabled);
|
|
269
183
|
this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
|
|
270
184
|
// Override inactive timeout if test config or gc options to override it is set.
|
|
271
|
-
this.inactiveTimeoutMs =
|
|
185
|
+
this.inactiveTimeoutMs =
|
|
186
|
+
(_g = (_f = this.mc.config.getNumber("Fluid.GarbageCollection.TestOverride.InactiveTimeoutMs")) !== null && _f !== void 0 ? _f : this.gcOptions.inactiveTimeoutMs) !== null && _g !== void 0 ? _g : defaultInactiveTimeoutMs;
|
|
272
187
|
// Inactive timeout must be greater than sweep timeout since a node goes from active -> inactive -> sweep ready.
|
|
273
188
|
if (this.sweepTimeoutMs !== undefined && this.inactiveTimeoutMs > this.sweepTimeoutMs) {
|
|
274
189
|
throw new UsageError("inactive timeout should not be greater than the sweep timeout");
|
|
275
190
|
}
|
|
276
191
|
// Whether we are running in test mode. In this mode, unreferenced nodes are immediately deleted.
|
|
277
|
-
this.testMode =
|
|
192
|
+
this.testMode =
|
|
193
|
+
(_h = this.mc.config.getBoolean(gcTestModeKey)) !== null && _h !== void 0 ? _h : this.gcOptions.runGCInTestMode === true;
|
|
278
194
|
// Whether we are running in tombstone mode. This is enabled by default if sweep won't run. It can be disabled
|
|
279
195
|
// via feature flags.
|
|
280
|
-
this.tombstoneMode =
|
|
281
|
-
|
|
282
|
-
this.
|
|
196
|
+
this.tombstoneMode =
|
|
197
|
+
!this.shouldRunSweep && this.mc.config.getBoolean(disableTombstoneKey) !== true;
|
|
198
|
+
this.summaryStateTracker = new GCSummaryStateTracker(this.shouldRunGC, this.trackGCState, this.tombstoneMode, this.mc, (baseSnapshot === null || baseSnapshot === void 0 ? void 0 : baseSnapshot.trees[gcTreeKey]) !== undefined /* wasGCRunInBaseSnapshot */, gcVersionInBaseSnapshot);
|
|
283
199
|
// Get the GC data from the base snapshot. Use LazyPromise because we only want to do this once since it
|
|
284
200
|
// it involves fetching blobs from storage which is expensive.
|
|
285
201
|
this.baseSnapshotDataP = new LazyPromise(async () => {
|
|
@@ -296,7 +212,9 @@ export class GarbageCollector {
|
|
|
296
212
|
// back-compat - Older documents will have the GC blobs in each data store's summary tree. Get them and
|
|
297
213
|
// consolidate into IGarbageCollectionState format.
|
|
298
214
|
// Add a node for the root node that is not present in older snapshot format.
|
|
299
|
-
const gcState = {
|
|
215
|
+
const gcState = {
|
|
216
|
+
gcNodes: { "/": { outboundRoutes: [] } },
|
|
217
|
+
};
|
|
300
218
|
const dataStoreSnapshotTree = getSummaryForDatastores(baseSnapshot, metadata);
|
|
301
219
|
assert(dataStoreSnapshotTree !== undefined, 0x2a8 /* "Expected data store snapshot tree in base snapshot" */);
|
|
302
220
|
for (const [dsId, dsSnapshotTree] of Object.entries(dataStoreSnapshotTree.trees)) {
|
|
@@ -320,10 +238,13 @@ export class GarbageCollector {
|
|
|
320
238
|
// Prefix the data store id to the GC node ids to make them relative to the root from being
|
|
321
239
|
// relative to the data store. Similar to how its done in DataStore::getGCData.
|
|
322
240
|
const rootId = id === "/" ? dsRootId : `${dsRootId}${id}`;
|
|
323
|
-
gcState.gcNodes[rootId] = {
|
|
241
|
+
gcState.gcNodes[rootId] = {
|
|
242
|
+
outboundRoutes: Array.from(outboundRoutes),
|
|
243
|
+
};
|
|
324
244
|
}
|
|
325
245
|
assert(gcState.gcNodes[dsRootId] !== undefined, 0x2a9 /* GC nodes for data store not in GC blob */);
|
|
326
|
-
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
246
|
+
gcState.gcNodes[dsRootId].unreferencedTimestampMs =
|
|
247
|
+
gcSummaryDetails.unrefTimestamp;
|
|
327
248
|
}
|
|
328
249
|
// If there is only one node (root node just added above), either GC is disabled or we are loading from
|
|
329
250
|
// the first summary generated by detached container. In both cases, GC was not run - return undefined.
|
|
@@ -401,46 +322,13 @@ export class GarbageCollector {
|
|
|
401
322
|
static create(createParams) {
|
|
402
323
|
return new GarbageCollector(createParams);
|
|
403
324
|
}
|
|
404
|
-
/**
|
|
405
|
-
* Tells whether the GC state needs to be reset in the next summary. We need to do this if:
|
|
406
|
-
*
|
|
407
|
-
* 1. GC was enabled and is now disabled. The GC state needs to be removed and everything becomes referenced.
|
|
408
|
-
*
|
|
409
|
-
* 2. GC was disabled and is now enabled. The GC state needs to be regenerated and added to summary.
|
|
410
|
-
*
|
|
411
|
-
* 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
|
|
412
|
-
*
|
|
413
|
-
* 4. The GC version in the latest summary is different from the current GC version. This can happen if:
|
|
414
|
-
*
|
|
415
|
-
* 4.1. The summary this client loaded with has data from a different GC version.
|
|
416
|
-
*
|
|
417
|
-
* 4.2. This client's latest summary was updated from a snapshot that has a different GC version.
|
|
418
|
-
*/
|
|
419
|
-
get summaryStateNeedsReset() {
|
|
420
|
-
return this.gcStateNeedsReset ||
|
|
421
|
-
(this.shouldRunGC && this.latestSummaryGCVersion !== this.currentGCVersion);
|
|
422
|
-
}
|
|
423
|
-
/**
|
|
424
|
-
* Tells whether the GC state needs to be reset. This can happen under 3 conditions:
|
|
425
|
-
*
|
|
426
|
-
* 1. The base snapshot contains GC state but GC is disabled. This will happen the first time GC is disabled after
|
|
427
|
-
* it was enabled before. GC state needs to be removed from summary and all nodes should be marked referenced.
|
|
428
|
-
*
|
|
429
|
-
* 2. The base snapshot does not have GC state but GC is enabled. This will happen the very first time GC runs on
|
|
430
|
-
* a document and the first time GC is enabled after is was disabled before.
|
|
431
|
-
*
|
|
432
|
-
* 3. GC is enabled and the latest summary state is refreshed from a snapshot that had GC disabled and vice-versa.
|
|
433
|
-
*
|
|
434
|
-
* Note that the state will be reset only once for the first summary generated after this returns true. After that,
|
|
435
|
-
* this will return false.
|
|
436
|
-
*/
|
|
437
|
-
get gcStateNeedsReset() {
|
|
438
|
-
return this.wasGCRunInLatestSummary !== this.shouldRunGC;
|
|
439
|
-
}
|
|
440
325
|
/** Returns a list of all the configurations for garbage collection. */
|
|
441
326
|
get configs() {
|
|
442
327
|
return Object.assign({ gcEnabled: this.gcEnabled, sweepEnabled: this.sweepEnabled, runGC: this.shouldRunGC, runSweep: this.shouldRunSweep, testMode: this.testMode, tombstoneMode: this.tombstoneMode, sessionExpiry: this.sessionExpiryTimeoutMs, sweepTimeout: this.sweepTimeoutMs, inactiveTimeout: this.inactiveTimeoutMs, trackGCState: this.trackGCState }, this.gcOptions);
|
|
443
328
|
}
|
|
329
|
+
get summaryStateNeedsReset() {
|
|
330
|
+
return this.summaryStateTracker.doesSummaryStateNeedReset();
|
|
331
|
+
}
|
|
444
332
|
/**
|
|
445
333
|
* Called during container initialization. Initialize from the tombstone state in the base snapshot. This is done
|
|
446
334
|
* during initialization so that deleted or tombstoned objects are marked as such before they are loaded or used.
|
|
@@ -464,6 +352,7 @@ export class GarbageCollector {
|
|
|
464
352
|
// If running in tombstone mode, initialize the tombstone state from the snapshot. Also, notify the runtime of
|
|
465
353
|
// tombstone routes.
|
|
466
354
|
if (this.tombstoneMode && baseSnapshotData.tombstones !== undefined) {
|
|
355
|
+
// Create a copy since we are writing from a source we don't control
|
|
467
356
|
this.tombstones = Array.from(baseSnapshotData.tombstones);
|
|
468
357
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
469
358
|
}
|
|
@@ -491,7 +380,6 @@ export class GarbageCollector {
|
|
|
491
380
|
for (const [, nodeStateTracker] of this.unreferencedNodesState) {
|
|
492
381
|
nodeStateTracker.stopTracking();
|
|
493
382
|
}
|
|
494
|
-
;
|
|
495
383
|
this.unreferencedNodesState.clear();
|
|
496
384
|
// If running sweep, the tombstone state represents the list of nodes that have been deleted during sweep.
|
|
497
385
|
// If running in tombstone mode, the tombstone state represents the list of nodes that have been marked as
|
|
@@ -499,7 +387,9 @@ export class GarbageCollector {
|
|
|
499
387
|
// If this call is because we are refreshing from a snapshot due to an ack, it is likely that the GC state
|
|
500
388
|
// in the snapshot is newer than this client's. And so, the deleted / tombstone nodes need to be updated.
|
|
501
389
|
if (this.shouldRunSweep) {
|
|
502
|
-
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.
|
|
390
|
+
const snapshotDeletedNodes = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.deletedNodes)
|
|
391
|
+
? new Set(snapshotData.deletedNodes)
|
|
392
|
+
: undefined;
|
|
503
393
|
// If the snapshot contains deleted nodes that are not yet deleted by this client, ask the runtime to
|
|
504
394
|
// delete them.
|
|
505
395
|
if (snapshotDeletedNodes !== undefined) {
|
|
@@ -520,10 +410,11 @@ export class GarbageCollector {
|
|
|
520
410
|
this.tombstones = (snapshotData === null || snapshotData === void 0 ? void 0 : snapshotData.tombstones) ? Array.from(snapshotData.tombstones) : [];
|
|
521
411
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
522
412
|
}
|
|
413
|
+
// Update the summary state tracker's state from this snapshot.
|
|
414
|
+
this.summaryStateTracker.updateStateFromSnapshotData(snapshotData);
|
|
523
415
|
// If there is no snapshot data, it means this snapshot was generated with GC disabled. Unset all GC state.
|
|
524
416
|
if (snapshotData === undefined) {
|
|
525
417
|
this.gcDataFromLastRun = undefined;
|
|
526
|
-
this.latestSummaryData = undefined;
|
|
527
418
|
return;
|
|
528
419
|
}
|
|
529
420
|
// Update unreferenced state tracking as per the GC state in the snapshot data and update gcDataFromLastRun
|
|
@@ -536,14 +427,6 @@ export class GarbageCollector {
|
|
|
536
427
|
gcNodes[nodeId] = Array.from(nodeData.outboundRoutes);
|
|
537
428
|
}
|
|
538
429
|
this.gcDataFromLastRun = { gcNodes };
|
|
539
|
-
// If tracking state across summaries, update latest summary data from the snapshot's GC data.
|
|
540
|
-
if (this.trackGCState) {
|
|
541
|
-
this.latestSummaryData = {
|
|
542
|
-
serializedGCState: JSON.stringify(generateSortedGCState(snapshotData.gcState)),
|
|
543
|
-
serializedTombstones: JSON.stringify(snapshotData.tombstones),
|
|
544
|
-
serializedDeletedNodes: JSON.stringify(snapshotData.deletedNodes),
|
|
545
|
-
};
|
|
546
|
-
}
|
|
547
430
|
}
|
|
548
431
|
/**
|
|
549
432
|
* Called when the connection state of the runtime changes, i.e., it connects or disconnects. GC subscribes to this
|
|
@@ -575,9 +458,12 @@ export class GarbageCollector {
|
|
|
575
458
|
*/
|
|
576
459
|
async collectGarbage(options, telemetryContext) {
|
|
577
460
|
var _a;
|
|
578
|
-
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true ||
|
|
461
|
+
const fullGC = (_a = options.fullGC) !== null && _a !== void 0 ? _a : (this.gcOptions.runFullGC === true ||
|
|
462
|
+
this.summaryStateTracker.doesSummaryStateNeedReset());
|
|
579
463
|
const logger = options.logger
|
|
580
|
-
? ChildLogger.create(options.logger, undefined, {
|
|
464
|
+
? ChildLogger.create(options.logger, undefined, {
|
|
465
|
+
all: { completedGCRuns: () => this.completedRuns },
|
|
466
|
+
})
|
|
581
467
|
: this.mc.logger;
|
|
582
468
|
/**
|
|
583
469
|
* If there is no current reference timestamp, skip running GC. We need the current timestamp to track
|
|
@@ -597,7 +483,10 @@ export class GarbageCollector {
|
|
|
597
483
|
return undefined;
|
|
598
484
|
}
|
|
599
485
|
// Add the options that are used to run GC to the telemetry context.
|
|
600
|
-
telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.
|
|
486
|
+
telemetryContext === null || telemetryContext === void 0 ? void 0 : telemetryContext.setMultiple("fluid_GC", "Options", {
|
|
487
|
+
fullGC,
|
|
488
|
+
runSweep: options.runSweep,
|
|
489
|
+
});
|
|
601
490
|
return PerformanceEvent.timedExecAsync(logger, { eventName: "GarbageCollection" }, async (event) => {
|
|
602
491
|
await this.runPreGCSteps();
|
|
603
492
|
// Get the runtime's GC data and run GC on the reference graph in it.
|
|
@@ -619,26 +508,29 @@ export class GarbageCollector {
|
|
|
619
508
|
// Generate statistics from the current run. This is done before updating the current state because it
|
|
620
509
|
// generates some of its data based on previous state of the system.
|
|
621
510
|
const gcStats = this.generateStats(gcResult);
|
|
622
|
-
// Update the state
|
|
623
|
-
|
|
624
|
-
this.updateStateSinceLastRun(gcData, logger);
|
|
625
|
-
// Update the current state and update the runtime of all routes or ids that used as per the GC run.
|
|
626
|
-
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
511
|
+
// Update the current mark state and update the runtime of all used routes or ids that used as per the GC run.
|
|
512
|
+
const sweepReadyNodes = this.updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger);
|
|
627
513
|
this.runtime.updateUsedRoutes(gcResult.referencedNodeIds);
|
|
628
514
|
// Log events for objects that are ready to be deleted by sweep. When we have sweep enabled, we will
|
|
629
515
|
// delete these objects here instead.
|
|
630
516
|
this.logSweepEvents(logger, currentReferenceTimestampMs);
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
517
|
+
let updatedGCData = gcData;
|
|
518
|
+
if (this.shouldRunSweep) {
|
|
519
|
+
updatedGCData = this.runSweepPhase(sweepReadyNodes, gcData);
|
|
520
|
+
}
|
|
521
|
+
else if (this.testMode) {
|
|
522
|
+
// If we are running in GC test mode, delete objects for unused routes. This enables testing scenarios
|
|
523
|
+
// involving access to deleted data.
|
|
634
524
|
this.runtime.updateUnusedRoutes(gcResult.deletedNodeIds);
|
|
635
525
|
}
|
|
636
526
|
else if (this.tombstoneMode) {
|
|
527
|
+
this.tombstones = sweepReadyNodes;
|
|
637
528
|
// If we are running in GC tombstone mode, update tombstoned routes. This enables testing scenarios
|
|
638
529
|
// involving access to "deleted" data without actually deleting the data from summaries.
|
|
639
530
|
// Note: we will not tombstone in test mode.
|
|
640
531
|
this.runtime.updateTombstonedRoutes(this.tombstones);
|
|
641
532
|
}
|
|
533
|
+
this.gcDataFromLastRun = cloneGCData(updatedGCData);
|
|
642
534
|
// Log pending unreferenced events such as a node being used after inactive. This is done after GC runs and
|
|
643
535
|
// updates its state so that we don't send false positives based on intermediate state. For example, we may get
|
|
644
536
|
// reference to an unreferenced node from another unreferenced node which means the node wasn't revived.
|
|
@@ -662,89 +554,7 @@ export class GarbageCollector {
|
|
|
662
554
|
unreferencedTimestampMs: (_a = this.unreferencedNodesState.get(nodeId)) === null || _a === void 0 ? void 0 : _a.unreferencedTimestampMs,
|
|
663
555
|
};
|
|
664
556
|
}
|
|
665
|
-
|
|
666
|
-
// Serialize and write deleted nodes, if any. This is done irrespective of whether sweep is enabled or not so
|
|
667
|
-
// to identify deleted nodes' usage.
|
|
668
|
-
const serializedDeletedNodes = this.deletedNodes.size > 0
|
|
669
|
-
? JSON.stringify(Array.from(this.deletedNodes).sort())
|
|
670
|
-
: undefined;
|
|
671
|
-
// If running in tombstone mode, serialize and write tombstones, if any.
|
|
672
|
-
const serializedTombstones = this.tombstoneMode
|
|
673
|
-
? (this.tombstones.length > 0 ? JSON.stringify(this.tombstones.sort()) : undefined)
|
|
674
|
-
: undefined;
|
|
675
|
-
/**
|
|
676
|
-
* Incremental summary of GC data - If none of GC state, deleted nodes or tombstones changed since last summary,
|
|
677
|
-
* write summary handle instead of summary tree for GC.
|
|
678
|
-
* Otherwise, write the GC summary tree. In the tree, for each of these that changed, write a summary blob and
|
|
679
|
-
* for each of these that did not change, write a summary handle.
|
|
680
|
-
*/
|
|
681
|
-
if (this.trackGCState) {
|
|
682
|
-
this.pendingSummaryData = { serializedGCState, serializedTombstones, serializedDeletedNodes };
|
|
683
|
-
if (trackState && !fullTree && this.latestSummaryData !== undefined) {
|
|
684
|
-
// If nothing changed since last summary, send a summary handle for the entire GC data.
|
|
685
|
-
if (this.latestSummaryData.serializedGCState === serializedGCState
|
|
686
|
-
&& this.latestSummaryData.serializedTombstones === serializedTombstones) {
|
|
687
|
-
const stats = mergeStats();
|
|
688
|
-
stats.handleNodeCount++;
|
|
689
|
-
return {
|
|
690
|
-
summary: {
|
|
691
|
-
type: SummaryType.Handle,
|
|
692
|
-
handle: `/${gcTreeKey}`,
|
|
693
|
-
handleType: SummaryType.Tree,
|
|
694
|
-
},
|
|
695
|
-
stats,
|
|
696
|
-
};
|
|
697
|
-
}
|
|
698
|
-
// If some state changed, build a GC summary tree.
|
|
699
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, true /* trackState */);
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
// If not tracking GC state, build a GC summary tree without any summary handles.
|
|
703
|
-
return this.buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, false /* trackState */);
|
|
704
|
-
}
|
|
705
|
-
/**
|
|
706
|
-
* Builds the GC summary tree which contains GC state, deleted nodes and tombstones.
|
|
707
|
-
* If trackState is false, all of GC state, deleted nodes and tombstones are written as summary blobs.
|
|
708
|
-
* If trackState is true, only states that changed are written. Rest are written as handles.
|
|
709
|
-
* @param serializedGCState - The GC state serialized as string.
|
|
710
|
-
* @param serializedTombstones - The tombstone state serialized as string.
|
|
711
|
-
* @param serializedDeletedNodes - Deleted nodes serialized as string.
|
|
712
|
-
* @param trackState - Whether we are tracking GC state across summaries.
|
|
713
|
-
* @returns the GC summary tree.
|
|
714
|
-
*/
|
|
715
|
-
buildGCSummaryTree(serializedGCState, serializedTombstones, serializedDeletedNodes, trackState) {
|
|
716
|
-
var _a, _b, _c;
|
|
717
|
-
const gcStateBlobKey = `${gcBlobPrefix}_root`;
|
|
718
|
-
const builder = new SummaryTreeBuilder();
|
|
719
|
-
// If the GC state hasn't changed, write a summary handle, else write a summary blob for it.
|
|
720
|
-
if (((_a = this.latestSummaryData) === null || _a === void 0 ? void 0 : _a.serializedGCState) === serializedGCState && trackState) {
|
|
721
|
-
builder.addHandle(gcStateBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcStateBlobKey}`);
|
|
722
|
-
}
|
|
723
|
-
else {
|
|
724
|
-
builder.addBlob(gcStateBlobKey, serializedGCState);
|
|
725
|
-
}
|
|
726
|
-
// If tombstones exist, write a summary handle if it hasn't changed. If it has changed, write a
|
|
727
|
-
// summary blob.
|
|
728
|
-
if (serializedTombstones !== undefined) {
|
|
729
|
-
if (((_b = this.latestSummaryData) === null || _b === void 0 ? void 0 : _b.serializedTombstones) === serializedTombstones && trackState) {
|
|
730
|
-
builder.addHandle(gcTombstoneBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcTombstoneBlobKey}`);
|
|
731
|
-
}
|
|
732
|
-
else {
|
|
733
|
-
builder.addBlob(gcTombstoneBlobKey, serializedTombstones);
|
|
734
|
-
}
|
|
735
|
-
}
|
|
736
|
-
// If there are no deleted nodes, return the summary tree.
|
|
737
|
-
if (serializedDeletedNodes === undefined) {
|
|
738
|
-
return builder.getSummaryTree();
|
|
739
|
-
}
|
|
740
|
-
// If the deleted nodes hasn't changed, write a summary handle, else write a summary blob for it.
|
|
741
|
-
if (((_c = this.latestSummaryData) === null || _c === void 0 ? void 0 : _c.serializedDeletedNodes) === serializedDeletedNodes && trackState) {
|
|
742
|
-
builder.addHandle(gcDeletedBlobKey, SummaryType.Blob, `/${gcTreeKey}/${gcDeletedBlobKey}`);
|
|
743
|
-
}
|
|
744
|
-
else {
|
|
745
|
-
builder.addBlob(gcDeletedBlobKey, serializedDeletedNodes);
|
|
746
|
-
}
|
|
747
|
-
return builder.getSummaryTree();
|
|
557
|
+
return this.summaryStateTracker.summarize(fullTree, trackState, gcState, this.deletedNodes, this.tombstones);
|
|
748
558
|
}
|
|
749
559
|
getMetadata() {
|
|
750
560
|
return {
|
|
@@ -752,7 +562,7 @@ export class GarbageCollector {
|
|
|
752
562
|
* If GC is enabled, the GC data is written using the current GC version and that is the gcFeature that goes
|
|
753
563
|
* into the metadata blob. If GC is disabled, the gcFeature is 0.
|
|
754
564
|
*/
|
|
755
|
-
gcFeature: this.gcEnabled ? this.currentGCVersion : 0,
|
|
565
|
+
gcFeature: this.gcEnabled ? this.summaryStateTracker.currentGCVersion : 0,
|
|
756
566
|
gcFeatureMatrix: this.persistedGcFeatureMatrix,
|
|
757
567
|
sessionExpiryTimeoutMs: this.sessionExpiryTimeoutMs,
|
|
758
568
|
sweepEnabled: this.sweepEnabled,
|
|
@@ -771,48 +581,22 @@ export class GarbageCollector {
|
|
|
771
581
|
* is downloaded and should be used to update the state.
|
|
772
582
|
*/
|
|
773
583
|
async refreshLatestSummary(proposalHandle, result, readAndParseBlob) {
|
|
774
|
-
|
|
775
|
-
// summary
|
|
776
|
-
//
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
this.latestSummaryGCVersion = this.currentGCVersion;
|
|
788
|
-
if (this.trackGCState) {
|
|
789
|
-
this.latestSummaryData = this.pendingSummaryData;
|
|
790
|
-
this.pendingSummaryData = undefined;
|
|
584
|
+
const latestSnapshotData = await this.summaryStateTracker.refreshLatestSummary(proposalHandle, result, readAndParseBlob);
|
|
585
|
+
// If the latest summary was updated but it was not tracked by this client, our state needs to be updated from
|
|
586
|
+
// this snapshot data.
|
|
587
|
+
if (this.shouldRunGC && result.latestSummaryUpdated && !result.wasSummaryTracked) {
|
|
588
|
+
// The current reference timestamp should be available if we are refreshing state from a snapshot. There has
|
|
589
|
+
// to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
|
|
590
|
+
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
591
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
592
|
+
throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, {
|
|
593
|
+
proposalHandle,
|
|
594
|
+
summaryRefSeq: result.summaryRefSeq,
|
|
595
|
+
details: JSON.stringify(this.configs),
|
|
596
|
+
});
|
|
791
597
|
}
|
|
792
|
-
|
|
793
|
-
}
|
|
794
|
-
// If the summary was not tracked by this client, the state should be updated from the downloaded snapshot.
|
|
795
|
-
const snapshotTree = result.snapshotTree;
|
|
796
|
-
const metadataBlobId = snapshotTree.blobs[metadataBlobName];
|
|
797
|
-
if (metadataBlobId) {
|
|
798
|
-
const metadata = await readAndParseBlob(metadataBlobId);
|
|
799
|
-
this.latestSummaryGCVersion = getGCVersion(metadata);
|
|
800
|
-
}
|
|
801
|
-
// The current reference timestamp should be available if we are refreshing state from a snapshot. There has
|
|
802
|
-
// to be at least one op (summary op / ack, if nothing else) if a snapshot was taken.
|
|
803
|
-
const currentReferenceTimestampMs = this.runtime.getCurrentReferenceTimestampMs();
|
|
804
|
-
if (currentReferenceTimestampMs === undefined) {
|
|
805
|
-
throw DataProcessingError.create("No reference timestamp when updating GC state from snapshot", "refreshLatestSummary", undefined, { proposalHandle, summaryRefSeq: result.summaryRefSeq, details: JSON.stringify(this.configs) });
|
|
598
|
+
this.updateStateFromSnapshotData(latestSnapshotData, currentReferenceTimestampMs);
|
|
806
599
|
}
|
|
807
|
-
const gcSnapshotTree = snapshotTree.trees[gcTreeKey];
|
|
808
|
-
// If GC ran in the container that generated this snapshot, it will have a GC tree.
|
|
809
|
-
this.wasGCRunInLatestSummary = gcSnapshotTree !== undefined;
|
|
810
|
-
let latestGCData;
|
|
811
|
-
if (gcSnapshotTree !== undefined) {
|
|
812
|
-
latestGCData = await getGCDataFromSnapshot(gcSnapshotTree, readAndParseBlob);
|
|
813
|
-
}
|
|
814
|
-
this.updateStateFromSnapshotData(latestGCData, currentReferenceTimestampMs);
|
|
815
|
-
this.pendingSummaryData = undefined;
|
|
816
600
|
}
|
|
817
601
|
/**
|
|
818
602
|
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
@@ -859,12 +643,12 @@ export class GarbageCollector {
|
|
|
859
643
|
else if (nodeType === GCNodeType.Blob) {
|
|
860
644
|
eventName = "GC_Tombstone_Blob_Revived";
|
|
861
645
|
}
|
|
862
|
-
|
|
646
|
+
sendGCUnexpectedUsageEvent(this.mc, {
|
|
863
647
|
eventName,
|
|
864
648
|
category: "generic",
|
|
865
|
-
isSummarizerClient: this.isSummarizerClient,
|
|
866
649
|
url: trimLeadingSlashes(toNodePath),
|
|
867
650
|
nodeType,
|
|
651
|
+
gcTombstoneEnforcementAllowed: this.runtime.gcTombstoneEnforcementAllowed,
|
|
868
652
|
}, undefined /* packagePath */);
|
|
869
653
|
}
|
|
870
654
|
}
|
|
@@ -888,13 +672,16 @@ export class GarbageCollector {
|
|
|
888
672
|
* @param gcData - The data representing the reference graph on which GC is run.
|
|
889
673
|
* @param gcResult - The result of the GC run on the gcData.
|
|
890
674
|
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
675
|
+
* @returns - A list of sweep ready nodes. (Nodes ready to be deleted)
|
|
891
676
|
*/
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
677
|
+
updateMarkPhase(gcData, gcResult, currentReferenceTimestampMs, logger) {
|
|
678
|
+
var _a;
|
|
679
|
+
// Get references from the current GC run + references between previous and current run and then update each
|
|
680
|
+
// node's state
|
|
681
|
+
const allNodesReferencedBetweenGCs = (_a = this.findAllNodesReferencedBetweenGCs(gcData, this.gcDataFromLastRun, logger)) !== null && _a !== void 0 ? _a : gcResult.referencedNodeIds;
|
|
895
682
|
this.newReferencesSinceLastRun.clear();
|
|
896
683
|
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
897
|
-
for (const nodeId of
|
|
684
|
+
for (const nodeId of allNodesReferencedBetweenGCs) {
|
|
898
685
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
899
686
|
if (nodeStateTracker !== undefined) {
|
|
900
687
|
// Stop tracking so as to clear out any running timers.
|
|
@@ -907,7 +694,10 @@ export class GarbageCollector {
|
|
|
907
694
|
* If a node became unreferenced in this run, start tracking it.
|
|
908
695
|
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
909
696
|
* from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
|
|
697
|
+
*
|
|
698
|
+
* If a node is sweep ready, store and then return it.
|
|
910
699
|
*/
|
|
700
|
+
const sweepReadyNodes = [];
|
|
911
701
|
for (const nodeId of gcResult.deletedNodeIds) {
|
|
912
702
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
913
703
|
if (nodeStateTracker === undefined) {
|
|
@@ -915,14 +705,51 @@ export class GarbageCollector {
|
|
|
915
705
|
}
|
|
916
706
|
else {
|
|
917
707
|
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
918
|
-
if (
|
|
919
|
-
|
|
920
|
-
if (nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) {
|
|
921
|
-
this.tombstones.push(nodeId);
|
|
922
|
-
}
|
|
708
|
+
if (nodeStateTracker.state === UnreferencedState.SweepReady) {
|
|
709
|
+
sweepReadyNodes.push(nodeId);
|
|
923
710
|
}
|
|
924
711
|
}
|
|
925
712
|
}
|
|
713
|
+
return sweepReadyNodes;
|
|
714
|
+
}
|
|
715
|
+
/**
|
|
716
|
+
* Deletes nodes from both the runtime and garbage collection
|
|
717
|
+
* @param sweepReadyNodes - nodes that are ready to be deleted
|
|
718
|
+
*/
|
|
719
|
+
runSweepPhase(sweepReadyNodes, gcData) {
|
|
720
|
+
// TODO: GC:Validation - validate that removed routes are not double deleted
|
|
721
|
+
// TODO: GC:Validation - validate that the child routes of removed routes are deleted as well
|
|
722
|
+
const sweptRoutes = this.runtime.deleteSweepReadyNodes(sweepReadyNodes);
|
|
723
|
+
const updatedGCData = this.deleteSweptRoutes(sweptRoutes, gcData);
|
|
724
|
+
for (const nodeId of sweptRoutes) {
|
|
725
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
726
|
+
// TODO: GC:Validation - assert that the nodeStateTracker is defined
|
|
727
|
+
if (nodeStateTracker !== undefined) {
|
|
728
|
+
// Stop tracking so as to clear out any running timers.
|
|
729
|
+
nodeStateTracker.stopTracking();
|
|
730
|
+
// Delete the node as we don't need to track it any more.
|
|
731
|
+
this.unreferencedNodesState.delete(nodeId);
|
|
732
|
+
}
|
|
733
|
+
// TODO: GC:Validation - assert that the deleted node is not a duplicate
|
|
734
|
+
this.deletedNodes.add(nodeId);
|
|
735
|
+
}
|
|
736
|
+
return updatedGCData;
|
|
737
|
+
}
|
|
738
|
+
/**
|
|
739
|
+
* @returns IGarbageCollectionData after deleting the sweptRoutes from the gcData
|
|
740
|
+
*/
|
|
741
|
+
deleteSweptRoutes(sweptRoutes, gcData) {
|
|
742
|
+
const sweptRoutesSet = new Set(sweptRoutes);
|
|
743
|
+
const gcNodes = {};
|
|
744
|
+
for (const [id, outboundRoutes] of Object.entries(gcData.gcNodes)) {
|
|
745
|
+
if (!sweptRoutesSet.has(id)) {
|
|
746
|
+
gcNodes[id] = Array.from(outboundRoutes);
|
|
747
|
+
}
|
|
748
|
+
}
|
|
749
|
+
// TODO: GC:Validation - assert that the nodeId is in gcData
|
|
750
|
+
return {
|
|
751
|
+
gcNodes,
|
|
752
|
+
};
|
|
926
753
|
}
|
|
927
754
|
/**
|
|
928
755
|
* Since GC runs periodically, the GC data that is generated only tells us the state of the world at that point in
|
|
@@ -934,16 +761,19 @@ export class GarbageCollector {
|
|
|
934
761
|
* 2. A reference is added from one unreferenced node to one or more unreferenced nodes. Even though the node[s] were
|
|
935
762
|
* unreferenced, they could have been accessed and in-memory reference to them added.
|
|
936
763
|
*
|
|
937
|
-
* This function identifies nodes that were referenced since last run
|
|
764
|
+
* This function identifies nodes that were referenced since the last run.
|
|
938
765
|
* If these nodes are currently unreferenced, they will be assigned new unreferenced state by the current run.
|
|
766
|
+
*
|
|
767
|
+
* @returns - a list of all nodes referenced from the last local summary until now.
|
|
939
768
|
*/
|
|
940
|
-
|
|
769
|
+
findAllNodesReferencedBetweenGCs(currentGCData, previousGCData, logger) {
|
|
941
770
|
// If we haven't run GC before there is nothing to do.
|
|
942
|
-
|
|
943
|
-
|
|
771
|
+
// No previousGCData, means nothing is unreferenced, and there are no reference state trackers to clear
|
|
772
|
+
if (previousGCData === undefined) {
|
|
773
|
+
return undefined;
|
|
944
774
|
}
|
|
945
775
|
// Find any references that haven't been identified correctly.
|
|
946
|
-
const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData,
|
|
776
|
+
const missingExplicitReferences = this.findMissingExplicitReferences(currentGCData, previousGCData, this.newReferencesSinceLastRun);
|
|
947
777
|
if (missingExplicitReferences.length > 0) {
|
|
948
778
|
missingExplicitReferences.forEach((missingExplicitReference) => {
|
|
949
779
|
logger.sendErrorEvent({
|
|
@@ -954,9 +784,9 @@ export class GarbageCollector {
|
|
|
954
784
|
});
|
|
955
785
|
}
|
|
956
786
|
// No references were added since the last run so we don't have to update reference states of any unreferenced
|
|
957
|
-
// nodes
|
|
787
|
+
// nodes. There is no in between state at this point.
|
|
958
788
|
if (this.newReferencesSinceLastRun.size === 0) {
|
|
959
|
-
return;
|
|
789
|
+
return undefined;
|
|
960
790
|
}
|
|
961
791
|
/**
|
|
962
792
|
* Generate a super set of the GC data that contains the nodes and edges from last run, plus any new node and
|
|
@@ -974,7 +804,7 @@ export class GarbageCollector {
|
|
|
974
804
|
* - We don't require DDSes handles to be stored in a referenced DDS.
|
|
975
805
|
* - A new data store may have "root" DDSes already created and we don't detect them today.
|
|
976
806
|
*/
|
|
977
|
-
const gcDataSuperSet = concatGarbageCollectionData(
|
|
807
|
+
const gcDataSuperSet = concatGarbageCollectionData(previousGCData, currentGCData);
|
|
978
808
|
const newOutboundRoutesSinceLastRun = [];
|
|
979
809
|
this.newReferencesSinceLastRun.forEach((outboundRoutes, sourceNodeId) => {
|
|
980
810
|
if (gcDataSuperSet.gcNodes[sourceNodeId] === undefined) {
|
|
@@ -992,16 +822,11 @@ export class GarbageCollector {
|
|
|
992
822
|
* Note that some of these nodes may be unreferenced now and if so, the current run will mark them as
|
|
993
823
|
* unreferenced and add unreferenced state.
|
|
994
824
|
*/
|
|
995
|
-
const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
nodeStateTracker.stopTracking();
|
|
1001
|
-
// Delete the unreferenced state as we don't need to track it any more.
|
|
1002
|
-
this.unreferencedNodesState.delete(nodeId);
|
|
1003
|
-
}
|
|
1004
|
-
}
|
|
825
|
+
const gcResult = runGarbageCollection(gcDataSuperSet.gcNodes, [
|
|
826
|
+
"/",
|
|
827
|
+
...newOutboundRoutesSinceLastRun,
|
|
828
|
+
]);
|
|
829
|
+
return gcResult.referencedNodeIds;
|
|
1005
830
|
}
|
|
1006
831
|
/**
|
|
1007
832
|
* Finds all new references or outbound routes in the current graph that haven't been explicitly notified to GC.
|
|
@@ -1017,7 +842,7 @@ export class GarbageCollector {
|
|
|
1017
842
|
* @returns - a list of missing explicit references
|
|
1018
843
|
*/
|
|
1019
844
|
findMissingExplicitReferences(currentGCData, previousGCData, explicitReferences) {
|
|
1020
|
-
assert(previousGCData !== undefined, 0x2b7);
|
|
845
|
+
assert(previousGCData !== undefined, 0x2b7 /* "Can't validate correctness without GC data from last run" */);
|
|
1021
846
|
const currentGraph = Object.entries(currentGCData.gcNodes);
|
|
1022
847
|
const missingExplicitReferences = [];
|
|
1023
848
|
currentGraph.forEach(([nodeId, currentOutboundRoutes]) => {
|
|
@@ -1035,9 +860,10 @@ export class GarbageCollector {
|
|
|
1035
860
|
*/
|
|
1036
861
|
currentOutboundRoutes.forEach((route) => {
|
|
1037
862
|
const nodeType = this.runtime.getNodeType(route);
|
|
1038
|
-
if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob)
|
|
1039
|
-
|
|
1040
|
-
|
|
863
|
+
if ((nodeType === GCNodeType.DataStore || nodeType === GCNodeType.Blob) &&
|
|
864
|
+
!nodeId.startsWith(route) &&
|
|
865
|
+
!previousRoutes.includes(route) &&
|
|
866
|
+
!explicitRoutes.includes(route)) {
|
|
1041
867
|
missingExplicitRoutes.push(route);
|
|
1042
868
|
}
|
|
1043
869
|
});
|
|
@@ -1109,7 +935,8 @@ export class GarbageCollector {
|
|
|
1109
935
|
* this will give us a view into how much deleted content a container has.
|
|
1110
936
|
*/
|
|
1111
937
|
logSweepEvents(logger, currentReferenceTimestampMs) {
|
|
1112
|
-
if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
|
|
938
|
+
if (this.mc.config.getBoolean(disableSweepLogKey) === true ||
|
|
939
|
+
this.sweepTimeoutMs === undefined) {
|
|
1113
940
|
return;
|
|
1114
941
|
}
|
|
1115
942
|
this.unreferencedNodesState.forEach((nodeStateTracker, nodeId) => {
|
|
@@ -1144,7 +971,8 @@ export class GarbageCollector {
|
|
|
1144
971
|
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
1145
972
|
// logging as nothing interesting would have happened worth logging.
|
|
1146
973
|
// If the node is active, skip logging.
|
|
1147
|
-
if (currentReferenceTimestampMs === undefined ||
|
|
974
|
+
if (currentReferenceTimestampMs === undefined ||
|
|
975
|
+
nodeStateTracker.state === UnreferencedState.Active) {
|
|
1148
976
|
return;
|
|
1149
977
|
}
|
|
1150
978
|
// We only care about data stores and attachment blobs for this telemetry since GC only marks these objects
|
|
@@ -1161,7 +989,7 @@ export class GarbageCollector {
|
|
|
1161
989
|
this.loggedUnreferencedEvents.add(uniqueEventId);
|
|
1162
990
|
const propsToLog = Object.assign(Object.assign({ id: nodeId, type: nodeType, unrefTime: nodeStateTracker.unreferencedTimestampMs, age: currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs, timeout: nodeStateTracker.state === UnreferencedState.Inactive
|
|
1163
991
|
? this.inactiveTimeoutMs
|
|
1164
|
-
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), {
|
|
992
|
+
: this.sweepTimeoutMs, completedGCRuns: this.completedRuns, lastSummaryTime: this.getLastSummaryTimestampMs() }, this.createContainerMetadata), { viaHandle: requestHeaders === null || requestHeaders === void 0 ? void 0 : requestHeaders[RuntimeHeaders.viaHandle], fromId: fromNodeId });
|
|
1165
993
|
// For summarizer client, queue the event so it is logged the next time GC runs if the event is still valid.
|
|
1166
994
|
// For non-summarizer client, log the event now since GC won't run on it. This may result in false positives
|
|
1167
995
|
// but it's a good signal nonetheless and we can consume it with a grain of salt.
|
|
@@ -1209,11 +1037,18 @@ export class GarbageCollector {
|
|
|
1209
1037
|
* revived and a Revived event will be logged for it.
|
|
1210
1038
|
*/
|
|
1211
1039
|
const nodeStateTracker = this.unreferencedNodesState.get(eventProps.id);
|
|
1212
|
-
const active = nodeStateTracker === undefined ||
|
|
1040
|
+
const active = nodeStateTracker === undefined ||
|
|
1041
|
+
nodeStateTracker.state === UnreferencedState.Active;
|
|
1213
1042
|
if ((usageType === "Revived") === active) {
|
|
1214
1043
|
const pkg = await this.getNodePackagePath(eventProps.id);
|
|
1215
|
-
const fromPkg = eventProps.fromId
|
|
1216
|
-
|
|
1044
|
+
const fromPkg = eventProps.fromId
|
|
1045
|
+
? await this.getNodePackagePath(eventProps.fromId)
|
|
1046
|
+
: undefined;
|
|
1047
|
+
const event = Object.assign(Object.assign({}, propsToLog), { eventName: `${state}Object_${usageType}`, pkg: pkg
|
|
1048
|
+
? { value: pkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
|
|
1049
|
+
: undefined, fromPkg: fromPkg
|
|
1050
|
+
? { value: fromPkg.join("/"), tag: TelemetryDataTag.CodeArtifact }
|
|
1051
|
+
: undefined });
|
|
1217
1052
|
if (state === UnreferencedState.Inactive) {
|
|
1218
1053
|
logger.sendTelemetryEvent(event);
|
|
1219
1054
|
}
|
|
@@ -1225,28 +1060,4 @@ export class GarbageCollector {
|
|
|
1225
1060
|
this.pendingEventsQueue = [];
|
|
1226
1061
|
}
|
|
1227
1062
|
}
|
|
1228
|
-
function generateSortedGCState(gcState) {
|
|
1229
|
-
const sortableArray = Object.entries(gcState.gcNodes);
|
|
1230
|
-
sortableArray.sort(([a], [b]) => a.localeCompare(b));
|
|
1231
|
-
const sortedGCState = { gcNodes: {} };
|
|
1232
|
-
for (const [nodeId, nodeData] of sortableArray) {
|
|
1233
|
-
nodeData.outboundRoutes.sort();
|
|
1234
|
-
sortedGCState.gcNodes[nodeId] = nodeData;
|
|
1235
|
-
}
|
|
1236
|
-
return sortedGCState;
|
|
1237
|
-
}
|
|
1238
|
-
/** A wrapper around common-utils Timer that requires the timeout when calling start/restart */
|
|
1239
|
-
class TimerWithNoDefaultTimeout extends Timer {
|
|
1240
|
-
constructor(callback) {
|
|
1241
|
-
// The default timeout/handlers will never be used since start/restart pass overrides below
|
|
1242
|
-
super(0, () => { throw new Error("DefaultHandler should not be used"); });
|
|
1243
|
-
this.callback = callback;
|
|
1244
|
-
}
|
|
1245
|
-
start(timeoutMs) {
|
|
1246
|
-
super.start(timeoutMs, this.callback);
|
|
1247
|
-
}
|
|
1248
|
-
restart(timeoutMs) {
|
|
1249
|
-
super.restart(timeoutMs, this.callback);
|
|
1250
|
-
}
|
|
1251
|
-
}
|
|
1252
1063
|
//# sourceMappingURL=garbageCollection.js.map
|