@fluidframework/container-runtime 2.0.0-dev.5.3.2.178189 → 2.0.0-dev.6.4.0.191258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +123 -0
- package/README.md +4 -3
- package/dist/batchTracker.d.ts +3 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +6 -5
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +10 -16
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +184 -172
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +25 -16
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +161 -35
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +697 -449
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +15 -7
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -2
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +83 -87
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +1 -2
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +8 -9
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.js +2 -2
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +22 -6
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +123 -81
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerProxyBase.d.ts +35 -0
- package/dist/deltaManagerProxyBase.d.ts.map +1 -0
- package/dist/deltaManagerProxyBase.js +77 -0
- package/dist/deltaManagerProxyBase.js.map +1 -0
- package/dist/deltaManagerSummarizerProxy.d.ts +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/dist/deltaManagerSummarizerProxy.js +4 -2
- package/dist/deltaManagerSummarizerProxy.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +10 -10
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/error.d.ts +14 -0
- package/dist/error.d.ts.map +1 -0
- package/dist/error.js +21 -0
- package/dist/error.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +10 -9
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +61 -53
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +18 -14
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +17 -4
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +14 -15
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +0 -8
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +11 -24
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +4 -7
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +19 -58
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +45 -35
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +4 -4
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +3 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.js +26 -68
- package/dist/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/dist/id-compressor/finalSpace.d.ts +29 -0
- package/dist/id-compressor/finalSpace.d.ts.map +1 -0
- package/dist/id-compressor/finalSpace.js +62 -0
- package/dist/id-compressor/finalSpace.js.map +1 -0
- package/dist/id-compressor/idCompressor.d.ts +25 -250
- package/dist/id-compressor/idCompressor.d.ts.map +1 -1
- package/dist/id-compressor/idCompressor.js +390 -1153
- package/dist/id-compressor/idCompressor.js.map +1 -1
- package/dist/id-compressor/identifiers.d.ts +32 -0
- package/dist/id-compressor/identifiers.d.ts.map +1 -0
- package/dist/id-compressor/identifiers.js +15 -0
- package/dist/id-compressor/identifiers.js.map +1 -0
- package/dist/id-compressor/index.d.ts +5 -6
- package/dist/id-compressor/index.d.ts.map +1 -1
- package/dist/id-compressor/index.js +20 -26
- package/dist/id-compressor/index.js.map +1 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +22 -0
- package/dist/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/dist/id-compressor/persistanceUtilities.js +43 -0
- package/dist/id-compressor/persistanceUtilities.js.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js +80 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/dist/id-compressor/sessions.d.ts +115 -0
- package/dist/id-compressor/sessions.d.ts.map +1 -0
- package/dist/id-compressor/sessions.js +305 -0
- package/dist/id-compressor/sessions.js.map +1 -0
- package/dist/id-compressor/utilities.d.ts +49 -0
- package/dist/id-compressor/utilities.d.ts.map +1 -0
- package/dist/id-compressor/utilities.js +166 -0
- package/dist/id-compressor/utilities.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.js +10 -6
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +2 -2
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +12 -7
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -2
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +23 -20
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +19 -9
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +2 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +22 -19
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +7 -5
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +22 -31
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +19 -7
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.js +1 -2
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +18 -8
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +73 -51
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +36 -32
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summary/index.d.ts +3 -3
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +2 -1
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +3 -3
- package/dist/summary/orderedClientElection.d.ts.map +1 -1
- package/dist/summary/orderedClientElection.js +26 -27
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +3 -3
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/runningSummarizer.d.ts +31 -10
- package/dist/summary/runningSummarizer.d.ts.map +1 -1
- package/dist/summary/runningSummarizer.js +271 -139
- package/dist/summary/runningSummarizer.js.map +1 -1
- package/dist/summary/summarizer.d.ts +8 -7
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +79 -78
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerClientElection.d.ts +2 -2
- package/dist/summary/summarizerClientElection.d.ts.map +1 -1
- package/dist/summary/summarizerClientElection.js +7 -11
- package/dist/summary/summarizerClientElection.js.map +1 -1
- package/dist/summary/summarizerHeuristics.js +10 -14
- package/dist/summary/summarizerHeuristics.js.map +1 -1
- package/dist/summary/summarizerNode/index.d.ts +1 -1
- package/dist/summary/summarizerNode/index.d.ts.map +1 -1
- package/dist/summary/summarizerNode/index.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.d.ts +10 -19
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.js +57 -130
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +6 -30
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +4 -14
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +15 -101
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +38 -25
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +2 -3
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +12 -13
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +3 -0
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +6 -4
- package/dist/summary/summaryFormat.js.map +1 -1
- package/dist/summary/summaryGenerator.d.ts +10 -4
- package/dist/summary/summaryGenerator.d.ts.map +1 -1
- package/dist/summary/summaryGenerator.js +106 -57
- package/dist/summary/summaryGenerator.js.map +1 -1
- package/dist/summary/summaryManager.d.ts +8 -8
- package/dist/summary/summaryManager.d.ts.map +1 -1
- package/dist/summary/summaryManager.js +38 -28
- package/dist/summary/summaryManager.js.map +1 -1
- package/lib/batchTracker.d.ts +3 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +5 -4
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +10 -16
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +159 -147
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +15 -6
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +161 -35
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +651 -402
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +13 -5
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -2
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +47 -51
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +1 -2
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +3 -4
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +22 -6
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +107 -65
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerProxyBase.d.ts +35 -0
- package/lib/deltaManagerProxyBase.d.ts.map +1 -0
- package/lib/deltaManagerProxyBase.js +73 -0
- package/lib/deltaManagerProxyBase.js.map +1 -0
- package/lib/deltaManagerSummarizerProxy.d.ts +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/lib/deltaManagerSummarizerProxy.js +3 -1
- package/lib/deltaManagerSummarizerProxy.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +7 -7
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/error.d.ts +14 -0
- package/lib/error.d.ts.map +1 -0
- package/lib/error.js +17 -0
- package/lib/error.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +10 -9
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +61 -53
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +16 -12
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +17 -4
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +13 -14
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +0 -8
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +5 -17
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +4 -7
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +20 -59
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +46 -36
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -2
- package/lib/gc/index.js.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.js +25 -66
- package/lib/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/lib/id-compressor/finalSpace.d.ts +29 -0
- package/lib/id-compressor/finalSpace.d.ts.map +1 -0
- package/lib/id-compressor/finalSpace.js +58 -0
- package/lib/id-compressor/finalSpace.js.map +1 -0
- package/lib/id-compressor/idCompressor.d.ts +25 -250
- package/lib/id-compressor/idCompressor.d.ts.map +1 -1
- package/lib/id-compressor/idCompressor.js +385 -1142
- package/lib/id-compressor/idCompressor.js.map +1 -1
- package/lib/id-compressor/identifiers.d.ts +32 -0
- package/lib/id-compressor/identifiers.d.ts.map +1 -0
- package/lib/id-compressor/identifiers.js +11 -0
- package/lib/id-compressor/identifiers.js.map +1 -0
- package/lib/id-compressor/index.d.ts +5 -6
- package/lib/id-compressor/index.d.ts.map +1 -1
- package/lib/id-compressor/index.js +5 -6
- package/lib/id-compressor/index.js.map +1 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +22 -0
- package/lib/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/lib/id-compressor/persistanceUtilities.js +34 -0
- package/lib/id-compressor/persistanceUtilities.js.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js +76 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/lib/id-compressor/sessions.d.ts +115 -0
- package/lib/id-compressor/sessions.d.ts.map +1 -0
- package/lib/id-compressor/sessions.js +290 -0
- package/lib/id-compressor/sessions.js.map +1 -0
- package/lib/id-compressor/utilities.d.ts +49 -0
- package/lib/id-compressor/utilities.d.ts.map +1 -0
- package/lib/id-compressor/utilities.js +148 -0
- package/lib/id-compressor/utilities.js.map +1 -0
- package/lib/index.d.ts +3 -3
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -2
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.js +10 -6
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +2 -2
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +10 -5
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -2
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +15 -12
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +17 -7
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +2 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +13 -10
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +7 -5
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +13 -22
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +20 -8
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.js +1 -2
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +18 -8
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +62 -40
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +18 -14
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summary/index.d.ts +3 -3
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +1 -1
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +3 -3
- package/lib/summary/orderedClientElection.d.ts.map +1 -1
- package/lib/summary/orderedClientElection.js +21 -22
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/runningSummarizer.d.ts +31 -10
- package/lib/summary/runningSummarizer.d.ts.map +1 -1
- package/lib/summary/runningSummarizer.js +265 -133
- package/lib/summary/runningSummarizer.js.map +1 -1
- package/lib/summary/summarizer.d.ts +8 -7
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +75 -74
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerClientElection.d.ts +2 -2
- package/lib/summary/summarizerClientElection.d.ts.map +1 -1
- package/lib/summary/summarizerClientElection.js +6 -10
- package/lib/summary/summarizerClientElection.js.map +1 -1
- package/lib/summary/summarizerHeuristics.js +9 -13
- package/lib/summary/summarizerHeuristics.js.map +1 -1
- package/lib/summary/summarizerNode/index.d.ts +1 -1
- package/lib/summary/summarizerNode/index.d.ts.map +1 -1
- package/lib/summary/summarizerNode/index.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.d.ts +10 -19
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.js +42 -115
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +6 -30
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +4 -14
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +14 -100
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +38 -25
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +2 -3
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +5 -6
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +3 -0
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +5 -3
- package/lib/summary/summaryFormat.js.map +1 -1
- package/lib/summary/summaryGenerator.d.ts +10 -4
- package/lib/summary/summaryGenerator.d.ts.map +1 -1
- package/lib/summary/summaryGenerator.js +100 -51
- package/lib/summary/summaryGenerator.js.map +1 -1
- package/lib/summary/summaryManager.d.ts +8 -8
- package/lib/summary/summaryManager.d.ts.map +1 -1
- package/lib/summary/summaryManager.js +35 -25
- package/lib/summary/summaryManager.js.map +1 -1
- package/package.json +28 -31
- package/src/batchTracker.ts +7 -5
- package/src/blobManager.ts +188 -166
- package/src/connectionTelemetry.ts +9 -4
- package/src/containerRuntime.ts +806 -441
- package/src/dataStore.ts +12 -4
- package/src/dataStoreContext.ts +39 -43
- package/src/dataStoreContexts.ts +4 -6
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +114 -80
- package/src/deltaManagerProxyBase.ts +111 -0
- package/src/deltaManagerSummarizerProxy.ts +4 -1
- package/src/deltaScheduler.ts +7 -11
- package/src/error.ts +18 -0
- package/src/gc/garbageCollection.md +53 -5
- package/src/gc/garbageCollection.ts +58 -52
- package/src/gc/gcConfigs.ts +4 -2
- package/src/gc/gcDefinitions.ts +17 -20
- package/src/gc/gcEarlyAdoption.md +145 -0
- package/src/gc/gcHelpers.ts +1 -12
- package/src/gc/gcSummaryStateTracker.ts +19 -65
- package/src/gc/gcTelemetry.ts +14 -12
- package/src/gc/gcUnreferencedStateTracker.ts +1 -1
- package/src/gc/index.ts +2 -4
- package/src/id-compressor/appendOnlySortedMap.ts +26 -87
- package/src/id-compressor/finalSpace.ts +67 -0
- package/src/id-compressor/idCompressor.ts +458 -1682
- package/src/id-compressor/identifiers.ts +42 -0
- package/src/id-compressor/index.ts +11 -20
- package/src/id-compressor/persistanceUtilities.ts +58 -0
- package/src/id-compressor/sessionSpaceNormalizer.ts +83 -0
- package/src/id-compressor/sessions.ts +405 -0
- package/src/id-compressor/utilities.ts +187 -0
- package/src/index.ts +7 -2
- package/src/opLifecycle/opCompressor.ts +6 -5
- package/src/opLifecycle/opDecompressor.ts +6 -4
- package/src/opLifecycle/opGroupingManager.ts +9 -6
- package/src/opLifecycle/opSplitter.ts +7 -6
- package/src/opLifecycle/outbox.ts +23 -32
- package/src/opLifecycle/remoteMessageProcessor.ts +27 -8
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +72 -42
- package/src/scheduleManager.ts +7 -5
- package/src/summary/index.ts +4 -3
- package/src/summary/orderedClientElection.ts +10 -6
- package/src/summary/runWhileConnectedCoordinator.ts +1 -1
- package/src/summary/runningSummarizer.ts +291 -163
- package/src/summary/summarizer.ts +27 -16
- package/src/summary/summarizerClientElection.ts +2 -2
- package/src/summary/summarizerHeuristics.ts +1 -1
- package/src/summary/summarizerNode/index.ts +1 -2
- package/src/summary/summarizerNode/summarizerNode.ts +36 -160
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +7 -38
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +11 -130
- package/src/summary/summarizerTypes.ts +40 -25
- package/src/summary/summaryCollection.ts +3 -3
- package/src/summary/summaryFormat.ts +4 -1
- package/src/summary/summaryGenerator.ts +52 -52
- package/src/summary/summaryManager.ts +44 -17
- package/dist/id-compressor/idRange.d.ts +0 -11
- package/dist/id-compressor/idRange.d.ts.map +0 -1
- package/dist/id-compressor/idRange.js +0 -29
- package/dist/id-compressor/idRange.js.map +0 -1
- package/dist/id-compressor/numericUuid.d.ts +0 -59
- package/dist/id-compressor/numericUuid.d.ts.map +0 -1
- package/dist/id-compressor/numericUuid.js +0 -325
- package/dist/id-compressor/numericUuid.js.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/dist/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.js +0 -488
- package/dist/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/dist/id-compressor/utils.d.ts +0 -57
- package/dist/id-compressor/utils.d.ts.map +0 -1
- package/dist/id-compressor/utils.js +0 -90
- package/dist/id-compressor/utils.js.map +0 -1
- package/dist/id-compressor/uuidUtilities.d.ts +0 -28
- package/dist/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/dist/id-compressor/uuidUtilities.js +0 -104
- package/dist/id-compressor/uuidUtilities.js.map +0 -1
- package/lib/id-compressor/idRange.d.ts +0 -11
- package/lib/id-compressor/idRange.d.ts.map +0 -1
- package/lib/id-compressor/idRange.js +0 -25
- package/lib/id-compressor/idRange.js.map +0 -1
- package/lib/id-compressor/numericUuid.d.ts +0 -59
- package/lib/id-compressor/numericUuid.d.ts.map +0 -1
- package/lib/id-compressor/numericUuid.js +0 -315
- package/lib/id-compressor/numericUuid.js.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/lib/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.js +0 -484
- package/lib/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/lib/id-compressor/utils.d.ts +0 -57
- package/lib/id-compressor/utils.d.ts.map +0 -1
- package/lib/id-compressor/utils.js +0 -79
- package/lib/id-compressor/utils.js.map +0 -1
- package/lib/id-compressor/uuidUtilities.d.ts +0 -28
- package/lib/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/lib/id-compressor/uuidUtilities.js +0 -96
- package/lib/id-compressor/uuidUtilities.js.map +0 -1
- package/src/id-compressor/idRange.ts +0 -35
- package/src/id-compressor/numericUuid.ts +0 -383
- package/src/id-compressor/sessionIdNormalizer.ts +0 -609
- package/src/id-compressor/utils.ts +0 -114
- package/src/id-compressor/uuidUtilities.ts +0 -120
package/lib/containerRuntime.js
CHANGED
|
@@ -1,21 +1,11 @@
|
|
|
1
|
-
var __rest = (this && this.__rest) || function (s, e) {
|
|
2
|
-
var t = {};
|
|
3
|
-
for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
|
|
4
|
-
t[p] = s[p];
|
|
5
|
-
if (s != null && typeof Object.getOwnPropertySymbols === "function")
|
|
6
|
-
for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
|
|
7
|
-
if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
|
|
8
|
-
t[p[i]] = s[p[i]];
|
|
9
|
-
}
|
|
10
|
-
return t;
|
|
11
|
-
};
|
|
12
1
|
import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
|
|
13
|
-
import { assert, delay,
|
|
14
|
-
import {
|
|
15
|
-
import {
|
|
2
|
+
import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
|
|
3
|
+
import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
|
|
4
|
+
import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent,
|
|
5
|
+
// eslint-disable-next-line import/no-deprecated
|
|
6
|
+
TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
|
|
16
7
|
import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
|
|
17
8
|
import { readAndParse } from "@fluidframework/driver-utils";
|
|
18
|
-
import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
|
|
19
9
|
import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
|
|
20
10
|
import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
|
|
21
11
|
import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, requestFluidObject, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
|
|
@@ -56,6 +46,18 @@ export var ContainerMessageType;
|
|
|
56
46
|
*/
|
|
57
47
|
ContainerMessageType["IdAllocation"] = "idAllocation";
|
|
58
48
|
})(ContainerMessageType || (ContainerMessageType = {}));
|
|
49
|
+
/**
|
|
50
|
+
* Utility to implement compat behaviors given an unknown message type
|
|
51
|
+
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|
|
52
|
+
*
|
|
53
|
+
* @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
|
|
54
|
+
* handled before calling this function (e.g. in a switch statement).
|
|
55
|
+
* @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
|
|
56
|
+
*/
|
|
57
|
+
function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
|
|
58
|
+
// undefined defaults to same behavior as "FailToProcess"
|
|
59
|
+
return compatBehavior === "Ignore";
|
|
60
|
+
}
|
|
59
61
|
export const DefaultSummaryConfiguration = {
|
|
60
62
|
state: "enabled",
|
|
61
63
|
minIdleTime: 0,
|
|
@@ -82,8 +84,12 @@ export var RuntimeHeaders;
|
|
|
82
84
|
})(RuntimeHeaders || (RuntimeHeaders = {}));
|
|
83
85
|
/** True if a tombstoned object should be returned without erroring */
|
|
84
86
|
export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
|
|
87
|
+
/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
|
|
88
|
+
export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
|
|
85
89
|
/** Tombstone error responses will have this header set to true */
|
|
86
90
|
export const TombstoneResponseHeaderKey = "isTombstoned";
|
|
91
|
+
/** Inactive error responses will have this header set to true */
|
|
92
|
+
export const InactiveResponseHeaderKey = "isInactive";
|
|
87
93
|
/** Default values for Runtime Headers */
|
|
88
94
|
export const defaultRuntimeHeaderData = {
|
|
89
95
|
wait: true,
|
|
@@ -110,12 +116,16 @@ const defaultCompressionConfig = {
|
|
|
110
116
|
compressionAlgorithm: CompressionAlgorithms.lz4,
|
|
111
117
|
};
|
|
112
118
|
const defaultChunkSizeInBytes = 204800;
|
|
119
|
+
/** The default time to wait for pending ops to be processed during summarization */
|
|
120
|
+
export const defaultPendingOpsWaitTimeoutMs = 1000;
|
|
121
|
+
/** The default time to delay a summarization retry attempt when there are pending ops */
|
|
122
|
+
export const defaultPendingOpsRetryDelayMs = 1000;
|
|
113
123
|
/**
|
|
114
124
|
* Instead of refreshing from latest because we do not have 100% confidence in the state
|
|
115
125
|
* of the current system, we should close the summarizer and let it recover.
|
|
116
126
|
* This delay's goal is to prevent tight restart loops
|
|
117
127
|
*/
|
|
118
|
-
const defaultCloseSummarizerDelayMs =
|
|
128
|
+
const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
|
|
119
129
|
/**
|
|
120
130
|
* @deprecated - use ContainerRuntimeMessage instead
|
|
121
131
|
*/
|
|
@@ -151,9 +161,23 @@ export function getDeviceSpec() {
|
|
|
151
161
|
};
|
|
152
162
|
}
|
|
153
163
|
}
|
|
154
|
-
catch
|
|
164
|
+
catch { }
|
|
155
165
|
return {};
|
|
156
166
|
}
|
|
167
|
+
/**
|
|
168
|
+
* Older loader doesn't have a submitBatchFn member, this is the older way of submitting a batch.
|
|
169
|
+
* Rather than exposing the submitFn (now deprecated) and IDeltaManager (dangerous to hand out) to the Outbox,
|
|
170
|
+
* we can provide a partially-applied function to keep those items private to the ContainerRuntime.
|
|
171
|
+
*/
|
|
172
|
+
export const makeLegacySendBatchFn = (submitFn, deltaManager) => (batch) => {
|
|
173
|
+
for (const message of batch.content) {
|
|
174
|
+
submitFn(MessageType.Operation,
|
|
175
|
+
// For back-compat (submitFn only works on deserialized content)
|
|
176
|
+
message.contents === undefined ? undefined : JSON.parse(message.contents), true, // batch
|
|
177
|
+
message.metadata);
|
|
178
|
+
}
|
|
179
|
+
deltaManager.flush();
|
|
180
|
+
};
|
|
157
181
|
/**
|
|
158
182
|
* Represents the runtime of the container. Contains helper functions/state of the container.
|
|
159
183
|
* It will define the store level mappings.
|
|
@@ -162,11 +186,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
162
186
|
/**
|
|
163
187
|
* @internal
|
|
164
188
|
*/
|
|
165
|
-
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration
|
|
166
|
-
|
|
167
|
-
|
|
189
|
+
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration = {
|
|
190
|
+
// the defaults
|
|
191
|
+
...DefaultSummaryConfiguration,
|
|
192
|
+
// the runtime configuration overrides
|
|
193
|
+
...runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
194
|
+
}, initializeEntryPoint) {
|
|
168
195
|
super();
|
|
169
|
-
this.context = context;
|
|
170
196
|
this.registry = registry;
|
|
171
197
|
this.runtimeOptions = runtimeOptions;
|
|
172
198
|
this.containerScope = containerScope;
|
|
@@ -195,51 +221,56 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
195
221
|
signalTimestamp: 0,
|
|
196
222
|
trackingSignalSequenceNumber: undefined,
|
|
197
223
|
};
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
this.
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
222
|
-
// because it is a misuse of the API rather than an expected failure.
|
|
223
|
-
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
224
|
-
}
|
|
224
|
+
const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
|
|
225
|
+
this.innerDeltaManager = deltaManager;
|
|
226
|
+
this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
|
|
227
|
+
// Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
|
|
228
|
+
// This makes ContainerRuntime the final gatekeeper for outgoing messages.
|
|
229
|
+
this.submitFn = submitFn;
|
|
230
|
+
this.submitBatchFn = submitBatchFn;
|
|
231
|
+
this.submitSummaryFn = submitSummaryFn;
|
|
232
|
+
this.submitSignalFn = submitSignalFn;
|
|
233
|
+
this.options = options;
|
|
234
|
+
this.clientDetails = clientDetails;
|
|
235
|
+
this.isSummarizerClient = this.clientDetails.type === summarizerClientType;
|
|
236
|
+
this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
|
|
237
|
+
this._getClientId = () => context.clientId;
|
|
238
|
+
this._getAttachState = () => context.attachState;
|
|
239
|
+
this.getAbsoluteUrl = async (relativeUrl) => {
|
|
240
|
+
if (context.getAbsoluteUrl === undefined) {
|
|
241
|
+
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
242
|
+
}
|
|
243
|
+
if (this.attachState !== AttachState.Attached) {
|
|
244
|
+
return undefined;
|
|
245
|
+
}
|
|
246
|
+
return context.getAbsoluteUrl(relativeUrl);
|
|
225
247
|
};
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
this.
|
|
248
|
+
// TODO: Consider that the Container could just listen to these events itself, or even more appropriately maybe the
|
|
249
|
+
// customer should observe dirty state on the runtime (the owner of dirty state) directly, rather than on the IContainer.
|
|
250
|
+
this.on("dirty", () => context.updateDirtyContainerState(true));
|
|
251
|
+
this.on("saved", () => context.updateDirtyContainerState(false));
|
|
252
|
+
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
253
|
+
this.disposeFn = disposeFn ?? closeFn;
|
|
254
|
+
// In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
|
|
255
|
+
this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
|
|
256
|
+
this.mc = createChildMonitoringContext({
|
|
257
|
+
logger: this.logger,
|
|
258
|
+
namespace: "ContainerRuntime",
|
|
259
|
+
});
|
|
229
260
|
let loadSummaryNumber;
|
|
230
261
|
// Get the container creation metadata. For new container, we initialize these. For existing containers,
|
|
231
262
|
// get the values from the metadata blob.
|
|
232
263
|
if (existing) {
|
|
233
264
|
this.createContainerMetadata = {
|
|
234
|
-
createContainerRuntimeVersion: metadata
|
|
235
|
-
createContainerTimestamp: metadata
|
|
265
|
+
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
266
|
+
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
236
267
|
};
|
|
237
268
|
// summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
|
|
238
269
|
// the count is reset to 0.
|
|
239
|
-
loadSummaryNumber =
|
|
270
|
+
loadSummaryNumber = metadata?.summaryNumber ?? 0;
|
|
240
271
|
// Enabling the IdCompressor is a one-way operation and we only want to
|
|
241
272
|
// allow new containers to turn it on
|
|
242
|
-
this.idCompressorEnabled =
|
|
273
|
+
this.idCompressorEnabled = metadata?.idCompressorEnabled ?? false;
|
|
243
274
|
}
|
|
244
275
|
else {
|
|
245
276
|
this.createContainerMetadata = {
|
|
@@ -248,24 +279,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
248
279
|
};
|
|
249
280
|
loadSummaryNumber = 0;
|
|
250
281
|
this.idCompressorEnabled =
|
|
251
|
-
|
|
282
|
+
this.mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled") ??
|
|
283
|
+
idCompressor !== undefined;
|
|
252
284
|
}
|
|
253
285
|
this.nextSummaryNumber = loadSummaryNumber + 1;
|
|
254
|
-
this.messageAtLastSummary = metadata
|
|
255
|
-
|
|
256
|
-
|
|
286
|
+
this.messageAtLastSummary = metadata?.message;
|
|
287
|
+
// Note that we only need to pull the *initial* connected state from the context.
|
|
288
|
+
// Later updates come through calls to setConnectionState.
|
|
289
|
+
this._connected = connected;
|
|
290
|
+
this.gcTombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement(metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName] /* current */);
|
|
257
291
|
this.mc.logger.sendTelemetryEvent({
|
|
258
292
|
eventName: "GCFeatureMatrix",
|
|
259
|
-
metadataValue: JSON.stringify(metadata
|
|
293
|
+
metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
|
|
260
294
|
inputs: JSON.stringify({
|
|
261
295
|
gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
|
|
262
296
|
}),
|
|
263
297
|
});
|
|
264
|
-
this.telemetryDocumentId =
|
|
298
|
+
this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
|
|
265
299
|
this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
|
|
266
300
|
const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
|
|
267
301
|
const opGroupingManager = new OpGroupingManager(this.groupedBatchingEnabled);
|
|
268
|
-
const opSplitter = new OpSplitter(chunks, this.
|
|
302
|
+
const opSplitter = new OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
|
|
269
303
|
this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
|
|
270
304
|
this.handleContext = new ContainerFluidHandleContext("", this);
|
|
271
305
|
if (this.summaryConfiguration.state === "enabled") {
|
|
@@ -284,9 +318,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
284
318
|
this.idCompressor = idCompressor;
|
|
285
319
|
}
|
|
286
320
|
this.maxConsecutiveReconnects =
|
|
287
|
-
|
|
321
|
+
this.mc.config.getNumber(maxConsecutiveReconnectsKey) ??
|
|
322
|
+
this.defaultMaxConsecutiveReconnects;
|
|
288
323
|
if (runtimeOptions.flushMode === FlushModeExperimental.Async &&
|
|
289
|
-
|
|
324
|
+
supportedFeatures?.get("referenceSequenceNumbers") !== true) {
|
|
290
325
|
// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
|
|
291
326
|
this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
|
|
292
327
|
this._flushMode = FlushMode.TurnBased;
|
|
@@ -294,8 +329,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
294
329
|
else {
|
|
295
330
|
this._flushMode = runtimeOptions.flushMode;
|
|
296
331
|
}
|
|
297
|
-
const pendingRuntimeState =
|
|
298
|
-
const maxSnapshotCacheDurationMs =
|
|
332
|
+
const pendingRuntimeState = pendingLocalState;
|
|
333
|
+
const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
|
|
299
334
|
if (maxSnapshotCacheDurationMs !== undefined &&
|
|
300
335
|
maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
|
|
301
336
|
// This is a runtime enforcement of what's already explicit in the policy's type itself,
|
|
@@ -306,27 +341,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
306
341
|
this.garbageCollector = GarbageCollector.create({
|
|
307
342
|
runtime: this,
|
|
308
343
|
gcOptions: this.runtimeOptions.gcOptions,
|
|
309
|
-
baseSnapshot
|
|
344
|
+
baseSnapshot,
|
|
310
345
|
baseLogger: this.mc.logger,
|
|
311
346
|
existing,
|
|
312
347
|
metadata,
|
|
313
348
|
createContainerMetadata: this.createContainerMetadata,
|
|
314
|
-
isSummarizerClient: this.
|
|
349
|
+
isSummarizerClient: this.isSummarizerClient,
|
|
315
350
|
getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
|
|
316
|
-
getLastSummaryTimestampMs: () =>
|
|
351
|
+
getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
|
|
317
352
|
readAndParseBlob: async (id) => readAndParse(this.storage, id),
|
|
318
353
|
// GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
|
|
319
354
|
// delta manager would always return false for summarizer client.
|
|
320
355
|
activeConnection: () => this.innerDeltaManager.active,
|
|
321
356
|
});
|
|
322
357
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
323
|
-
this.summarizerNode = createRootSummarizerNodeWithGC(
|
|
358
|
+
this.summarizerNode = createRootSummarizerNodeWithGC(createChildLogger({ logger: this.logger, namespace: "SummarizerNode" }),
|
|
324
359
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
325
360
|
async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext),
|
|
326
361
|
// Latest change sequence number, no changes since summary applied yet
|
|
327
362
|
loadedFromSequenceNumber,
|
|
328
363
|
// Summary reference sequence number, undefined if no summary yet
|
|
329
|
-
|
|
364
|
+
baseSnapshot !== undefined ? loadedFromSequenceNumber : undefined, {
|
|
330
365
|
// Must set to false to prevent sending summary handle which would be pointing to
|
|
331
366
|
// a summary with an older protocol state.
|
|
332
367
|
canReuseHandle: false,
|
|
@@ -340,19 +375,19 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
340
375
|
async (fullGC) => this.getGCDataInternal(fullGC),
|
|
341
376
|
// Function to get the GC details from the base snapshot we loaded from.
|
|
342
377
|
async () => this.garbageCollector.getBaseGCDetails());
|
|
343
|
-
if (
|
|
344
|
-
this.summarizerNode.updateBaseSummaryState(
|
|
378
|
+
if (baseSnapshot) {
|
|
379
|
+
this.summarizerNode.updateBaseSummaryState(baseSnapshot);
|
|
345
380
|
}
|
|
346
|
-
this.dataStores = new DataStores(getSummaryForDatastores(
|
|
381
|
+
this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit({ type: ContainerMessageType.Attach, contents: attachMsg }), (id, createParam) => (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
|
|
347
382
|
this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
|
|
348
383
|
if (!this.disposed) {
|
|
349
|
-
this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
|
|
384
|
+
this.submit({ type: ContainerMessageType.BlobAttach, contents: undefined }, undefined, {
|
|
350
385
|
localId,
|
|
351
386
|
blobId,
|
|
352
387
|
});
|
|
353
388
|
}
|
|
354
|
-
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState
|
|
355
|
-
this.scheduleManager = new ScheduleManager(
|
|
389
|
+
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState?.pendingAttachmentBlobs, (error) => this.closeFn(error));
|
|
390
|
+
this.scheduleManager = new ScheduleManager(this.innerDeltaManager, this, () => this.clientId, createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }));
|
|
356
391
|
this.pendingStateManager = new PendingStateManager({
|
|
357
392
|
applyStashedOp: this.applyStashedOp.bind(this),
|
|
358
393
|
clientId: () => this.clientId,
|
|
@@ -360,7 +395,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
360
395
|
connected: () => this.connected,
|
|
361
396
|
reSubmit: this.reSubmit.bind(this),
|
|
362
397
|
reSubmitBatch: this.reSubmitBatch.bind(this),
|
|
363
|
-
|
|
398
|
+
isActiveConnection: () => this.innerDeltaManager.active,
|
|
399
|
+
}, pendingRuntimeState?.pending, this.logger);
|
|
364
400
|
const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
|
|
365
401
|
const compressionOptions = disableCompression === true
|
|
366
402
|
? {
|
|
@@ -369,10 +405,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
369
405
|
}
|
|
370
406
|
: runtimeOptions.compressionOptions;
|
|
371
407
|
const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
|
|
408
|
+
const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
|
|
372
409
|
this.outbox = new Outbox({
|
|
373
410
|
shouldSend: () => this.canSendOps(),
|
|
374
411
|
pendingStateManager: this.pendingStateManager,
|
|
375
|
-
|
|
412
|
+
submitBatchFn: this.submitBatchFn,
|
|
413
|
+
legacySendBatchFn,
|
|
376
414
|
compressor: new OpCompressor(this.mc.logger),
|
|
377
415
|
splitter: opSplitter,
|
|
378
416
|
config: {
|
|
@@ -391,44 +429,50 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
391
429
|
opReentrancy: () => this.ensureNoDataModelChangesCalls > 0,
|
|
392
430
|
closeContainer: this.closeFn,
|
|
393
431
|
});
|
|
394
|
-
this.
|
|
432
|
+
this._quorum = quorum;
|
|
433
|
+
this._quorum.on("removeMember", (clientId) => {
|
|
395
434
|
this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
|
|
396
435
|
});
|
|
397
|
-
|
|
436
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
437
|
+
this._audience = audience;
|
|
398
438
|
const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
|
|
399
|
-
this.closeSummarizerDelayMs = closeSummarizerDelayOverride
|
|
439
|
+
this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
|
|
400
440
|
this.validateSummaryBeforeUpload =
|
|
401
|
-
|
|
441
|
+
this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
|
|
402
442
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
403
443
|
this.dirtyContainer =
|
|
404
|
-
this.
|
|
405
|
-
|
|
406
|
-
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
444
|
+
this.attachState !== AttachState.Attached || this.hasPendingMessages();
|
|
445
|
+
context.updateDirtyContainerState(this.dirtyContainer);
|
|
407
446
|
if (this.summariesDisabled) {
|
|
408
447
|
this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
|
|
409
448
|
}
|
|
410
449
|
else {
|
|
411
|
-
const orderedClientLogger =
|
|
412
|
-
|
|
413
|
-
|
|
450
|
+
const orderedClientLogger = createChildLogger({
|
|
451
|
+
logger: this.logger,
|
|
452
|
+
namespace: "OrderedClientElection",
|
|
453
|
+
});
|
|
454
|
+
const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
|
|
455
|
+
const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
|
|
414
456
|
this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
|
|
415
|
-
if (this.
|
|
457
|
+
if (this.isSummarizerClient) {
|
|
416
458
|
this._summarizer = new Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime,
|
|
417
459
|
// Summarization runs in summarizer client and needs access to the real (non-proxy) active
|
|
418
460
|
// information. The proxy delta manager would always return false for summarizer client.
|
|
419
461
|
() => this.innerDeltaManager.active));
|
|
420
462
|
}
|
|
421
|
-
else if (SummarizerClientElection.clientDetailsPermitElection(this.
|
|
463
|
+
else if (SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
|
|
422
464
|
// Only create a SummaryManager and SummarizerClientElection
|
|
423
465
|
// if summaries are enabled and we are not the summarizer client.
|
|
424
466
|
const defaultAction = () => {
|
|
425
467
|
if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
|
|
426
|
-
this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
468
|
+
this.mc.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
427
469
|
// unregister default to no log on every op after falling behind
|
|
428
470
|
// and register summary ack handler to re-register this handler
|
|
429
471
|
// after successful summary
|
|
430
472
|
this.summaryCollection.once(MessageType.SummaryAck, () => {
|
|
431
|
-
this.logger.sendTelemetryEvent({
|
|
473
|
+
this.mc.logger.sendTelemetryEvent({
|
|
474
|
+
eventName: "SummaryStatus:CaughtUp",
|
|
475
|
+
});
|
|
432
476
|
// we've caught up, so re-register the default action to monitor for
|
|
433
477
|
// falling behind, and unregister ourself
|
|
434
478
|
this.summaryCollection.on("default", defaultAction);
|
|
@@ -439,12 +483,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
439
483
|
this.summaryCollection.on("default", defaultAction);
|
|
440
484
|
// Create the SummaryManager and mark the initial state
|
|
441
485
|
this.summaryManager = new SummaryManager(this.summarizerClientElection, this, // IConnectedState
|
|
442
|
-
this.summaryCollection, this.logger, this.formRequestSummarizerFn(
|
|
486
|
+
this.summaryCollection, this.logger, this.formRequestSummarizerFn(loader), new Throttler(60 * 1000, // 60 sec delay window
|
|
443
487
|
30 * 1000, // 30 sec max delay
|
|
444
488
|
// throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
|
|
445
489
|
formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
|
|
446
490
|
initialDelayMs: this.initialSummarizerDelayMs,
|
|
447
491
|
}, this.heuristicsDisabled);
|
|
492
|
+
this.summaryManager.on("summarize", (eventProps) => {
|
|
493
|
+
this.emit("summarize", eventProps);
|
|
494
|
+
});
|
|
448
495
|
this.summaryManager.start();
|
|
449
496
|
}
|
|
450
497
|
}
|
|
@@ -469,27 +516,44 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
469
516
|
this.replayPendingStates();
|
|
470
517
|
});
|
|
471
518
|
// logging hardware telemetry
|
|
472
|
-
logger.sendTelemetryEvent(
|
|
473
|
-
|
|
519
|
+
logger.sendTelemetryEvent({
|
|
520
|
+
eventName: "DeviceSpec",
|
|
521
|
+
...getDeviceSpec(),
|
|
522
|
+
});
|
|
523
|
+
this.mc.logger.sendTelemetryEvent({
|
|
524
|
+
eventName: "ContainerLoadStats",
|
|
525
|
+
...this.createContainerMetadata,
|
|
526
|
+
...this.dataStores.containerLoadStats,
|
|
527
|
+
summaryNumber: loadSummaryNumber,
|
|
528
|
+
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
529
|
+
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
530
|
+
gcVersion: metadata?.gcFeature,
|
|
531
|
+
options: JSON.stringify(runtimeOptions),
|
|
532
|
+
featureGates: JSON.stringify({
|
|
474
533
|
disableCompression,
|
|
475
534
|
disableOpReentryCheck,
|
|
476
535
|
disableChunking,
|
|
477
536
|
disableAttachReorder: this.disableAttachReorder,
|
|
478
537
|
disablePartialFlush,
|
|
479
538
|
idCompressorEnabled: this.idCompressorEnabled,
|
|
480
|
-
summaryStateUpdateMethod: this.summaryStateUpdateMethod,
|
|
481
539
|
closeSummarizerDelayOverride,
|
|
482
|
-
}),
|
|
483
|
-
|
|
540
|
+
}),
|
|
541
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
542
|
+
groupedBatchingEnabled: this.groupedBatchingEnabled,
|
|
543
|
+
});
|
|
544
|
+
ReportOpPerfTelemetry(this.clientId, this.deltaManager, this.logger);
|
|
484
545
|
BindBatchTracker(this, this.logger);
|
|
485
546
|
this.entryPoint = new LazyPromise(async () => {
|
|
486
|
-
if (this.
|
|
547
|
+
if (this.isSummarizerClient) {
|
|
487
548
|
assert(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
|
|
488
549
|
return this._summarizer;
|
|
489
550
|
}
|
|
490
|
-
return initializeEntryPoint
|
|
551
|
+
return initializeEntryPoint?.(this);
|
|
491
552
|
});
|
|
492
553
|
}
|
|
554
|
+
/**
|
|
555
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
556
|
+
*/
|
|
493
557
|
get IFluidRouter() {
|
|
494
558
|
return this;
|
|
495
559
|
}
|
|
@@ -535,22 +599,34 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
535
599
|
* This object should provide all the functionality that the Container is expected to provide to the loader layer.
|
|
536
600
|
*/
|
|
537
601
|
static async loadRuntime(params) {
|
|
538
|
-
|
|
539
|
-
const
|
|
602
|
+
const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
|
|
603
|
+
const initializeEntryPoint = params.initializeEntryPoint ??
|
|
604
|
+
(async (containerRuntime) => ({
|
|
605
|
+
get IFluidRouter() {
|
|
606
|
+
return this;
|
|
607
|
+
},
|
|
608
|
+
async request(req) {
|
|
609
|
+
return containerRuntime.request(req);
|
|
610
|
+
},
|
|
611
|
+
}));
|
|
540
612
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
541
613
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
542
614
|
const backCompatContext = context;
|
|
543
|
-
const passLogger =
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
615
|
+
const passLogger = backCompatContext.taggedLogger ??
|
|
616
|
+
// eslint-disable-next-line import/no-deprecated
|
|
617
|
+
new TaggedLoggerAdapter(backCompatContext.logger);
|
|
618
|
+
const logger = createChildLogger({
|
|
619
|
+
logger: passLogger,
|
|
620
|
+
properties: {
|
|
621
|
+
all: {
|
|
622
|
+
runtimeVersion: pkgVersion,
|
|
623
|
+
},
|
|
547
624
|
},
|
|
548
625
|
});
|
|
549
626
|
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = false, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
|
|
550
627
|
const registry = new FluidDataStoreRegistry(registryEntries);
|
|
551
628
|
const tryFetchBlob = async (blobName) => {
|
|
552
|
-
|
|
553
|
-
const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
|
|
629
|
+
const blobId = context.baseSnapshot?.blobs[blobName];
|
|
554
630
|
if (context.baseSnapshot && blobId) {
|
|
555
631
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
556
632
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
@@ -565,16 +641,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
565
641
|
tryFetchBlob(aliasBlobName),
|
|
566
642
|
tryFetchBlob(idCompressorBlobName),
|
|
567
643
|
]);
|
|
568
|
-
const loadExisting = existing === true || context.existing === true;
|
|
569
644
|
// read snapshot blobs needed for BlobManager to load
|
|
570
|
-
const blobManagerSnapshot = await BlobManager.load(
|
|
645
|
+
const blobManagerSnapshot = await BlobManager.load(context.baseSnapshot?.trees[blobsTreeName], async (id) => {
|
|
571
646
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
572
647
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
573
648
|
assert(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
|
|
574
649
|
return readAndParse(context.storage, id);
|
|
575
650
|
});
|
|
576
651
|
// Verify summary runtime sequence number matches protocol sequence number.
|
|
577
|
-
const runtimeSequenceNumber =
|
|
652
|
+
const runtimeSequenceNumber = metadata?.message?.sequenceNumber;
|
|
578
653
|
// When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
|
|
579
654
|
if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
|
|
580
655
|
const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
|
|
@@ -589,22 +664,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
589
664
|
logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
|
|
590
665
|
}
|
|
591
666
|
else {
|
|
592
|
-
// Call both close and dispose as closeFn implementation will no longer dispose runtime in future
|
|
593
667
|
context.closeFn(error);
|
|
594
|
-
(_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
|
|
595
668
|
}
|
|
596
669
|
}
|
|
597
670
|
}
|
|
598
|
-
const idCompressorEnabled =
|
|
671
|
+
const idCompressorEnabled = metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
|
|
599
672
|
let idCompressor;
|
|
600
673
|
if (idCompressorEnabled) {
|
|
601
674
|
const { IdCompressor, createSessionId } = await import("./id-compressor");
|
|
602
675
|
idCompressor =
|
|
603
676
|
serializedIdCompressor !== undefined
|
|
604
677
|
? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
|
|
605
|
-
:
|
|
678
|
+
: IdCompressor.create(logger);
|
|
606
679
|
}
|
|
607
|
-
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks
|
|
680
|
+
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
|
|
608
681
|
summaryOptions,
|
|
609
682
|
gcOptions,
|
|
610
683
|
loadSequenceNumberVerification,
|
|
@@ -615,8 +688,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
615
688
|
enableRuntimeIdCompressor,
|
|
616
689
|
enableOpReentryCheck,
|
|
617
690
|
enableGroupedBatching,
|
|
618
|
-
}, containerScope, logger,
|
|
691
|
+
}, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
|
|
619
692
|
initializeEntryPoint);
|
|
693
|
+
await runtime.blobManager.processStashedChanges();
|
|
620
694
|
// It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
|
|
621
695
|
// at 1, so we won't see a replayed saved op with a sequence number of 0.
|
|
622
696
|
await runtime.pendingStateManager.applyStashedOpsAt(0);
|
|
@@ -624,38 +698,16 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
624
698
|
await runtime.initializeBaseState();
|
|
625
699
|
return runtime;
|
|
626
700
|
}
|
|
627
|
-
get options() {
|
|
628
|
-
return this.context.options;
|
|
629
|
-
}
|
|
630
701
|
get clientId() {
|
|
631
|
-
return this.
|
|
632
|
-
}
|
|
633
|
-
get clientDetails() {
|
|
634
|
-
return this.context.clientDetails;
|
|
702
|
+
return this._getClientId();
|
|
635
703
|
}
|
|
636
704
|
get storage() {
|
|
637
705
|
return this._storage;
|
|
638
706
|
}
|
|
707
|
+
/** @deprecated - The functionality is no longer exposed publicly */
|
|
639
708
|
get reSubmitFn() {
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
}
|
|
643
|
-
get disposeFn() {
|
|
644
|
-
var _a;
|
|
645
|
-
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
646
|
-
return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
|
|
647
|
-
}
|
|
648
|
-
get closeFn() {
|
|
649
|
-
if (this._summarizer !== undefined) {
|
|
650
|
-
// In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
|
|
651
|
-
return this.disposeFn;
|
|
652
|
-
}
|
|
653
|
-
// Also call disposeFn to retain functionality of runtime being disposed on close
|
|
654
|
-
return (error) => {
|
|
655
|
-
var _a, _b;
|
|
656
|
-
this.context.closeFn(error);
|
|
657
|
-
(_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
|
|
658
|
-
};
|
|
709
|
+
return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
|
|
710
|
+
// Note: compatDetails is not included in this deprecated API
|
|
659
711
|
}
|
|
660
712
|
get flushMode() {
|
|
661
713
|
return this._flushMode;
|
|
@@ -667,7 +719,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
667
719
|
return this.registry;
|
|
668
720
|
}
|
|
669
721
|
get attachState() {
|
|
670
|
-
return this.
|
|
722
|
+
return this._getAttachState();
|
|
671
723
|
}
|
|
672
724
|
get IFluidHandleContext() {
|
|
673
725
|
return this.handleContext;
|
|
@@ -694,8 +746,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
694
746
|
}
|
|
695
747
|
/** clientId of parent (non-summarizing) container that owns summarizer container */
|
|
696
748
|
get summarizerClientId() {
|
|
697
|
-
|
|
698
|
-
return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
|
|
749
|
+
return this.summarizerClientElection?.electedClientId;
|
|
699
750
|
}
|
|
700
751
|
get disposed() {
|
|
701
752
|
return this._disposed;
|
|
@@ -732,12 +783,11 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
732
783
|
await this.garbageCollector.initializeBaseState();
|
|
733
784
|
}
|
|
734
785
|
dispose(error) {
|
|
735
|
-
var _a;
|
|
736
786
|
if (this._disposed) {
|
|
737
787
|
return;
|
|
738
788
|
}
|
|
739
789
|
this._disposed = true;
|
|
740
|
-
this.logger.sendTelemetryEvent({
|
|
790
|
+
this.mc.logger.sendTelemetryEvent({
|
|
741
791
|
eventName: "ContainerRuntimeDisposed",
|
|
742
792
|
isDirty: this.isDirty,
|
|
743
793
|
lastSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
@@ -747,7 +797,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
747
797
|
this.summaryManager.dispose();
|
|
748
798
|
}
|
|
749
799
|
this.garbageCollector.dispose();
|
|
750
|
-
|
|
800
|
+
this._summarizer?.dispose();
|
|
751
801
|
this.dataStores.dispose();
|
|
752
802
|
this.pendingStateManager.dispose();
|
|
753
803
|
this.emit("dispose");
|
|
@@ -756,6 +806,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
756
806
|
/**
|
|
757
807
|
* Notifies this object about the request made to the container.
|
|
758
808
|
* @param request - Request made to the handler.
|
|
809
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
759
810
|
*/
|
|
760
811
|
async request(request) {
|
|
761
812
|
try {
|
|
@@ -807,7 +858,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
807
858
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
808
859
|
// unintentionally modifying the url if that changes.
|
|
809
860
|
assert(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
|
|
810
|
-
return dataStore.
|
|
861
|
+
return dataStore.request(subRequest);
|
|
811
862
|
}
|
|
812
863
|
return create404Response(request);
|
|
813
864
|
}
|
|
@@ -822,19 +873,17 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
822
873
|
return this.entryPoint;
|
|
823
874
|
}
|
|
824
875
|
internalId(maybeAlias) {
|
|
825
|
-
|
|
826
|
-
return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
|
|
876
|
+
return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
|
|
827
877
|
}
|
|
828
878
|
async getDataStoreFromRequest(id, request) {
|
|
829
|
-
var _a, _b, _c;
|
|
830
879
|
const headerData = {};
|
|
831
|
-
if (typeof
|
|
880
|
+
if (typeof request.headers?.[RuntimeHeaders.wait] === "boolean") {
|
|
832
881
|
headerData.wait = request.headers[RuntimeHeaders.wait];
|
|
833
882
|
}
|
|
834
|
-
if (typeof
|
|
883
|
+
if (typeof request.headers?.[RuntimeHeaders.viaHandle] === "boolean") {
|
|
835
884
|
headerData.viaHandle = request.headers[RuntimeHeaders.viaHandle];
|
|
836
885
|
}
|
|
837
|
-
if (typeof
|
|
886
|
+
if (typeof request.headers?.[AllowTombstoneRequestHeaderKey] === "boolean") {
|
|
838
887
|
headerData.allowTombstone = request.headers[AllowTombstoneRequestHeaderKey];
|
|
839
888
|
}
|
|
840
889
|
await this.dataStores.waitIfPendingAlias(id);
|
|
@@ -844,22 +893,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
844
893
|
// Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
|
|
845
894
|
// the same as GC nodes id.
|
|
846
895
|
const urlWithoutQuery = trimLeadingAndTrailingSlashes(request.url.split("?")[0]);
|
|
847
|
-
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request
|
|
896
|
+
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request?.headers);
|
|
848
897
|
return dataStoreChannel;
|
|
849
898
|
}
|
|
850
899
|
/** Adds the container's metadata to the given summary tree. */
|
|
851
900
|
addMetadataToSummary(summaryTree) {
|
|
852
|
-
|
|
853
|
-
|
|
901
|
+
const metadata = {
|
|
902
|
+
...this.createContainerMetadata,
|
|
854
903
|
// Increment the summary number for the next summary that will be generated.
|
|
855
|
-
summaryNumber: this.nextSummaryNumber++,
|
|
904
|
+
summaryNumber: this.nextSummaryNumber++,
|
|
905
|
+
summaryFormatVersion: 1,
|
|
906
|
+
...this.garbageCollector.getMetadata(),
|
|
856
907
|
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
857
908
|
// last summary.
|
|
858
|
-
message:
|
|
909
|
+
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
|
|
910
|
+
this.messageAtLastSummary,
|
|
911
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
912
|
+
idCompressorEnabled: this.idCompressorEnabled ? true : undefined,
|
|
913
|
+
};
|
|
859
914
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
|
|
860
915
|
}
|
|
861
916
|
addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
|
|
862
|
-
var _a;
|
|
863
917
|
this.addMetadataToSummary(summaryTree);
|
|
864
918
|
if (this.idCompressorEnabled) {
|
|
865
919
|
assert(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
|
|
@@ -875,7 +929,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
875
929
|
addBlobToSummary(summaryTree, aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
876
930
|
}
|
|
877
931
|
if (this.summarizerClientElection) {
|
|
878
|
-
const electedSummarizerContent = JSON.stringify(
|
|
932
|
+
const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
|
|
879
933
|
addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
|
|
880
934
|
}
|
|
881
935
|
const blobManagerSummary = this.blobManager.summarize();
|
|
@@ -912,7 +966,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
912
966
|
this.mc.logger.sendTelemetryEvent({
|
|
913
967
|
eventName: "ReconnectsWithNoProgress",
|
|
914
968
|
attempts: this.consecutiveReconnects,
|
|
915
|
-
pendingMessages: this.
|
|
969
|
+
pendingMessages: this.pendingMessagesCount,
|
|
916
970
|
});
|
|
917
971
|
}
|
|
918
972
|
return this.consecutiveReconnects < this.maxConsecutiveReconnects;
|
|
@@ -922,7 +976,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
922
976
|
// in their own batches before the originating batch is sent.
|
|
923
977
|
// Therefore, receiving them while attempting to send the originating batch
|
|
924
978
|
// does not mean that the container is making any progress.
|
|
925
|
-
if (
|
|
979
|
+
if (message?.type !== ContainerMessageType.ChunkedOp) {
|
|
926
980
|
this.consecutiveReconnects = 0;
|
|
927
981
|
}
|
|
928
982
|
}
|
|
@@ -972,13 +1026,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
972
1026
|
*/
|
|
973
1027
|
parseOpContent(serializedContent) {
|
|
974
1028
|
assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
|
|
975
|
-
const
|
|
976
|
-
assert(
|
|
977
|
-
return { type
|
|
1029
|
+
const { type, contents, compatDetails } = JSON.parse(serializedContent);
|
|
1030
|
+
assert(type !== undefined, 0x6d6 /* incorrect op content format */);
|
|
1031
|
+
return { type, contents, compatDetails };
|
|
978
1032
|
}
|
|
979
1033
|
async applyStashedOp(op) {
|
|
980
1034
|
// Need to parse from string for back-compat
|
|
981
|
-
const { type, contents } = this.parseOpContent(op);
|
|
1035
|
+
const { type, contents, compatDetails } = this.parseOpContent(op);
|
|
982
1036
|
switch (type) {
|
|
983
1037
|
case ContainerMessageType.FluidDataStoreOp:
|
|
984
1038
|
return this.dataStores.applyStashedOp(contents);
|
|
@@ -994,8 +1048,22 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
994
1048
|
throw new Error("chunkedOp not expected here");
|
|
995
1049
|
case ContainerMessageType.Rejoin:
|
|
996
1050
|
throw new Error("rejoin not expected here");
|
|
997
|
-
default:
|
|
998
|
-
|
|
1051
|
+
default: {
|
|
1052
|
+
// This should be extremely rare for stashed ops.
|
|
1053
|
+
// It would require a newer runtime stashing ops and then an older one applying them,
|
|
1054
|
+
// e.g. if an app rolled back its container version
|
|
1055
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1056
|
+
if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
|
|
1057
|
+
const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
|
|
1058
|
+
messageDetails: JSON.stringify({
|
|
1059
|
+
type,
|
|
1060
|
+
compatBehavior,
|
|
1061
|
+
}),
|
|
1062
|
+
});
|
|
1063
|
+
this.closeFn(error);
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
999
1067
|
}
|
|
1000
1068
|
}
|
|
1001
1069
|
setConnectionState(connected, clientId) {
|
|
@@ -1007,24 +1075,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1007
1075
|
// Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
|
|
1008
1076
|
return;
|
|
1009
1077
|
}
|
|
1010
|
-
// If attachment blobs were added while disconnected, we need to delay
|
|
1011
|
-
// propagation of the "connected" event until we have uploaded them to
|
|
1012
|
-
// ensure we don't submit ops referencing a blob that has not been uploaded
|
|
1013
|
-
// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
|
|
1014
|
-
const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
|
|
1015
|
-
if (connecting && this.blobManager.hasPendingOfflineUploads) {
|
|
1016
|
-
assert(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
|
|
1017
|
-
assert(!!clientId, 0x393 /* Must have clientId when connecting */);
|
|
1018
|
-
this.delayConnectClientId = clientId;
|
|
1019
|
-
this.blobManager.onConnected().then(() => {
|
|
1020
|
-
// make sure we didn't reconnect before the promise resolved
|
|
1021
|
-
if (this.delayConnectClientId === clientId && !this.disposed) {
|
|
1022
|
-
this.delayConnectClientId = undefined;
|
|
1023
|
-
this.setConnectionStateCore(connected, clientId);
|
|
1024
|
-
}
|
|
1025
|
-
}, (error) => this.closeFn(error));
|
|
1026
|
-
return;
|
|
1027
|
-
}
|
|
1028
1078
|
this.setConnectionStateCore(connected, clientId);
|
|
1029
1079
|
}
|
|
1030
1080
|
setConnectionStateCore(connected, clientId) {
|
|
@@ -1033,6 +1083,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1033
1083
|
// There might be no change of state due to Container calling this API after loading runtime.
|
|
1034
1084
|
const changeOfState = this._connected !== connected;
|
|
1035
1085
|
const reconnection = changeOfState && !connected;
|
|
1086
|
+
// We need to flush the ops currently collected by Outbox to preserve original order.
|
|
1087
|
+
// This flush NEEDS to happen before we set the ContainerRuntime to "connected".
|
|
1088
|
+
// We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
|
|
1089
|
+
if (changeOfState && connected) {
|
|
1090
|
+
this.flush();
|
|
1091
|
+
}
|
|
1036
1092
|
this._connected = connected;
|
|
1037
1093
|
if (!connected) {
|
|
1038
1094
|
this._perfSignalData.signalsLost = 0;
|
|
@@ -1049,7 +1105,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1049
1105
|
this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
|
|
1050
1106
|
dataLoss: 1,
|
|
1051
1107
|
attempts: this.consecutiveReconnects,
|
|
1052
|
-
pendingMessages: this.
|
|
1108
|
+
pendingMessages: this.pendingMessagesCount,
|
|
1053
1109
|
}));
|
|
1054
1110
|
return;
|
|
1055
1111
|
}
|
|
@@ -1066,18 +1122,23 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1066
1122
|
}
|
|
1067
1123
|
process(messageArg, local) {
|
|
1068
1124
|
this.verifyNotClosed();
|
|
1069
|
-
// Whether or not the message
|
|
1125
|
+
// Whether or not the message appears to be a runtime message from an up-to-date client.
|
|
1070
1126
|
// It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
|
|
1071
1127
|
// or something different, like a system message.
|
|
1072
|
-
const
|
|
1128
|
+
const modernRuntimeMessage = messageArg.type === MessageType.Operation;
|
|
1073
1129
|
// Do shallow copy of message, as the processing flow will modify it.
|
|
1074
|
-
const messageCopy =
|
|
1130
|
+
const messageCopy = { ...messageArg };
|
|
1075
1131
|
for (const message of this.remoteMessageProcessor.process(messageCopy)) {
|
|
1076
|
-
this.processCore(message, local,
|
|
1132
|
+
this.processCore(message, local, modernRuntimeMessage);
|
|
1077
1133
|
}
|
|
1078
1134
|
}
|
|
1079
|
-
|
|
1080
|
-
|
|
1135
|
+
/**
|
|
1136
|
+
* Direct the message to the correct subsystem for processing, and implement other side effects
|
|
1137
|
+
* @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
|
|
1138
|
+
* @param local - Did this client send the op?
|
|
1139
|
+
* @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
|
|
1140
|
+
*/
|
|
1141
|
+
processCore(message, local, modernRuntimeMessage) {
|
|
1081
1142
|
// Surround the actual processing of the operation with messages to the schedule manager indicating
|
|
1082
1143
|
// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
|
|
1083
1144
|
// messages once a batch has been fully processed.
|
|
@@ -1085,7 +1146,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1085
1146
|
this._processedClientSequenceNumber = message.clientSequenceNumber;
|
|
1086
1147
|
try {
|
|
1087
1148
|
let localOpMetadata;
|
|
1088
|
-
if (local &&
|
|
1149
|
+
if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
|
|
1089
1150
|
localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
|
|
1090
1151
|
}
|
|
1091
1152
|
// If there are no more pending messages after processing a local message,
|
|
@@ -1093,45 +1154,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1093
1154
|
if (!this.hasPendingMessages()) {
|
|
1094
1155
|
this.updateDocumentDirtyState(false);
|
|
1095
1156
|
}
|
|
1096
|
-
|
|
1097
|
-
|
|
1098
|
-
case ContainerMessageType.Attach:
|
|
1099
|
-
this.dataStores.processAttachMessage(message, local);
|
|
1100
|
-
break;
|
|
1101
|
-
case ContainerMessageType.Alias:
|
|
1102
|
-
this.processAliasMessage(message, localOpMetadata, local);
|
|
1103
|
-
break;
|
|
1104
|
-
case ContainerMessageType.FluidDataStoreOp:
|
|
1105
|
-
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1106
|
-
break;
|
|
1107
|
-
case ContainerMessageType.BlobAttach:
|
|
1108
|
-
this.blobManager.processBlobAttachOp(message, local);
|
|
1109
|
-
break;
|
|
1110
|
-
case ContainerMessageType.IdAllocation:
|
|
1111
|
-
assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1112
|
-
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1113
|
-
break;
|
|
1114
|
-
case ContainerMessageType.ChunkedOp:
|
|
1115
|
-
case ContainerMessageType.Rejoin:
|
|
1116
|
-
break;
|
|
1117
|
-
default:
|
|
1118
|
-
if (runtimeMessage) {
|
|
1119
|
-
const error = DataProcessingError.create(
|
|
1120
|
-
// Former assert 0x3ce
|
|
1121
|
-
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1122
|
-
local,
|
|
1123
|
-
type: message.type,
|
|
1124
|
-
contentType: typeof message.contents,
|
|
1125
|
-
batch: (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch,
|
|
1126
|
-
compression: message.compression,
|
|
1127
|
-
});
|
|
1128
|
-
this.closeFn(error);
|
|
1129
|
-
throw error;
|
|
1130
|
-
}
|
|
1131
|
-
}
|
|
1132
|
-
if (runtimeMessage || this.groupedBatchingEnabled) {
|
|
1133
|
-
this.emit("op", message, runtimeMessage);
|
|
1134
|
-
}
|
|
1157
|
+
this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
|
|
1158
|
+
this.emit("op", message, modernRuntimeMessage);
|
|
1135
1159
|
this.scheduleManager.afterOpProcessing(undefined, message);
|
|
1136
1160
|
if (local) {
|
|
1137
1161
|
// If we have processed a local op, this means that the container is
|
|
@@ -1145,8 +1169,59 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1145
1169
|
throw e;
|
|
1146
1170
|
}
|
|
1147
1171
|
}
|
|
1148
|
-
|
|
1149
|
-
|
|
1172
|
+
/**
|
|
1173
|
+
* Assuming the given message is also a ContainerRuntimeMessage,
|
|
1174
|
+
* checks its type and dispatches the message to the appropriate handler in the runtime.
|
|
1175
|
+
* Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
|
|
1176
|
+
*/
|
|
1177
|
+
validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
|
|
1178
|
+
// Optimistically extract ContainerRuntimeMessage-specific props from the message
|
|
1179
|
+
const { type: maybeContainerMessageType, compatDetails } = message;
|
|
1180
|
+
switch (maybeContainerMessageType) {
|
|
1181
|
+
case ContainerMessageType.Attach:
|
|
1182
|
+
this.dataStores.processAttachMessage(message, local);
|
|
1183
|
+
break;
|
|
1184
|
+
case ContainerMessageType.Alias:
|
|
1185
|
+
this.dataStores.processAliasMessage(message, localOpMetadata, local);
|
|
1186
|
+
break;
|
|
1187
|
+
case ContainerMessageType.FluidDataStoreOp:
|
|
1188
|
+
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1189
|
+
break;
|
|
1190
|
+
case ContainerMessageType.BlobAttach:
|
|
1191
|
+
this.blobManager.processBlobAttachOp(message, local);
|
|
1192
|
+
break;
|
|
1193
|
+
case ContainerMessageType.IdAllocation:
|
|
1194
|
+
assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1195
|
+
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1196
|
+
break;
|
|
1197
|
+
case ContainerMessageType.ChunkedOp:
|
|
1198
|
+
case ContainerMessageType.Rejoin:
|
|
1199
|
+
break;
|
|
1200
|
+
default: {
|
|
1201
|
+
// If we didn't necessarily expect a runtime message type, then no worries - just return
|
|
1202
|
+
// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
|
|
1203
|
+
if (!expectRuntimeMessageType) {
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1207
|
+
if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
|
|
1208
|
+
const error = DataProcessingError.create(
|
|
1209
|
+
// Former assert 0x3ce
|
|
1210
|
+
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1211
|
+
local,
|
|
1212
|
+
messageDetails: JSON.stringify({
|
|
1213
|
+
type: message.type,
|
|
1214
|
+
contentType: typeof message.contents,
|
|
1215
|
+
compatBehavior,
|
|
1216
|
+
batch: message.metadata?.batch,
|
|
1217
|
+
compression: message.compression,
|
|
1218
|
+
}),
|
|
1219
|
+
});
|
|
1220
|
+
this.closeFn(error);
|
|
1221
|
+
throw error;
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1150
1225
|
}
|
|
1151
1226
|
/**
|
|
1152
1227
|
* Emits the Signal event and update the perf signal data.
|
|
@@ -1154,7 +1229,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1154
1229
|
*/
|
|
1155
1230
|
sendSignalTelemetryEvent(clientSignalSequenceNumber) {
|
|
1156
1231
|
const duration = Date.now() - this._perfSignalData.signalTimestamp;
|
|
1157
|
-
this.logger.sendPerformanceEvent({
|
|
1232
|
+
this.mc.logger.sendPerformanceEvent({
|
|
1158
1233
|
eventName: "SignalLatency",
|
|
1159
1234
|
duration,
|
|
1160
1235
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1177,7 +1252,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1177
1252
|
this._perfSignalData.trackingSignalSequenceNumber) {
|
|
1178
1253
|
this._perfSignalData.signalsLost++;
|
|
1179
1254
|
this._perfSignalData.trackingSignalSequenceNumber = undefined;
|
|
1180
|
-
this.logger.sendErrorEvent({
|
|
1255
|
+
this.mc.logger.sendErrorEvent({
|
|
1181
1256
|
eventName: "SignalLost",
|
|
1182
1257
|
type: envelope.contents.type,
|
|
1183
1258
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1201,6 +1276,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1201
1276
|
}
|
|
1202
1277
|
this.dataStores.processSignal(envelope.address, transformed, local);
|
|
1203
1278
|
}
|
|
1279
|
+
/**
|
|
1280
|
+
* Returns the runtime of the data store.
|
|
1281
|
+
* @param id - Id supplied during creating the data store.
|
|
1282
|
+
* @param wait - True if you want to wait for it.
|
|
1283
|
+
* @deprecated - Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
|
|
1284
|
+
*/
|
|
1204
1285
|
async getRootDataStore(id, wait = true) {
|
|
1205
1286
|
return this.getRootDataStoreChannel(id, wait);
|
|
1206
1287
|
}
|
|
@@ -1262,9 +1343,25 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1262
1343
|
}
|
|
1263
1344
|
return result;
|
|
1264
1345
|
}
|
|
1265
|
-
|
|
1266
|
-
|
|
1267
|
-
|
|
1346
|
+
/**
|
|
1347
|
+
* Returns the aliased data store's entryPoint, given the alias.
|
|
1348
|
+
* @param alias - The alias for the data store.
|
|
1349
|
+
* @returns The data store's entry point ({@link @fluidframework/core-interfaces#IFluidHandle}) if it exists and is aliased.
|
|
1350
|
+
* Returns undefined if no data store has been assigned the given alias.
|
|
1351
|
+
*/
|
|
1352
|
+
async getAliasedDataStoreEntryPoint(alias) {
|
|
1353
|
+
await this.dataStores.waitIfPendingAlias(alias);
|
|
1354
|
+
const internalId = this.internalId(alias);
|
|
1355
|
+
const context = await this.dataStores.getDataStoreIfAvailable(internalId, { wait: false });
|
|
1356
|
+
// If the data store is not available or not an alias, return undefined.
|
|
1357
|
+
if (context === undefined || !(await context.isRoot())) {
|
|
1358
|
+
return undefined;
|
|
1359
|
+
}
|
|
1360
|
+
const channel = await context.realize();
|
|
1361
|
+
if (channel.entryPoint === undefined) {
|
|
1362
|
+
throw new UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
|
|
1363
|
+
}
|
|
1364
|
+
return channel.entryPoint;
|
|
1268
1365
|
}
|
|
1269
1366
|
createDetachedRootDataStore(pkg, rootDataStoreId) {
|
|
1270
1367
|
if (rootDataStoreId.includes("/")) {
|
|
@@ -1275,16 +1372,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1275
1372
|
createDetachedDataStore(pkg) {
|
|
1276
1373
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1277
1374
|
}
|
|
1278
|
-
async
|
|
1279
|
-
const
|
|
1280
|
-
|
|
1281
|
-
.
|
|
1282
|
-
|
|
1375
|
+
async createDataStore(pkg) {
|
|
1376
|
+
const id = uuid();
|
|
1377
|
+
return channelToDataStore(await this.dataStores
|
|
1378
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id)
|
|
1379
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1283
1380
|
}
|
|
1284
|
-
|
|
1285
|
-
|
|
1381
|
+
/**
|
|
1382
|
+
* @deprecated 0.16 Issue #1537, #3631
|
|
1383
|
+
* @internal
|
|
1384
|
+
*/
|
|
1385
|
+
async _createDataStoreWithProps(pkg, props, id = uuid()) {
|
|
1386
|
+
return channelToDataStore(await this.dataStores
|
|
1286
1387
|
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
|
|
1287
|
-
.realize();
|
|
1388
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1288
1389
|
}
|
|
1289
1390
|
canSendOps() {
|
|
1290
1391
|
// Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
|
|
@@ -1298,11 +1399,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1298
1399
|
return this.flushMode !== FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
|
|
1299
1400
|
}
|
|
1300
1401
|
getQuorum() {
|
|
1301
|
-
return this.
|
|
1402
|
+
return this._quorum;
|
|
1302
1403
|
}
|
|
1303
1404
|
getAudience() {
|
|
1304
|
-
|
|
1305
|
-
return this.context.audience;
|
|
1405
|
+
return this._audience;
|
|
1306
1406
|
}
|
|
1307
1407
|
/**
|
|
1308
1408
|
* Returns true of container is dirty, i.e. there are some pending local changes that
|
|
@@ -1311,7 +1411,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1311
1411
|
get isDirty() {
|
|
1312
1412
|
return this.dirtyContainer;
|
|
1313
1413
|
}
|
|
1314
|
-
isContainerMessageDirtyable(type, contents) {
|
|
1414
|
+
isContainerMessageDirtyable({ type, contents }) {
|
|
1315
1415
|
// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
|
|
1316
1416
|
// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
|
|
1317
1417
|
if (type === ContainerMessageType.Attach) {
|
|
@@ -1351,11 +1451,11 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1351
1451
|
submitSignal(type, content) {
|
|
1352
1452
|
this.verifyNotClosed();
|
|
1353
1453
|
const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
|
|
1354
|
-
return this.
|
|
1454
|
+
return this.submitSignalFn(envelope);
|
|
1355
1455
|
}
|
|
1356
1456
|
submitDataStoreSignal(address, type, content) {
|
|
1357
1457
|
const envelope = this.createNewSignalEnvelope(address, type, content);
|
|
1358
|
-
return this.
|
|
1458
|
+
return this.submitSignalFn(envelope);
|
|
1359
1459
|
}
|
|
1360
1460
|
setAttachState(attachState) {
|
|
1361
1461
|
if (attachState === AttachState.Attaching) {
|
|
@@ -1388,22 +1488,17 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1388
1488
|
this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
|
|
1389
1489
|
return summarizeResult.summary;
|
|
1390
1490
|
}
|
|
1391
|
-
async getAbsoluteUrl(relativeUrl) {
|
|
1392
|
-
if (this.context.getAbsoluteUrl === undefined) {
|
|
1393
|
-
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
1394
|
-
}
|
|
1395
|
-
if (this.attachState !== AttachState.Attached) {
|
|
1396
|
-
return undefined;
|
|
1397
|
-
}
|
|
1398
|
-
return this.context.getAbsoluteUrl(relativeUrl);
|
|
1399
|
-
}
|
|
1400
1491
|
async summarizeInternal(fullTree, trackState, telemetryContext) {
|
|
1401
1492
|
const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
|
|
1402
1493
|
// Wrap data store summaries in .channels subtree.
|
|
1403
1494
|
wrapSummaryInChannelsTree(summarizeResult);
|
|
1404
1495
|
const pathPartsForChildren = [channelsTreeName];
|
|
1405
1496
|
this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
|
|
1406
|
-
return
|
|
1497
|
+
return {
|
|
1498
|
+
...summarizeResult,
|
|
1499
|
+
id: "",
|
|
1500
|
+
pathPartsForChildren,
|
|
1501
|
+
};
|
|
1407
1502
|
}
|
|
1408
1503
|
/**
|
|
1409
1504
|
* Returns a summary of the runtime at the current sequence number.
|
|
@@ -1421,16 +1516,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1421
1516
|
runSweep,
|
|
1422
1517
|
});
|
|
1423
1518
|
try {
|
|
1424
|
-
let gcStats;
|
|
1425
1519
|
if (runGC) {
|
|
1426
|
-
|
|
1520
|
+
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
|
|
1427
1521
|
}
|
|
1428
1522
|
const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
|
|
1429
1523
|
assert(summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1430
|
-
return { stats, summary
|
|
1524
|
+
return { stats, summary };
|
|
1431
1525
|
}
|
|
1432
1526
|
finally {
|
|
1433
|
-
this.logger.sendTelemetryEvent({
|
|
1527
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1434
1528
|
eventName: "SummarizeTelemetry",
|
|
1435
1529
|
details: telemetryContext.serialize(),
|
|
1436
1530
|
});
|
|
@@ -1492,7 +1586,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1492
1586
|
/**
|
|
1493
1587
|
* After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
|
|
1494
1588
|
* @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
|
|
1495
|
-
* @returns
|
|
1589
|
+
* @returns The routes of nodes that were deleted.
|
|
1496
1590
|
*/
|
|
1497
1591
|
deleteSweepReadyNodes(sweepReadyRoutes) {
|
|
1498
1592
|
const { dataStoreRoutes, blobManagerRoutes } = this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
|
|
@@ -1512,21 +1606,19 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1512
1606
|
* Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
|
|
1513
1607
|
*/
|
|
1514
1608
|
getCurrentReferenceTimestampMs() {
|
|
1515
|
-
var _a, _b, _c;
|
|
1516
1609
|
// Use the timestamp of the last message seen by this client as that is server generated. If no messages have
|
|
1517
1610
|
// been processed, use the timestamp of the message from the last summary.
|
|
1518
|
-
return
|
|
1611
|
+
return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
|
|
1519
1612
|
}
|
|
1520
1613
|
/**
|
|
1521
1614
|
* Returns the type of the GC node. Currently, there are nodes that belong to the root ("/"), data stores or
|
|
1522
1615
|
* blob manager.
|
|
1523
1616
|
*/
|
|
1524
1617
|
getNodeType(nodePath) {
|
|
1525
|
-
var _a;
|
|
1526
1618
|
if (this.isBlobPath(nodePath)) {
|
|
1527
1619
|
return GCNodeType.Blob;
|
|
1528
1620
|
}
|
|
1529
|
-
return
|
|
1621
|
+
return this.dataStores.getGCNodeType(nodePath) ?? GCNodeType.Other;
|
|
1530
1622
|
}
|
|
1531
1623
|
/**
|
|
1532
1624
|
* Called by GC to retrieve the package path of the node with the given path. The node should belong to a
|
|
@@ -1556,7 +1648,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1556
1648
|
/**
|
|
1557
1649
|
* From a given list of routes, separate and return routes that belong to blob manager and data stores.
|
|
1558
1650
|
* @param routes - A list of routes that can belong to data stores or blob manager.
|
|
1559
|
-
* @returns
|
|
1651
|
+
* @returns Two route lists - One that contains routes for blob manager and another one that contains routes
|
|
1560
1652
|
* for data stores.
|
|
1561
1653
|
*/
|
|
1562
1654
|
getDataStoreAndBlobManagerRoutes(routes) {
|
|
@@ -1597,23 +1689,62 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1597
1689
|
* @param options - options controlling how the summary is generated or submitted
|
|
1598
1690
|
*/
|
|
1599
1691
|
async submitSummary(options) {
|
|
1600
|
-
|
|
1601
|
-
const { fullTree = false, refreshLatestAck, summaryLogger } = options;
|
|
1692
|
+
const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
|
|
1602
1693
|
// The summary number for this summary. This will be updated during the summary process, so get it now and
|
|
1603
1694
|
// use it for all events logged during this summary.
|
|
1604
1695
|
const summaryNumber = this.nextSummaryNumber;
|
|
1605
|
-
const summaryNumberLogger =
|
|
1606
|
-
|
|
1696
|
+
const summaryNumberLogger = createChildLogger({
|
|
1697
|
+
logger: summaryLogger,
|
|
1698
|
+
properties: {
|
|
1699
|
+
all: { summaryNumber },
|
|
1700
|
+
},
|
|
1607
1701
|
});
|
|
1608
1702
|
assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
|
|
1609
1703
|
let latestSnapshotVersionId;
|
|
1610
1704
|
if (refreshLatestAck) {
|
|
1611
|
-
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(
|
|
1705
|
+
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(createChildLogger({
|
|
1706
|
+
logger: summaryNumberLogger,
|
|
1707
|
+
properties: { all: { safeSummary: true } },
|
|
1708
|
+
}));
|
|
1612
1709
|
const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
|
|
1613
1710
|
latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
|
|
1614
1711
|
// We might need to catch up to the latest summary's reference sequence number before pausing.
|
|
1615
1712
|
await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
|
|
1616
1713
|
}
|
|
1714
|
+
// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
|
|
1715
|
+
// incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
|
|
1716
|
+
// within the timeout, check if it should be failed or can continue.
|
|
1717
|
+
if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
|
|
1718
|
+
const countBefore = this.pendingMessagesCount;
|
|
1719
|
+
// The timeout for waiting for pending ops can be overridden via configurations.
|
|
1720
|
+
const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
|
|
1721
|
+
defaultPendingOpsWaitTimeoutMs;
|
|
1722
|
+
await new Promise((resolve, reject) => {
|
|
1723
|
+
const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
|
|
1724
|
+
this.once("saved", () => {
|
|
1725
|
+
clearTimeout(timeoutId);
|
|
1726
|
+
resolve();
|
|
1727
|
+
});
|
|
1728
|
+
this.once("dispose", () => {
|
|
1729
|
+
clearTimeout(timeoutId);
|
|
1730
|
+
reject(new Error("Runtime is disposed while summarizing"));
|
|
1731
|
+
});
|
|
1732
|
+
});
|
|
1733
|
+
// Log that there are pending ops while summarizing. This will help us gather data on how often this
|
|
1734
|
+
// happens, whether we attempted to wait for these ops to be acked and what was the result.
|
|
1735
|
+
summaryNumberLogger.sendTelemetryEvent({
|
|
1736
|
+
eventName: "PendingOpsWhileSummarizing",
|
|
1737
|
+
saved: this.hasPendingMessages() ? false : true,
|
|
1738
|
+
timeout: pendingOpsTimeout,
|
|
1739
|
+
countBefore,
|
|
1740
|
+
countAfter: this.pendingMessagesCount,
|
|
1741
|
+
});
|
|
1742
|
+
// There could still be pending ops. Check if summary should fail or continue.
|
|
1743
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
|
|
1744
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1745
|
+
return pendingMessagesFailResult;
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1617
1748
|
const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
|
|
1618
1749
|
let summaryRefSeqNum;
|
|
1619
1750
|
try {
|
|
@@ -1628,7 +1759,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1628
1759
|
this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
|
|
1629
1760
|
// Helper function to check whether we should still continue between each async step.
|
|
1630
1761
|
const checkContinue = () => {
|
|
1631
|
-
var _a;
|
|
1632
1762
|
// Do not check for loss of connectivity directly! Instead leave it up to
|
|
1633
1763
|
// RunWhileConnectedCoordinator to control policy in a single place.
|
|
1634
1764
|
// This will allow easier change of design if we chose to. For example, we may chose to allow
|
|
@@ -1651,7 +1781,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1651
1781
|
error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
|
|
1652
1782
|
};
|
|
1653
1783
|
}
|
|
1654
|
-
assert(summaryRefSeqNum ===
|
|
1784
|
+
assert(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
|
|
1655
1785
|
if (lastAck !== this.summaryCollection.latestAck) {
|
|
1656
1786
|
return {
|
|
1657
1787
|
continue: false,
|
|
@@ -1690,6 +1820,25 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1690
1820
|
error,
|
|
1691
1821
|
};
|
|
1692
1822
|
}
|
|
1823
|
+
// If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
|
|
1824
|
+
if (this.validateSummaryBeforeUpload) {
|
|
1825
|
+
// Validate that the summaries generated by summarize nodes is correct.
|
|
1826
|
+
const validateResult = this.summarizerNode.validateSummary();
|
|
1827
|
+
if (!validateResult.success) {
|
|
1828
|
+
const { success, ...loggingProps } = validateResult;
|
|
1829
|
+
const error = new RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
|
|
1830
|
+
return {
|
|
1831
|
+
stage: "base",
|
|
1832
|
+
referenceSequenceNumber: summaryRefSeqNum,
|
|
1833
|
+
minimumSequenceNumber,
|
|
1834
|
+
error,
|
|
1835
|
+
};
|
|
1836
|
+
}
|
|
1837
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
|
|
1838
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1839
|
+
return pendingMessagesFailResult;
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1693
1842
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
1694
1843
|
// Now that we have generated the summary, update the message at last summary to the last message processed.
|
|
1695
1844
|
this.messageAtLastSummary = this.deltaManager.lastMessage;
|
|
@@ -1702,7 +1851,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1702
1851
|
const gcSummaryTreeStats = summaryTree.tree[gcTreeKey]
|
|
1703
1852
|
? calculateStats(summaryTree.tree[gcTreeKey])
|
|
1704
1853
|
: undefined;
|
|
1705
|
-
const summaryStats =
|
|
1854
|
+
const summaryStats = {
|
|
1855
|
+
dataStoreCount: this.dataStores.size,
|
|
1856
|
+
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
1857
|
+
gcStateUpdatedDataStoreCount: this.garbageCollector.updatedDSCountSinceLastSummary,
|
|
1858
|
+
gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
|
|
1859
|
+
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
1860
|
+
summaryNumber,
|
|
1861
|
+
...partialStats,
|
|
1862
|
+
};
|
|
1706
1863
|
const generateSummaryData = {
|
|
1707
1864
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1708
1865
|
minimumSequenceNumber,
|
|
@@ -1711,19 +1868,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1711
1868
|
generateDuration: trace.trace().duration,
|
|
1712
1869
|
forcedFullTree,
|
|
1713
1870
|
};
|
|
1714
|
-
// If validateSummaryBeforeUpload is true, validate that the summary generated by the summarizer nodes is
|
|
1715
|
-
// correct before this summary is uploaded.
|
|
1716
|
-
if (this.validateSummaryBeforeUpload) {
|
|
1717
|
-
const validateResult = this.summarizerNode.validateSummary();
|
|
1718
|
-
if (!validateResult.success) {
|
|
1719
|
-
const { success } = validateResult, loggingProps = __rest(validateResult, ["success"]);
|
|
1720
|
-
const error = new RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, Object.assign({}, loggingProps));
|
|
1721
|
-
return Object.assign(Object.assign({ stage: "base" }, generateSummaryData), { error });
|
|
1722
|
-
}
|
|
1723
|
-
}
|
|
1724
1871
|
continueResult = checkContinue();
|
|
1725
1872
|
if (!continueResult.continue) {
|
|
1726
|
-
return
|
|
1873
|
+
return { stage: "generate", ...generateSummaryData, error: continueResult.error };
|
|
1727
1874
|
}
|
|
1728
1875
|
// It may happen that the lastAck it not correct due to missing summaryAck in case of single commit
|
|
1729
1876
|
// summary. So if the previous summarizer closes just after submitting the summary and before
|
|
@@ -1731,7 +1878,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1731
1878
|
// latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
|
|
1732
1879
|
// the one fetched from storage as parent as that is the latest.
|
|
1733
1880
|
let summaryContext;
|
|
1734
|
-
if (
|
|
1881
|
+
if (lastAck?.summaryAck.contents.handle !== latestSnapshotVersionId &&
|
|
1735
1882
|
latestSnapshotVersionId !== undefined) {
|
|
1736
1883
|
summaryContext = {
|
|
1737
1884
|
proposalHandle: undefined,
|
|
@@ -1742,7 +1889,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1742
1889
|
else if (lastAck === undefined) {
|
|
1743
1890
|
summaryContext = {
|
|
1744
1891
|
proposalHandle: undefined,
|
|
1745
|
-
ackHandle:
|
|
1892
|
+
ackHandle: this.loadedFromVersionId,
|
|
1746
1893
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1747
1894
|
};
|
|
1748
1895
|
}
|
|
@@ -1758,7 +1905,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1758
1905
|
handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
|
|
1759
1906
|
}
|
|
1760
1907
|
catch (error) {
|
|
1761
|
-
return
|
|
1908
|
+
return { stage: "generate", ...generateSummaryData, error };
|
|
1762
1909
|
}
|
|
1763
1910
|
const parent = summaryContext.ackHandle;
|
|
1764
1911
|
const summaryMessage = {
|
|
@@ -1768,25 +1915,34 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1768
1915
|
message,
|
|
1769
1916
|
parents: parent ? [parent] : [],
|
|
1770
1917
|
};
|
|
1771
|
-
const uploadData =
|
|
1918
|
+
const uploadData = {
|
|
1919
|
+
...generateSummaryData,
|
|
1920
|
+
handle,
|
|
1921
|
+
uploadDuration: trace.trace().duration,
|
|
1922
|
+
};
|
|
1772
1923
|
continueResult = checkContinue();
|
|
1773
1924
|
if (!continueResult.continue) {
|
|
1774
|
-
return
|
|
1925
|
+
return { stage: "upload", ...uploadData, error: continueResult.error };
|
|
1775
1926
|
}
|
|
1776
1927
|
let clientSequenceNumber;
|
|
1777
1928
|
try {
|
|
1778
1929
|
clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
|
|
1779
1930
|
}
|
|
1780
1931
|
catch (error) {
|
|
1781
|
-
return
|
|
1932
|
+
return { stage: "upload", ...uploadData, error };
|
|
1782
1933
|
}
|
|
1783
|
-
const submitData =
|
|
1934
|
+
const submitData = {
|
|
1935
|
+
stage: "submit",
|
|
1936
|
+
...uploadData,
|
|
1937
|
+
clientSequenceNumber,
|
|
1938
|
+
submitOpDuration: trace.trace().duration,
|
|
1939
|
+
};
|
|
1784
1940
|
try {
|
|
1785
1941
|
// If validateSummaryBeforeUpload is false, the summary should be validated in this step.
|
|
1786
1942
|
this.summarizerNode.completeSummary(handle, !this.validateSummaryBeforeUpload /* validate */);
|
|
1787
1943
|
}
|
|
1788
1944
|
catch (error) {
|
|
1789
|
-
return
|
|
1945
|
+
return { stage: "upload", ...uploadData, error };
|
|
1790
1946
|
}
|
|
1791
1947
|
return submitData;
|
|
1792
1948
|
}
|
|
@@ -1794,7 +1950,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1794
1950
|
// Cleanup wip summary in case of failure
|
|
1795
1951
|
this.summarizerNode.clearSummary();
|
|
1796
1952
|
// ! This needs to happen before we resume inbound queues to ensure heuristics are tracked correctly
|
|
1797
|
-
|
|
1953
|
+
this._summarizer?.recordSummaryAttempt?.(summaryRefSeqNum);
|
|
1798
1954
|
// Restart the delta manager
|
|
1799
1955
|
this.deltaManager.inbound.resume();
|
|
1800
1956
|
if (shouldPauseInboundSignal) {
|
|
@@ -1802,8 +1958,55 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1802
1958
|
}
|
|
1803
1959
|
}
|
|
1804
1960
|
}
|
|
1961
|
+
/**
|
|
1962
|
+
* This helper is called during summarization. If there are pending ops, it will return a failed summarize result
|
|
1963
|
+
* (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
|
|
1964
|
+
* @param logger - The logger to be used for sending telemetry.
|
|
1965
|
+
* @param referenceSequenceNumber - The reference sequence number of the summary attempt.
|
|
1966
|
+
* @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
|
|
1967
|
+
* @param finalAttempt - Whether this is the final summary attempt.
|
|
1968
|
+
* @param beforeSummaryGeneration - Whether this is called before summary generation or after.
|
|
1969
|
+
* @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
|
|
1970
|
+
*/
|
|
1971
|
+
async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
|
|
1972
|
+
if (!this.hasPendingMessages()) {
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
// If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
|
|
1976
|
+
// This is a fallback to make progress in documents where there are consistently pending ops in
|
|
1977
|
+
// the summarizer.
|
|
1978
|
+
if (finalAttempt &&
|
|
1979
|
+
this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
|
|
1980
|
+
const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
|
|
1981
|
+
logger.sendErrorEvent({
|
|
1982
|
+
eventName: "SkipFailingIncorrectSummary",
|
|
1983
|
+
referenceSequenceNumber,
|
|
1984
|
+
minimumSequenceNumber,
|
|
1985
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
1986
|
+
}, error);
|
|
1987
|
+
}
|
|
1988
|
+
else {
|
|
1989
|
+
// The retry delay when there are pending ops can be overridden via config so that we can adjust it
|
|
1990
|
+
// based on telemetry while we decide on a stable number.
|
|
1991
|
+
const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
|
|
1992
|
+
defaultPendingOpsRetryDelayMs;
|
|
1993
|
+
const error = new RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
|
|
1994
|
+
count: this.pendingMessagesCount,
|
|
1995
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
1996
|
+
});
|
|
1997
|
+
return {
|
|
1998
|
+
stage: "base",
|
|
1999
|
+
referenceSequenceNumber,
|
|
2000
|
+
minimumSequenceNumber,
|
|
2001
|
+
error,
|
|
2002
|
+
};
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
get pendingMessagesCount() {
|
|
2006
|
+
return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
|
|
2007
|
+
}
|
|
1805
2008
|
hasPendingMessages() {
|
|
1806
|
-
return this.
|
|
2009
|
+
return this.pendingMessagesCount !== 0;
|
|
1807
2010
|
}
|
|
1808
2011
|
updateDocumentDirtyState(dirty) {
|
|
1809
2012
|
if (this.attachState !== AttachState.Attached) {
|
|
@@ -1819,7 +2022,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1819
2022
|
this.dirtyContainer = dirty;
|
|
1820
2023
|
if (this.emitDirtyDocumentEvent) {
|
|
1821
2024
|
this.emit(dirty ? "dirty" : "saved");
|
|
1822
|
-
this.context.updateDirtyContainerState(dirty);
|
|
1823
2025
|
}
|
|
1824
2026
|
}
|
|
1825
2027
|
submitDataStoreOp(id, contents, localOpMetadata = undefined) {
|
|
@@ -1827,21 +2029,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1827
2029
|
address: id,
|
|
1828
2030
|
contents,
|
|
1829
2031
|
};
|
|
1830
|
-
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2032
|
+
this.submit({ type: ContainerMessageType.FluidDataStoreOp, contents: envelope }, localOpMetadata);
|
|
1831
2033
|
}
|
|
1832
2034
|
submitDataStoreAliasOp(contents, localOpMetadata) {
|
|
1833
2035
|
const aliasMessage = contents;
|
|
1834
2036
|
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
1835
2037
|
throw new UsageError("malformedDataStoreAliasMessage");
|
|
1836
2038
|
}
|
|
1837
|
-
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2039
|
+
this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
|
|
1838
2040
|
}
|
|
1839
|
-
async uploadBlob(blob) {
|
|
2041
|
+
async uploadBlob(blob, signal) {
|
|
1840
2042
|
this.verifyNotClosed();
|
|
1841
|
-
return this.blobManager.createBlob(blob);
|
|
2043
|
+
return this.blobManager.createBlob(blob, signal);
|
|
1842
2044
|
}
|
|
1843
2045
|
maybeSubmitIdAllocationOp(type) {
|
|
1844
|
-
var _a, _b;
|
|
1845
2046
|
if (type !== ContainerMessageType.IdAllocation) {
|
|
1846
2047
|
let idAllocationBatchMessage;
|
|
1847
2048
|
let idRange;
|
|
@@ -1849,7 +2050,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1849
2050
|
assert(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
|
|
1850
2051
|
idRange = this.idCompressor.takeNextCreationRange();
|
|
1851
2052
|
// Don't include the idRange if there weren't any Ids allocated
|
|
1852
|
-
idRange =
|
|
2053
|
+
idRange = idRange?.ids !== undefined ? idRange : undefined;
|
|
1853
2054
|
}
|
|
1854
2055
|
if (idRange !== undefined) {
|
|
1855
2056
|
const idAllocationMessage = {
|
|
@@ -1860,7 +2061,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1860
2061
|
contents: JSON.stringify(idAllocationMessage),
|
|
1861
2062
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
1862
2063
|
metadata: undefined,
|
|
1863
|
-
localOpMetadata:
|
|
2064
|
+
localOpMetadata: this.idCompressor?.serialize(true),
|
|
1864
2065
|
type: ContainerMessageType.IdAllocation,
|
|
1865
2066
|
};
|
|
1866
2067
|
}
|
|
@@ -1869,20 +2070,21 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1869
2070
|
}
|
|
1870
2071
|
}
|
|
1871
2072
|
}
|
|
1872
|
-
submit(
|
|
2073
|
+
submit(containerRuntimeMessage, localOpMetadata = undefined, metadata = undefined) {
|
|
1873
2074
|
this.verifyNotClosed();
|
|
1874
2075
|
this.verifyCanSubmitOps();
|
|
1875
2076
|
// There should be no ops in detached container state!
|
|
1876
2077
|
assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
|
|
1877
|
-
const serializedContent = JSON.stringify(
|
|
2078
|
+
const serializedContent = JSON.stringify(containerRuntimeMessage);
|
|
1878
2079
|
// Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
|
|
1879
2080
|
// container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
|
|
1880
2081
|
if (this.innerDeltaManager.readOnlyInfo.readonly) {
|
|
1881
|
-
this.logger.sendTelemetryEvent({
|
|
2082
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1882
2083
|
eventName: "SubmitOpInReadonly",
|
|
1883
2084
|
connected: this.connected,
|
|
1884
2085
|
});
|
|
1885
2086
|
}
|
|
2087
|
+
const type = containerRuntimeMessage.type;
|
|
1886
2088
|
const message = {
|
|
1887
2089
|
contents: serializedContent,
|
|
1888
2090
|
type,
|
|
@@ -1939,7 +2141,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1939
2141
|
this.closeFn(error);
|
|
1940
2142
|
throw error;
|
|
1941
2143
|
}
|
|
1942
|
-
if (this.isContainerMessageDirtyable(
|
|
2144
|
+
if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
|
|
1943
2145
|
this.updateDocumentDirtyState(true);
|
|
1944
2146
|
}
|
|
1945
2147
|
}
|
|
@@ -1982,9 +2184,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1982
2184
|
// System message should not be sent in the middle of the batch.
|
|
1983
2185
|
assert(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
|
|
1984
2186
|
// back-compat: ADO #1385: Make this call unconditional in the future
|
|
1985
|
-
return this.
|
|
1986
|
-
? this.
|
|
1987
|
-
: this.
|
|
2187
|
+
return this.submitSummaryFn !== undefined
|
|
2188
|
+
? this.submitSummaryFn(contents, referenceSequenceNumber)
|
|
2189
|
+
: this.submitFn(MessageType.Summarize, contents, false);
|
|
1988
2190
|
}
|
|
1989
2191
|
/**
|
|
1990
2192
|
* Throw an error if the runtime is closed. Methods that are expected to potentially
|
|
@@ -2031,32 +2233,33 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2031
2233
|
}
|
|
2032
2234
|
reSubmit(message) {
|
|
2033
2235
|
// Need to parse from string for back-compat
|
|
2034
|
-
const
|
|
2035
|
-
this.reSubmitCore(
|
|
2236
|
+
const containerRuntimeMessage = this.parseOpContent(message.content);
|
|
2237
|
+
this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
|
|
2036
2238
|
}
|
|
2037
2239
|
/**
|
|
2038
2240
|
* Finds the right store and asks it to resubmit the message. This typically happens when we
|
|
2039
2241
|
* reconnect and there are pending messages.
|
|
2040
|
-
* @param
|
|
2242
|
+
* @param message - The original ContainerRuntimeMessage.
|
|
2041
2243
|
* @param localOpMetadata - The local metadata associated with the original message.
|
|
2042
2244
|
*/
|
|
2043
|
-
reSubmitCore(
|
|
2044
|
-
|
|
2245
|
+
reSubmitCore(message, localOpMetadata, opMetadata) {
|
|
2246
|
+
const contents = message.contents;
|
|
2247
|
+
switch (message.type) {
|
|
2045
2248
|
case ContainerMessageType.FluidDataStoreOp:
|
|
2046
2249
|
// For Operations, call resubmitDataStoreOp which will find the right store
|
|
2047
2250
|
// and trigger resubmission on it.
|
|
2048
|
-
this.dataStores.resubmitDataStoreOp(
|
|
2251
|
+
this.dataStores.resubmitDataStoreOp(contents, localOpMetadata);
|
|
2049
2252
|
break;
|
|
2050
2253
|
case ContainerMessageType.Attach:
|
|
2051
2254
|
case ContainerMessageType.Alias:
|
|
2052
|
-
this.submit(
|
|
2255
|
+
this.submit(message, localOpMetadata);
|
|
2053
2256
|
break;
|
|
2054
2257
|
case ContainerMessageType.IdAllocation:
|
|
2055
2258
|
// Remove the stashedState from the op if it's a stashed op
|
|
2056
|
-
if (
|
|
2057
|
-
delete
|
|
2259
|
+
if (contents.stashedState !== undefined) {
|
|
2260
|
+
delete contents.stashedState;
|
|
2058
2261
|
}
|
|
2059
|
-
this.submit(
|
|
2262
|
+
this.submit(message, localOpMetadata);
|
|
2060
2263
|
break;
|
|
2061
2264
|
case ContainerMessageType.ChunkedOp:
|
|
2062
2265
|
throw new Error(`chunkedOp not expected here`);
|
|
@@ -2064,10 +2267,29 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2064
2267
|
this.blobManager.reSubmit(opMetadata);
|
|
2065
2268
|
break;
|
|
2066
2269
|
case ContainerMessageType.Rejoin:
|
|
2067
|
-
this.submit(
|
|
2270
|
+
this.submit(message);
|
|
2068
2271
|
break;
|
|
2069
|
-
default:
|
|
2070
|
-
|
|
2272
|
+
default: {
|
|
2273
|
+
// This case should be very rare - it would imply an op was stashed from a
|
|
2274
|
+
// future version of runtime code and now is being applied on an older version
|
|
2275
|
+
const compatBehavior = message.compatDetails?.behavior;
|
|
2276
|
+
if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
|
|
2277
|
+
this.logger.sendTelemetryEvent({
|
|
2278
|
+
eventName: "resubmitUnrecognizedMessageTypeAllowed",
|
|
2279
|
+
messageDetails: { type: message.type, compatBehavior },
|
|
2280
|
+
});
|
|
2281
|
+
}
|
|
2282
|
+
else {
|
|
2283
|
+
const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
|
|
2284
|
+
messageDetails: JSON.stringify({
|
|
2285
|
+
type: message.type,
|
|
2286
|
+
compatBehavior,
|
|
2287
|
+
}),
|
|
2288
|
+
});
|
|
2289
|
+
this.closeFn(error);
|
|
2290
|
+
throw error;
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2071
2293
|
}
|
|
2072
2294
|
}
|
|
2073
2295
|
rollback(content, localOpMetadata) {
|
|
@@ -2080,6 +2302,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2080
2302
|
this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
|
|
2081
2303
|
break;
|
|
2082
2304
|
default:
|
|
2305
|
+
// Don't check message.compatDetails because this is for rolling back a local op so the type will be known
|
|
2083
2306
|
throw new Error(`Can't rollback ${type}`);
|
|
2084
2307
|
}
|
|
2085
2308
|
}
|
|
@@ -2097,29 +2320,22 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2097
2320
|
/** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
|
|
2098
2321
|
async refreshLatestSummaryAck(options) {
|
|
2099
2322
|
const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
|
|
2323
|
+
// proposalHandle is always passed from RunningSummarizer.
|
|
2324
|
+
assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
|
|
2100
2325
|
const readAndParseBlob = async (id) => readAndParse(this.storage, id);
|
|
2101
|
-
|
|
2102
|
-
|
|
2103
|
-
|
|
2104
|
-
|
|
2105
|
-
|
|
2326
|
+
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
|
|
2327
|
+
/**
|
|
2328
|
+
* When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
|
|
2329
|
+
* current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
|
|
2330
|
+
* it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
|
|
2331
|
+
* and then close as the current main client is likely to be re-elected as the parent summarizer again.
|
|
2332
|
+
*/
|
|
2333
|
+
if (!result.isSummaryTracked && result.isSummaryNewer) {
|
|
2334
|
+
const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2106
2335
|
eventName: "RefreshLatestSummaryAckFetch",
|
|
2107
2336
|
ackHandle,
|
|
2108
2337
|
targetSequenceNumber: summaryRefSeq,
|
|
2109
|
-
}, readAndParseBlob);
|
|
2110
|
-
/**
|
|
2111
|
-
* back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
|
|
2112
|
-
* getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
|
|
2113
|
-
* these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
|
|
2114
|
-
* change that started fetching latest snapshot always.
|
|
2115
|
-
*/
|
|
2116
|
-
if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
|
|
2117
|
-
fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2118
|
-
eventName: "RefreshLatestSummaryAckFetchBackCompat",
|
|
2119
|
-
ackHandle,
|
|
2120
|
-
targetSequenceNumber: summaryRefSeq,
|
|
2121
|
-
}, readAndParseBlob, ackHandle);
|
|
2122
|
-
}
|
|
2338
|
+
}, readAndParseBlob, null);
|
|
2123
2339
|
/**
|
|
2124
2340
|
* If the fetched snapshot is older than the one for which the ack was received, close the container.
|
|
2125
2341
|
* This should never happen because an ack should be sent after the latest summary is updated in the server.
|
|
@@ -2136,20 +2352,14 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2136
2352
|
summaryRefSeq,
|
|
2137
2353
|
fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2138
2354
|
});
|
|
2139
|
-
this.
|
|
2355
|
+
this.disposeFn(error);
|
|
2140
2356
|
throw error;
|
|
2141
2357
|
}
|
|
2142
|
-
|
|
2143
|
-
|
|
2144
|
-
|
|
2145
|
-
return {
|
|
2146
|
-
snapshotTree: fetchResult.snapshotTree,
|
|
2147
|
-
snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2148
|
-
};
|
|
2149
|
-
};
|
|
2150
|
-
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2358
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
|
|
2359
|
+
return;
|
|
2360
|
+
}
|
|
2151
2361
|
// Notify the garbage collector so it can update its latest summary state.
|
|
2152
|
-
await this.garbageCollector.refreshLatestSummary(
|
|
2362
|
+
await this.garbageCollector.refreshLatestSummary(result);
|
|
2153
2363
|
}
|
|
2154
2364
|
/**
|
|
2155
2365
|
* Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
|
|
@@ -2159,24 +2369,31 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2159
2369
|
*/
|
|
2160
2370
|
async refreshLatestSummaryAckFromServer(summaryLogger) {
|
|
2161
2371
|
const readAndParseBlob = async (id) => readAndParse(this.storage, id);
|
|
2162
|
-
const {
|
|
2372
|
+
const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2163
2373
|
eventName: "RefreshLatestSummaryFromServerFetch",
|
|
2164
|
-
}, readAndParseBlob);
|
|
2165
|
-
|
|
2166
|
-
snapshotTree,
|
|
2167
|
-
snapshotRefSeq: latestSnapshotRefSeq,
|
|
2168
|
-
};
|
|
2169
|
-
const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2170
|
-
// Notify the garbage collector so it can update its latest summary state.
|
|
2171
|
-
await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
|
|
2374
|
+
}, readAndParseBlob, null);
|
|
2375
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
|
|
2172
2376
|
return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
|
|
2173
2377
|
}
|
|
2174
|
-
async
|
|
2175
|
-
|
|
2378
|
+
async closeStaleSummarizer(codePath) {
|
|
2379
|
+
this.mc.logger.sendTelemetryEvent({
|
|
2380
|
+
eventName: "ClosingSummarizerOnSummaryStale",
|
|
2381
|
+
codePath,
|
|
2382
|
+
message: "Stopping fetch from storage",
|
|
2383
|
+
closeSummarizerDelayMs: this.closeSummarizerDelayMs,
|
|
2384
|
+
}, new GenericError("Restarting summarizer instead of refreshing"));
|
|
2385
|
+
// Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2386
|
+
await delay(this.closeSummarizerDelayMs);
|
|
2387
|
+
this._summarizer?.stop("latestSummaryStateStale");
|
|
2388
|
+
this.disposeFn();
|
|
2176
2389
|
}
|
|
2390
|
+
/**
|
|
2391
|
+
* Downloads snapshot from storage with the given versionId or latest if versionId is null.
|
|
2392
|
+
* By default, it also closes the container after downloading the snapshot. However, this may be
|
|
2393
|
+
* overridden via options.
|
|
2394
|
+
*/
|
|
2177
2395
|
async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
|
|
2178
|
-
|
|
2179
|
-
const snapshotResults = await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2396
|
+
return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2180
2397
|
const stats = {};
|
|
2181
2398
|
const trace = Trace.start();
|
|
2182
2399
|
const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? FetchSource.noCache : undefined);
|
|
@@ -2195,33 +2412,65 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2195
2412
|
latestSnapshotRefSeq,
|
|
2196
2413
|
};
|
|
2197
2414
|
});
|
|
2198
|
-
// We choose to close the summarizer after the snapshot cache is updated to avoid
|
|
2199
|
-
// situations which the main client (which is likely to be re-elected as the leader again)
|
|
2200
|
-
// loads the summarizer from cache.
|
|
2201
|
-
if (this.summaryStateUpdateMethod === "restart") {
|
|
2202
|
-
const error = new GenericError("Restarting summarizer instead of refreshing");
|
|
2203
|
-
this.mc.logger.sendTelemetryEvent(Object.assign(Object.assign({}, event), { eventName: "ClosingSummarizerOnSummaryStale", codePath: event.eventName, message: "Stopping fetch from storage", versionId: versionId != null ? versionId : undefined, closeSummarizerDelayMs: this.closeSummarizerDelayMs }), error);
|
|
2204
|
-
// Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2205
|
-
await delay(this.closeSummarizerDelayMs);
|
|
2206
|
-
(_a = this._summarizer) === null || _a === void 0 ? void 0 : _a.stop("latestSummaryStateStale");
|
|
2207
|
-
this.closeFn();
|
|
2208
|
-
throw error;
|
|
2209
|
-
}
|
|
2210
|
-
return snapshotResults;
|
|
2211
2415
|
}
|
|
2212
2416
|
notifyAttaching() { } // do nothing (deprecated method)
|
|
2213
|
-
getPendingLocalState() {
|
|
2214
|
-
|
|
2215
|
-
|
|
2417
|
+
async getPendingLocalState(props) {
|
|
2418
|
+
return PerformanceEvent.timedExecAsync(this.mc.logger, {
|
|
2419
|
+
eventName: "getPendingLocalState",
|
|
2420
|
+
notifyImminentClosure: props?.notifyImminentClosure,
|
|
2421
|
+
}, async (event) => {
|
|
2422
|
+
this.verifyNotClosed();
|
|
2423
|
+
const waitBlobsToAttach = props?.notifyImminentClosure;
|
|
2424
|
+
if (this._orderSequentiallyCalls !== 0) {
|
|
2425
|
+
throw new UsageError("can't get state during orderSequentially");
|
|
2426
|
+
}
|
|
2427
|
+
const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
|
|
2428
|
+
const pending = this.pendingStateManager.getLocalState();
|
|
2429
|
+
if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
|
|
2430
|
+
return; // no pending state to save
|
|
2431
|
+
}
|
|
2432
|
+
// Flush pending batch.
|
|
2433
|
+
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2434
|
+
// to close current batch.
|
|
2435
|
+
this.flush();
|
|
2436
|
+
const pendingState = {
|
|
2437
|
+
pending,
|
|
2438
|
+
pendingAttachmentBlobs,
|
|
2439
|
+
};
|
|
2440
|
+
event.end({
|
|
2441
|
+
attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
|
|
2442
|
+
pendingOpsSize: pending?.pendingStates.length,
|
|
2443
|
+
});
|
|
2444
|
+
return pendingState;
|
|
2445
|
+
});
|
|
2446
|
+
}
|
|
2447
|
+
summarizeOnDemand(options) {
|
|
2448
|
+
if (this.isSummarizerClient) {
|
|
2449
|
+
return this.summarizer.summarizeOnDemand(options);
|
|
2450
|
+
}
|
|
2451
|
+
else if (this.summaryManager !== undefined) {
|
|
2452
|
+
return this.summaryManager.summarizeOnDemand(options);
|
|
2453
|
+
}
|
|
2454
|
+
else {
|
|
2455
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2456
|
+
// disableSummaries is turned on. We are throwing instead of returning a failure here,
|
|
2457
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2458
|
+
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
enqueueSummarize(options) {
|
|
2462
|
+
if (this.isSummarizerClient) {
|
|
2463
|
+
return this.summarizer.enqueueSummarize(options);
|
|
2464
|
+
}
|
|
2465
|
+
else if (this.summaryManager !== undefined) {
|
|
2466
|
+
return this.summaryManager.enqueueSummarize(options);
|
|
2467
|
+
}
|
|
2468
|
+
else {
|
|
2469
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2470
|
+
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
2471
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2472
|
+
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2216
2473
|
}
|
|
2217
|
-
// Flush pending batch.
|
|
2218
|
-
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2219
|
-
// to close current batch.
|
|
2220
|
-
this.flush();
|
|
2221
|
-
return {
|
|
2222
|
-
pending: this.pendingStateManager.getLocalState(),
|
|
2223
|
-
pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
|
|
2224
|
-
};
|
|
2225
2474
|
}
|
|
2226
2475
|
/**
|
|
2227
2476
|
* * Forms a function that will request a Summarizer.
|