@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +4 -3
- package/dist/batchTracker.d.ts +3 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +6 -5
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +15 -18
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +212 -171
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +33 -17
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +172 -35
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +722 -425
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +15 -7
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +87 -90
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +10 -10
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.js +2 -2
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +23 -7
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +125 -82
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerProxyBase.d.ts +35 -0
- package/dist/deltaManagerProxyBase.d.ts.map +1 -0
- package/dist/deltaManagerProxyBase.js +77 -0
- package/dist/deltaManagerProxyBase.js.map +1 -0
- package/dist/deltaManagerSummarizerProxy.d.ts +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/dist/deltaManagerSummarizerProxy.js +4 -2
- package/dist/deltaManagerSummarizerProxy.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +10 -10
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/error.d.ts +14 -0
- package/dist/error.d.ts.map +1 -0
- package/dist/error.js +21 -0
- package/dist/error.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +10 -9
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +65 -56
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +18 -14
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +17 -5
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +14 -15
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +0 -8
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +11 -24
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +4 -7
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +19 -58
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +1 -1
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +45 -35
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +4 -4
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +3 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.js +26 -68
- package/dist/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/dist/id-compressor/finalSpace.d.ts +29 -0
- package/dist/id-compressor/finalSpace.d.ts.map +1 -0
- package/dist/id-compressor/finalSpace.js +62 -0
- package/dist/id-compressor/finalSpace.js.map +1 -0
- package/dist/id-compressor/idCompressor.d.ts +25 -250
- package/dist/id-compressor/idCompressor.d.ts.map +1 -1
- package/dist/id-compressor/idCompressor.js +390 -1153
- package/dist/id-compressor/idCompressor.js.map +1 -1
- package/dist/id-compressor/identifiers.d.ts +32 -0
- package/dist/id-compressor/identifiers.d.ts.map +1 -0
- package/dist/id-compressor/identifiers.js +15 -0
- package/dist/id-compressor/identifiers.js.map +1 -0
- package/dist/id-compressor/index.d.ts +5 -6
- package/dist/id-compressor/index.d.ts.map +1 -1
- package/dist/id-compressor/index.js +20 -26
- package/dist/id-compressor/index.js.map +1 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +22 -0
- package/dist/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/dist/id-compressor/persistanceUtilities.js +43 -0
- package/dist/id-compressor/persistanceUtilities.js.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js +80 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/dist/id-compressor/sessions.d.ts +115 -0
- package/dist/id-compressor/sessions.d.ts.map +1 -0
- package/dist/id-compressor/sessions.js +305 -0
- package/dist/id-compressor/sessions.js.map +1 -0
- package/dist/id-compressor/utilities.d.ts +49 -0
- package/dist/id-compressor/utilities.d.ts.map +1 -0
- package/dist/id-compressor/utilities.js +166 -0
- package/dist/id-compressor/utilities.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/metadata.d.ts +18 -0
- package/dist/metadata.d.ts.map +1 -0
- package/dist/metadata.js +7 -0
- package/dist/metadata.js.map +1 -0
- package/dist/opLifecycle/batchManager.d.ts +2 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +15 -7
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +11 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +2 -2
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +12 -7
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -2
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +30 -21
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +19 -13
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +2 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +24 -19
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +39 -6
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +138 -61
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +22 -8
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.js +1 -2
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +25 -10
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +101 -64
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +43 -33
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summary/index.d.ts +4 -4
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +3 -1
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +3 -3
- package/dist/summary/orderedClientElection.d.ts.map +1 -1
- package/dist/summary/orderedClientElection.js +26 -27
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +3 -3
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/runningSummarizer.d.ts +31 -10
- package/dist/summary/runningSummarizer.d.ts.map +1 -1
- package/dist/summary/runningSummarizer.js +271 -139
- package/dist/summary/runningSummarizer.js.map +1 -1
- package/dist/summary/summarizer.d.ts +8 -7
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +79 -78
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerClientElection.d.ts +2 -2
- package/dist/summary/summarizerClientElection.d.ts.map +1 -1
- package/dist/summary/summarizerClientElection.js +7 -11
- package/dist/summary/summarizerClientElection.js.map +1 -1
- package/dist/summary/summarizerHeuristics.js +10 -14
- package/dist/summary/summarizerHeuristics.js.map +1 -1
- package/dist/summary/summarizerNode/index.d.ts +1 -1
- package/dist/summary/summarizerNode/index.d.ts.map +1 -1
- package/dist/summary/summarizerNode/index.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.js +144 -149
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +74 -123
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +44 -24
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +2 -2
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +16 -13
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +4 -0
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +8 -5
- package/dist/summary/summaryFormat.js.map +1 -1
- package/dist/summary/summaryGenerator.d.ts +21 -6
- package/dist/summary/summaryGenerator.d.ts.map +1 -1
- package/dist/summary/summaryGenerator.js +117 -54
- package/dist/summary/summaryGenerator.js.map +1 -1
- package/dist/summary/summaryManager.d.ts +8 -7
- package/dist/summary/summaryManager.d.ts.map +1 -1
- package/dist/summary/summaryManager.js +38 -28
- package/dist/summary/summaryManager.js.map +1 -1
- package/lib/batchTracker.d.ts +3 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +5 -4
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +15 -18
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +187 -146
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +23 -7
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +172 -35
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +678 -380
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +13 -5
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +49 -52
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +3 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +23 -7
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +107 -64
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerProxyBase.d.ts +35 -0
- package/lib/deltaManagerProxyBase.d.ts.map +1 -0
- package/lib/deltaManagerProxyBase.js +73 -0
- package/lib/deltaManagerProxyBase.js.map +1 -0
- package/lib/deltaManagerSummarizerProxy.d.ts +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/lib/deltaManagerSummarizerProxy.js +3 -1
- package/lib/deltaManagerSummarizerProxy.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +7 -7
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/error.d.ts +14 -0
- package/lib/error.d.ts.map +1 -0
- package/lib/error.js +17 -0
- package/lib/error.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +10 -9
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +61 -52
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +16 -12
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +17 -5
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +13 -14
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +0 -8
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +5 -17
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +4 -7
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +20 -59
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +1 -1
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +46 -36
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -2
- package/lib/gc/index.js.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.js +25 -66
- package/lib/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/lib/id-compressor/finalSpace.d.ts +29 -0
- package/lib/id-compressor/finalSpace.d.ts.map +1 -0
- package/lib/id-compressor/finalSpace.js +58 -0
- package/lib/id-compressor/finalSpace.js.map +1 -0
- package/lib/id-compressor/idCompressor.d.ts +25 -250
- package/lib/id-compressor/idCompressor.d.ts.map +1 -1
- package/lib/id-compressor/idCompressor.js +385 -1142
- package/lib/id-compressor/idCompressor.js.map +1 -1
- package/lib/id-compressor/identifiers.d.ts +32 -0
- package/lib/id-compressor/identifiers.d.ts.map +1 -0
- package/lib/id-compressor/identifiers.js +11 -0
- package/lib/id-compressor/identifiers.js.map +1 -0
- package/lib/id-compressor/index.d.ts +5 -6
- package/lib/id-compressor/index.d.ts.map +1 -1
- package/lib/id-compressor/index.js +5 -6
- package/lib/id-compressor/index.js.map +1 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +22 -0
- package/lib/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/lib/id-compressor/persistanceUtilities.js +34 -0
- package/lib/id-compressor/persistanceUtilities.js.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js +76 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/lib/id-compressor/sessions.d.ts +115 -0
- package/lib/id-compressor/sessions.d.ts.map +1 -0
- package/lib/id-compressor/sessions.js +290 -0
- package/lib/id-compressor/sessions.js.map +1 -0
- package/lib/id-compressor/utilities.d.ts +49 -0
- package/lib/id-compressor/utilities.d.ts.map +1 -0
- package/lib/id-compressor/utilities.js +148 -0
- package/lib/id-compressor/utilities.js.map +1 -0
- package/lib/index.d.ts +3 -3
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -2
- package/lib/index.js.map +1 -1
- package/lib/metadata.d.ts +18 -0
- package/lib/metadata.d.ts.map +1 -0
- package/lib/metadata.js +6 -0
- package/lib/metadata.js.map +1 -0
- package/lib/opLifecycle/batchManager.d.ts +2 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +15 -7
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +11 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +2 -2
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +10 -5
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -2
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +22 -13
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +17 -11
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +2 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +15 -10
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +39 -6
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +132 -56
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +23 -9
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.js +1 -2
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +25 -10
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +90 -53
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +25 -15
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summary/index.d.ts +4 -4
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +2 -2
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +3 -3
- package/lib/summary/orderedClientElection.d.ts.map +1 -1
- package/lib/summary/orderedClientElection.js +21 -22
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/runningSummarizer.d.ts +31 -10
- package/lib/summary/runningSummarizer.d.ts.map +1 -1
- package/lib/summary/runningSummarizer.js +265 -133
- package/lib/summary/runningSummarizer.js.map +1 -1
- package/lib/summary/summarizer.d.ts +8 -7
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +75 -74
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerClientElection.d.ts +2 -2
- package/lib/summary/summarizerClientElection.d.ts.map +1 -1
- package/lib/summary/summarizerClientElection.js +6 -10
- package/lib/summary/summarizerClientElection.js.map +1 -1
- package/lib/summary/summarizerHeuristics.js +9 -13
- package/lib/summary/summarizerHeuristics.js.map +1 -1
- package/lib/summary/summarizerNode/index.d.ts +1 -1
- package/lib/summary/summarizerNode/index.d.ts.map +1 -1
- package/lib/summary/summarizerNode/index.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.js +132 -137
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +70 -119
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +44 -24
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +2 -2
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +9 -6
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +4 -0
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +7 -4
- package/lib/summary/summaryFormat.js.map +1 -1
- package/lib/summary/summaryGenerator.d.ts +21 -6
- package/lib/summary/summaryGenerator.d.ts.map +1 -1
- package/lib/summary/summaryGenerator.js +109 -47
- package/lib/summary/summaryGenerator.js.map +1 -1
- package/lib/summary/summaryManager.d.ts +8 -7
- package/lib/summary/summaryManager.d.ts.map +1 -1
- package/lib/summary/summaryManager.js +35 -25
- package/lib/summary/summaryManager.js.map +1 -1
- package/package.json +30 -32
- package/src/batchTracker.ts +7 -5
- package/src/blobManager.ts +235 -172
- package/src/connectionTelemetry.ts +19 -5
- package/src/containerRuntime.ts +853 -431
- package/src/dataStore.ts +12 -4
- package/src/dataStoreContext.ts +49 -46
- package/src/dataStoreContexts.ts +4 -4
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +119 -80
- package/src/deltaManagerProxyBase.ts +111 -0
- package/src/deltaManagerSummarizerProxy.ts +4 -1
- package/src/deltaScheduler.ts +7 -11
- package/src/error.ts +18 -0
- package/src/gc/garbageCollection.md +53 -5
- package/src/gc/garbageCollection.ts +58 -51
- package/src/gc/gcConfigs.ts +4 -2
- package/src/gc/gcDefinitions.ts +17 -21
- package/src/gc/gcEarlyAdoption.md +145 -0
- package/src/gc/gcHelpers.ts +1 -12
- package/src/gc/gcSummaryStateTracker.ts +19 -65
- package/src/gc/gcTelemetry.ts +15 -13
- package/src/gc/gcUnreferencedStateTracker.ts +1 -1
- package/src/gc/index.ts +2 -4
- package/src/id-compressor/appendOnlySortedMap.ts +26 -87
- package/src/id-compressor/finalSpace.ts +67 -0
- package/src/id-compressor/idCompressor.ts +458 -1682
- package/src/id-compressor/identifiers.ts +42 -0
- package/src/id-compressor/index.ts +11 -20
- package/src/id-compressor/persistanceUtilities.ts +58 -0
- package/src/id-compressor/sessionSpaceNormalizer.ts +83 -0
- package/src/id-compressor/sessions.ts +405 -0
- package/src/id-compressor/utilities.ts +187 -0
- package/src/index.ts +9 -2
- package/src/metadata.ts +19 -0
- package/src/opLifecycle/README.md +20 -0
- package/src/opLifecycle/batchManager.ts +9 -1
- package/src/opLifecycle/definitions.ts +11 -0
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +6 -5
- package/src/opLifecycle/opDecompressor.ts +47 -17
- package/src/opLifecycle/opGroupingManager.ts +18 -8
- package/src/opLifecycle/opSplitter.ts +10 -7
- package/src/opLifecycle/outbox.ts +177 -72
- package/src/opLifecycle/remoteMessageProcessor.ts +32 -9
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +123 -78
- package/src/scheduleManager.ts +22 -11
- package/src/summary/index.ts +7 -4
- package/src/summary/orderedClientElection.ts +10 -6
- package/src/summary/runWhileConnectedCoordinator.ts +1 -1
- package/src/summary/runningSummarizer.ts +291 -163
- package/src/summary/summarizer.ts +27 -16
- package/src/summary/summarizerClientElection.ts +2 -2
- package/src/summary/summarizerHeuristics.ts +1 -1
- package/src/summary/summarizerNode/index.ts +2 -2
- package/src/summary/summarizerNode/summarizerNode.ts +142 -184
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +27 -35
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +72 -148
- package/src/summary/summarizerTypes.ts +49 -24
- package/src/summary/summaryCollection.ts +9 -4
- package/src/summary/summaryFormat.ts +9 -2
- package/src/summary/summaryGenerator.ts +72 -49
- package/src/summary/summaryManager.ts +44 -16
- package/dist/id-compressor/idRange.d.ts +0 -11
- package/dist/id-compressor/idRange.d.ts.map +0 -1
- package/dist/id-compressor/idRange.js +0 -29
- package/dist/id-compressor/idRange.js.map +0 -1
- package/dist/id-compressor/numericUuid.d.ts +0 -59
- package/dist/id-compressor/numericUuid.d.ts.map +0 -1
- package/dist/id-compressor/numericUuid.js +0 -325
- package/dist/id-compressor/numericUuid.js.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/dist/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.js +0 -488
- package/dist/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/dist/id-compressor/utils.d.ts +0 -57
- package/dist/id-compressor/utils.d.ts.map +0 -1
- package/dist/id-compressor/utils.js +0 -90
- package/dist/id-compressor/utils.js.map +0 -1
- package/dist/id-compressor/uuidUtilities.d.ts +0 -30
- package/dist/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/dist/id-compressor/uuidUtilities.js +0 -106
- package/dist/id-compressor/uuidUtilities.js.map +0 -1
- package/lib/id-compressor/idRange.d.ts +0 -11
- package/lib/id-compressor/idRange.d.ts.map +0 -1
- package/lib/id-compressor/idRange.js +0 -25
- package/lib/id-compressor/idRange.js.map +0 -1
- package/lib/id-compressor/numericUuid.d.ts +0 -59
- package/lib/id-compressor/numericUuid.d.ts.map +0 -1
- package/lib/id-compressor/numericUuid.js +0 -315
- package/lib/id-compressor/numericUuid.js.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/lib/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.js +0 -484
- package/lib/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/lib/id-compressor/utils.d.ts +0 -57
- package/lib/id-compressor/utils.d.ts.map +0 -1
- package/lib/id-compressor/utils.js +0 -79
- package/lib/id-compressor/utils.js.map +0 -1
- package/lib/id-compressor/uuidUtilities.d.ts +0 -30
- package/lib/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/lib/id-compressor/uuidUtilities.js +0 -98
- package/lib/id-compressor/uuidUtilities.js.map +0 -1
- package/src/id-compressor/idRange.ts +0 -35
- package/src/id-compressor/numericUuid.ts +0 -383
- package/src/id-compressor/sessionIdNormalizer.ts +0 -609
- package/src/id-compressor/utils.ts +0 -114
- package/src/id-compressor/uuidUtilities.ts +0 -123
package/lib/containerRuntime.js
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
import { AttachState, LoaderHeader, } from "@fluidframework/container-definitions";
|
|
2
|
-
import { assert, delay, LazyPromise
|
|
3
|
-
import {
|
|
2
|
+
import { assert, delay, LazyPromise } from "@fluidframework/core-utils";
|
|
3
|
+
import { Trace, TypedEventEmitter } from "@fluid-internal/client-utils";
|
|
4
|
+
import { createChildLogger, createChildMonitoringContext, DataCorruptionError, DataProcessingError, GenericError, raiseConnectedEvent, PerformanceEvent,
|
|
5
|
+
// eslint-disable-next-line import/no-deprecated
|
|
6
|
+
TaggedLoggerAdapter, wrapError, UsageError, } from "@fluidframework/telemetry-utils";
|
|
4
7
|
import { DriverHeader, FetchSource, } from "@fluidframework/driver-definitions";
|
|
5
8
|
import { readAndParse } from "@fluidframework/driver-utils";
|
|
6
|
-
import { DataCorruptionError, DataProcessingError, GenericError, UsageError, } from "@fluidframework/container-utils";
|
|
7
9
|
import { MessageType, SummaryType, } from "@fluidframework/protocol-definitions";
|
|
8
10
|
import { FlushMode, FlushModeExperimental, gcTreeKey, channelsTreeName, } from "@fluidframework/runtime-definitions";
|
|
9
11
|
import { addBlobToSummary, addSummarizeResultToSummary, addTreeToSummary, RequestParser, create404Response, exceptionToResponse, GCDataBuilder, requestFluidObject, seqFromTree, calculateStats, TelemetryContext, } from "@fluidframework/runtime-utils";
|
|
@@ -11,17 +13,17 @@ import { v4 as uuid } from "uuid";
|
|
|
11
13
|
import { ContainerFluidHandleContext } from "./containerHandleContext";
|
|
12
14
|
import { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
13
15
|
import { ReportOpPerfTelemetry } from "./connectionTelemetry";
|
|
14
|
-
import { PendingStateManager } from "./pendingStateManager";
|
|
16
|
+
import { PendingStateManager, } from "./pendingStateManager";
|
|
15
17
|
import { pkgVersion } from "./packageVersion";
|
|
16
18
|
import { BlobManager } from "./blobManager";
|
|
17
19
|
import { DataStores, getSummaryForDatastores } from "./dataStores";
|
|
18
|
-
import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, } from "./summary";
|
|
20
|
+
import { aliasBlobName, blobsTreeName, chunksBlobName, createRootSummarizerNodeWithGC, electedSummarizerBlobName, extractSummaryMetadataMessage, idCompressorBlobName, metadataBlobName, Summarizer, SummaryManager, wrapSummaryInChannelsTree, SummaryCollection, OrderedClientCollection, OrderedClientElection, SummarizerClientElection, summarizerClientType, RunWhileConnectedCoordinator, RetriableSummaryError, } from "./summary";
|
|
19
21
|
import { formExponentialFn, Throttler } from "./throttler";
|
|
20
22
|
import { GarbageCollector, GCNodeType, gcTombstoneGenerationOptionName, shouldAllowGcTombstoneEnforcement, trimLeadingAndTrailingSlashes, } from "./gc";
|
|
21
23
|
import { channelToDataStore, isDataStoreAliasMessage } from "./dataStore";
|
|
22
24
|
import { BindBatchTracker } from "./batchTracker";
|
|
23
25
|
import { ScheduleManager } from "./scheduleManager";
|
|
24
|
-
import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, OpGroupingManager, } from "./opLifecycle";
|
|
26
|
+
import { OpCompressor, OpDecompressor, Outbox, OpSplitter, RemoteMessageProcessor, OpGroupingManager, getLongStack, } from "./opLifecycle";
|
|
25
27
|
import { DeltaManagerSummarizerProxy } from "./deltaManagerSummarizerProxy";
|
|
26
28
|
export var ContainerMessageType;
|
|
27
29
|
(function (ContainerMessageType) {
|
|
@@ -44,6 +46,18 @@ export var ContainerMessageType;
|
|
|
44
46
|
*/
|
|
45
47
|
ContainerMessageType["IdAllocation"] = "idAllocation";
|
|
46
48
|
})(ContainerMessageType || (ContainerMessageType = {}));
|
|
49
|
+
/**
|
|
50
|
+
* Utility to implement compat behaviors given an unknown message type
|
|
51
|
+
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|
|
52
|
+
*
|
|
53
|
+
* @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
|
|
54
|
+
* handled before calling this function (e.g. in a switch statement).
|
|
55
|
+
* @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
|
|
56
|
+
*/
|
|
57
|
+
function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
|
|
58
|
+
// undefined defaults to same behavior as "FailToProcess"
|
|
59
|
+
return compatBehavior === "Ignore";
|
|
60
|
+
}
|
|
47
61
|
export const DefaultSummaryConfiguration = {
|
|
48
62
|
state: "enabled",
|
|
49
63
|
minIdleTime: 0,
|
|
@@ -70,8 +84,12 @@ export var RuntimeHeaders;
|
|
|
70
84
|
})(RuntimeHeaders || (RuntimeHeaders = {}));
|
|
71
85
|
/** True if a tombstoned object should be returned without erroring */
|
|
72
86
|
export const AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
|
|
87
|
+
/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
|
|
88
|
+
export const AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
|
|
73
89
|
/** Tombstone error responses will have this header set to true */
|
|
74
90
|
export const TombstoneResponseHeaderKey = "isTombstoned";
|
|
91
|
+
/** Inactive error responses will have this header set to true */
|
|
92
|
+
export const InactiveResponseHeaderKey = "isInactive";
|
|
75
93
|
/** Default values for Runtime Headers */
|
|
76
94
|
export const defaultRuntimeHeaderData = {
|
|
77
95
|
wait: true,
|
|
@@ -98,12 +116,16 @@ const defaultCompressionConfig = {
|
|
|
98
116
|
compressionAlgorithm: CompressionAlgorithms.lz4,
|
|
99
117
|
};
|
|
100
118
|
const defaultChunkSizeInBytes = 204800;
|
|
119
|
+
/** The default time to wait for pending ops to be processed during summarization */
|
|
120
|
+
export const defaultPendingOpsWaitTimeoutMs = 1000;
|
|
121
|
+
/** The default time to delay a summarization retry attempt when there are pending ops */
|
|
122
|
+
export const defaultPendingOpsRetryDelayMs = 1000;
|
|
101
123
|
/**
|
|
102
124
|
* Instead of refreshing from latest because we do not have 100% confidence in the state
|
|
103
125
|
* of the current system, we should close the summarizer and let it recover.
|
|
104
126
|
* This delay's goal is to prevent tight restart loops
|
|
105
127
|
*/
|
|
106
|
-
const defaultCloseSummarizerDelayMs =
|
|
128
|
+
const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
|
|
107
129
|
/**
|
|
108
130
|
* @deprecated - use ContainerRuntimeMessage instead
|
|
109
131
|
*/
|
|
@@ -139,9 +161,23 @@ export function getDeviceSpec() {
|
|
|
139
161
|
};
|
|
140
162
|
}
|
|
141
163
|
}
|
|
142
|
-
catch
|
|
164
|
+
catch { }
|
|
143
165
|
return {};
|
|
144
166
|
}
|
|
167
|
+
/**
|
|
168
|
+
* Older loader doesn't have a submitBatchFn member, this is the older way of submitting a batch.
|
|
169
|
+
* Rather than exposing the submitFn (now deprecated) and IDeltaManager (dangerous to hand out) to the Outbox,
|
|
170
|
+
* we can provide a partially-applied function to keep those items private to the ContainerRuntime.
|
|
171
|
+
*/
|
|
172
|
+
export const makeLegacySendBatchFn = (submitFn, deltaManager) => (batch) => {
|
|
173
|
+
for (const message of batch.content) {
|
|
174
|
+
submitFn(MessageType.Operation,
|
|
175
|
+
// For back-compat (submitFn only works on deserialized content)
|
|
176
|
+
message.contents === undefined ? undefined : JSON.parse(message.contents), true, // batch
|
|
177
|
+
message.metadata);
|
|
178
|
+
}
|
|
179
|
+
deltaManager.flush();
|
|
180
|
+
};
|
|
145
181
|
/**
|
|
146
182
|
* Represents the runtime of the container. Contains helper functions/state of the container.
|
|
147
183
|
* It will define the store level mappings.
|
|
@@ -150,11 +186,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
150
186
|
/**
|
|
151
187
|
* @internal
|
|
152
188
|
*/
|
|
153
|
-
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration
|
|
154
|
-
|
|
155
|
-
|
|
189
|
+
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration = {
|
|
190
|
+
// the defaults
|
|
191
|
+
...DefaultSummaryConfiguration,
|
|
192
|
+
// the runtime configuration overrides
|
|
193
|
+
...runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
194
|
+
}, initializeEntryPoint) {
|
|
156
195
|
super();
|
|
157
|
-
this.context = context;
|
|
158
196
|
this.registry = registry;
|
|
159
197
|
this.runtimeOptions = runtimeOptions;
|
|
160
198
|
this.containerScope = containerScope;
|
|
@@ -183,51 +221,56 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
183
221
|
signalTimestamp: 0,
|
|
184
222
|
trackingSignalSequenceNumber: undefined,
|
|
185
223
|
};
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
200
|
-
this.
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
else if (this.summaryManager !== undefined) {
|
|
205
|
-
return this.summaryManager.enqueueSummarize(...args);
|
|
224
|
+
const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
|
|
225
|
+
this.innerDeltaManager = deltaManager;
|
|
226
|
+
this.deltaManager = new DeltaManagerSummarizerProxy(this.innerDeltaManager);
|
|
227
|
+
// Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
|
|
228
|
+
// This makes ContainerRuntime the final gatekeeper for outgoing messages.
|
|
229
|
+
this.submitFn = submitFn;
|
|
230
|
+
this.submitBatchFn = submitBatchFn;
|
|
231
|
+
this.submitSummaryFn = submitSummaryFn;
|
|
232
|
+
this.submitSignalFn = submitSignalFn;
|
|
233
|
+
this.options = options;
|
|
234
|
+
this.clientDetails = clientDetails;
|
|
235
|
+
this.isSummarizerClient = this.clientDetails.type === summarizerClientType;
|
|
236
|
+
this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
|
|
237
|
+
this._getClientId = () => context.clientId;
|
|
238
|
+
this._getAttachState = () => context.attachState;
|
|
239
|
+
this.getAbsoluteUrl = async (relativeUrl) => {
|
|
240
|
+
if (context.getAbsoluteUrl === undefined) {
|
|
241
|
+
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
206
242
|
}
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
210
|
-
// because it is a misuse of the API rather than an expected failure.
|
|
211
|
-
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
243
|
+
if (this.attachState !== AttachState.Attached) {
|
|
244
|
+
return undefined;
|
|
212
245
|
}
|
|
246
|
+
return context.getAbsoluteUrl(relativeUrl);
|
|
213
247
|
};
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
this.
|
|
248
|
+
// TODO: Consider that the Container could just listen to these events itself, or even more appropriately maybe the
|
|
249
|
+
// customer should observe dirty state on the runtime (the owner of dirty state) directly, rather than on the IContainer.
|
|
250
|
+
this.on("dirty", () => context.updateDirtyContainerState(true));
|
|
251
|
+
this.on("saved", () => context.updateDirtyContainerState(false));
|
|
252
|
+
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
253
|
+
this.disposeFn = disposeFn ?? closeFn;
|
|
254
|
+
// In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
|
|
255
|
+
this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
|
|
256
|
+
this.mc = createChildMonitoringContext({
|
|
257
|
+
logger: this.logger,
|
|
258
|
+
namespace: "ContainerRuntime",
|
|
259
|
+
});
|
|
217
260
|
let loadSummaryNumber;
|
|
218
261
|
// Get the container creation metadata. For new container, we initialize these. For existing containers,
|
|
219
262
|
// get the values from the metadata blob.
|
|
220
263
|
if (existing) {
|
|
221
264
|
this.createContainerMetadata = {
|
|
222
|
-
createContainerRuntimeVersion: metadata
|
|
223
|
-
createContainerTimestamp: metadata
|
|
265
|
+
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
266
|
+
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
224
267
|
};
|
|
225
268
|
// summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
|
|
226
269
|
// the count is reset to 0.
|
|
227
|
-
loadSummaryNumber =
|
|
270
|
+
loadSummaryNumber = metadata?.summaryNumber ?? 0;
|
|
228
271
|
// Enabling the IdCompressor is a one-way operation and we only want to
|
|
229
272
|
// allow new containers to turn it on
|
|
230
|
-
this.idCompressorEnabled =
|
|
273
|
+
this.idCompressorEnabled = metadata?.idCompressorEnabled ?? false;
|
|
231
274
|
}
|
|
232
275
|
else {
|
|
233
276
|
this.createContainerMetadata = {
|
|
@@ -236,24 +279,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
236
279
|
};
|
|
237
280
|
loadSummaryNumber = 0;
|
|
238
281
|
this.idCompressorEnabled =
|
|
239
|
-
|
|
282
|
+
this.mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled") ??
|
|
283
|
+
idCompressor !== undefined;
|
|
240
284
|
}
|
|
241
285
|
this.nextSummaryNumber = loadSummaryNumber + 1;
|
|
242
|
-
this.messageAtLastSummary = metadata
|
|
243
|
-
|
|
244
|
-
|
|
286
|
+
this.messageAtLastSummary = metadata?.message;
|
|
287
|
+
// Note that we only need to pull the *initial* connected state from the context.
|
|
288
|
+
// Later updates come through calls to setConnectionState.
|
|
289
|
+
this._connected = connected;
|
|
290
|
+
this.gcTombstoneEnforcementAllowed = shouldAllowGcTombstoneEnforcement(metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName] /* current */);
|
|
245
291
|
this.mc.logger.sendTelemetryEvent({
|
|
246
292
|
eventName: "GCFeatureMatrix",
|
|
247
|
-
metadataValue: JSON.stringify(metadata
|
|
293
|
+
metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
|
|
248
294
|
inputs: JSON.stringify({
|
|
249
295
|
gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gcTombstoneGenerationOptionName],
|
|
250
296
|
}),
|
|
251
297
|
});
|
|
252
|
-
this.telemetryDocumentId =
|
|
298
|
+
this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
|
|
253
299
|
this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
|
|
254
300
|
const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
|
|
255
301
|
const opGroupingManager = new OpGroupingManager(this.groupedBatchingEnabled);
|
|
256
|
-
const opSplitter = new OpSplitter(chunks, this.
|
|
302
|
+
const opSplitter = new OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
|
|
257
303
|
this.remoteMessageProcessor = new RemoteMessageProcessor(opSplitter, new OpDecompressor(this.mc.logger), opGroupingManager);
|
|
258
304
|
this.handleContext = new ContainerFluidHandleContext("", this);
|
|
259
305
|
if (this.summaryConfiguration.state === "enabled") {
|
|
@@ -272,9 +318,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
272
318
|
this.idCompressor = idCompressor;
|
|
273
319
|
}
|
|
274
320
|
this.maxConsecutiveReconnects =
|
|
275
|
-
|
|
321
|
+
this.mc.config.getNumber(maxConsecutiveReconnectsKey) ??
|
|
322
|
+
this.defaultMaxConsecutiveReconnects;
|
|
276
323
|
if (runtimeOptions.flushMode === FlushModeExperimental.Async &&
|
|
277
|
-
|
|
324
|
+
supportedFeatures?.get("referenceSequenceNumbers") !== true) {
|
|
278
325
|
// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
|
|
279
326
|
this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
|
|
280
327
|
this._flushMode = FlushMode.TurnBased;
|
|
@@ -282,8 +329,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
282
329
|
else {
|
|
283
330
|
this._flushMode = runtimeOptions.flushMode;
|
|
284
331
|
}
|
|
285
|
-
const pendingRuntimeState =
|
|
286
|
-
const maxSnapshotCacheDurationMs =
|
|
332
|
+
const pendingRuntimeState = pendingLocalState;
|
|
333
|
+
const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
|
|
287
334
|
if (maxSnapshotCacheDurationMs !== undefined &&
|
|
288
335
|
maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
|
|
289
336
|
// This is a runtime enforcement of what's already explicit in the policy's type itself,
|
|
@@ -294,28 +341,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
294
341
|
this.garbageCollector = GarbageCollector.create({
|
|
295
342
|
runtime: this,
|
|
296
343
|
gcOptions: this.runtimeOptions.gcOptions,
|
|
297
|
-
baseSnapshot
|
|
344
|
+
baseSnapshot,
|
|
298
345
|
baseLogger: this.mc.logger,
|
|
299
346
|
existing,
|
|
300
347
|
metadata,
|
|
301
348
|
createContainerMetadata: this.createContainerMetadata,
|
|
302
|
-
isSummarizerClient: this.
|
|
349
|
+
isSummarizerClient: this.isSummarizerClient,
|
|
303
350
|
getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
|
|
304
|
-
getLastSummaryTimestampMs: () =>
|
|
351
|
+
getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
|
|
305
352
|
readAndParseBlob: async (id) => readAndParse(this.storage, id),
|
|
306
|
-
getContainerDiagnosticId: () => this.context.id,
|
|
307
353
|
// GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
|
|
308
354
|
// delta manager would always return false for summarizer client.
|
|
309
355
|
activeConnection: () => this.innerDeltaManager.active,
|
|
310
356
|
});
|
|
311
357
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
312
|
-
this.summarizerNode = createRootSummarizerNodeWithGC(
|
|
358
|
+
this.summarizerNode = createRootSummarizerNodeWithGC(createChildLogger({ logger: this.logger, namespace: "SummarizerNode" }),
|
|
313
359
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
314
360
|
async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext),
|
|
315
361
|
// Latest change sequence number, no changes since summary applied yet
|
|
316
362
|
loadedFromSequenceNumber,
|
|
317
363
|
// Summary reference sequence number, undefined if no summary yet
|
|
318
|
-
|
|
364
|
+
baseSnapshot !== undefined ? loadedFromSequenceNumber : undefined, {
|
|
319
365
|
// Must set to false to prevent sending summary handle which would be pointing to
|
|
320
366
|
// a summary with an older protocol state.
|
|
321
367
|
canReuseHandle: false,
|
|
@@ -329,27 +375,28 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
329
375
|
async (fullGC) => this.getGCDataInternal(fullGC),
|
|
330
376
|
// Function to get the GC details from the base snapshot we loaded from.
|
|
331
377
|
async () => this.garbageCollector.getBaseGCDetails());
|
|
332
|
-
if (
|
|
333
|
-
this.summarizerNode.updateBaseSummaryState(
|
|
378
|
+
if (baseSnapshot) {
|
|
379
|
+
this.summarizerNode.updateBaseSummaryState(baseSnapshot);
|
|
334
380
|
}
|
|
335
|
-
this.dataStores = new DataStores(getSummaryForDatastores(
|
|
381
|
+
this.dataStores = new DataStores(getSummaryForDatastores(baseSnapshot, metadata), this, (attachMsg) => this.submit({ type: ContainerMessageType.Attach, contents: attachMsg }), (id, createParam) => (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
|
|
336
382
|
this.blobManager = new BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
|
|
337
383
|
if (!this.disposed) {
|
|
338
|
-
this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
|
|
384
|
+
this.submit({ type: ContainerMessageType.BlobAttach, contents: undefined }, undefined, {
|
|
339
385
|
localId,
|
|
340
386
|
blobId,
|
|
341
387
|
});
|
|
342
388
|
}
|
|
343
|
-
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState
|
|
344
|
-
this.scheduleManager = new ScheduleManager(
|
|
389
|
+
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState?.pendingAttachmentBlobs, (error) => this.closeFn(error));
|
|
390
|
+
this.scheduleManager = new ScheduleManager(this.innerDeltaManager, this, () => this.clientId, createChildLogger({ logger: this.logger, namespace: "ScheduleManager" }));
|
|
345
391
|
this.pendingStateManager = new PendingStateManager({
|
|
346
392
|
applyStashedOp: this.applyStashedOp.bind(this),
|
|
347
393
|
clientId: () => this.clientId,
|
|
348
394
|
close: this.closeFn,
|
|
349
395
|
connected: () => this.connected,
|
|
350
396
|
reSubmit: this.reSubmit.bind(this),
|
|
351
|
-
|
|
352
|
-
|
|
397
|
+
reSubmitBatch: this.reSubmitBatch.bind(this),
|
|
398
|
+
isActiveConnection: () => this.innerDeltaManager.active,
|
|
399
|
+
}, pendingRuntimeState?.pending, this.logger);
|
|
353
400
|
const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
|
|
354
401
|
const compressionOptions = disableCompression === true
|
|
355
402
|
? {
|
|
@@ -358,16 +405,19 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
358
405
|
}
|
|
359
406
|
: runtimeOptions.compressionOptions;
|
|
360
407
|
const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
|
|
408
|
+
const legacySendBatchFn = makeLegacySendBatchFn(this.submitFn, this.innerDeltaManager);
|
|
361
409
|
this.outbox = new Outbox({
|
|
362
410
|
shouldSend: () => this.canSendOps(),
|
|
363
411
|
pendingStateManager: this.pendingStateManager,
|
|
364
|
-
|
|
412
|
+
submitBatchFn: this.submitBatchFn,
|
|
413
|
+
legacySendBatchFn,
|
|
365
414
|
compressor: new OpCompressor(this.mc.logger),
|
|
366
415
|
splitter: opSplitter,
|
|
367
416
|
config: {
|
|
368
417
|
compressionOptions,
|
|
369
418
|
maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
|
|
370
419
|
disablePartialFlush: disablePartialFlush === true,
|
|
420
|
+
enableGroupedBatching: this.groupedBatchingEnabled,
|
|
371
421
|
},
|
|
372
422
|
logger: this.mc.logger,
|
|
373
423
|
groupingManager: opGroupingManager,
|
|
@@ -375,43 +425,54 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
375
425
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
376
426
|
clientSequenceNumber: this._processedClientSequenceNumber,
|
|
377
427
|
}),
|
|
428
|
+
reSubmit: this.reSubmit.bind(this),
|
|
429
|
+
opReentrancy: () => this.ensureNoDataModelChangesCalls > 0,
|
|
430
|
+
closeContainer: this.closeFn,
|
|
378
431
|
});
|
|
379
|
-
this.
|
|
432
|
+
this._quorum = quorum;
|
|
433
|
+
this._quorum.on("removeMember", (clientId) => {
|
|
380
434
|
this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
|
|
381
435
|
});
|
|
382
|
-
|
|
436
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
437
|
+
this._audience = audience;
|
|
383
438
|
const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
|
|
384
|
-
this.closeSummarizerDelayMs = closeSummarizerDelayOverride
|
|
439
|
+
this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
|
|
440
|
+
this.validateSummaryBeforeUpload =
|
|
441
|
+
this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
|
|
385
442
|
this.summaryCollection = new SummaryCollection(this.deltaManager, this.logger);
|
|
386
443
|
this.dirtyContainer =
|
|
387
|
-
this.
|
|
388
|
-
|
|
389
|
-
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
444
|
+
this.attachState !== AttachState.Attached || this.hasPendingMessages();
|
|
445
|
+
context.updateDirtyContainerState(this.dirtyContainer);
|
|
390
446
|
if (this.summariesDisabled) {
|
|
391
447
|
this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
|
|
392
448
|
}
|
|
393
449
|
else {
|
|
394
|
-
const orderedClientLogger =
|
|
395
|
-
|
|
396
|
-
|
|
450
|
+
const orderedClientLogger = createChildLogger({
|
|
451
|
+
logger: this.logger,
|
|
452
|
+
namespace: "OrderedClientElection",
|
|
453
|
+
});
|
|
454
|
+
const orderedClientCollection = new OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
|
|
455
|
+
const orderedClientElectionForSummarizer = new OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, SummarizerClientElection.isClientEligible);
|
|
397
456
|
this.summarizerClientElection = new SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
|
|
398
|
-
if (this.
|
|
457
|
+
if (this.isSummarizerClient) {
|
|
399
458
|
this._summarizer = new Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => RunWhileConnectedCoordinator.create(runtime,
|
|
400
459
|
// Summarization runs in summarizer client and needs access to the real (non-proxy) active
|
|
401
460
|
// information. The proxy delta manager would always return false for summarizer client.
|
|
402
461
|
() => this.innerDeltaManager.active));
|
|
403
462
|
}
|
|
404
|
-
else if (SummarizerClientElection.clientDetailsPermitElection(this.
|
|
463
|
+
else if (SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
|
|
405
464
|
// Only create a SummaryManager and SummarizerClientElection
|
|
406
465
|
// if summaries are enabled and we are not the summarizer client.
|
|
407
466
|
const defaultAction = () => {
|
|
408
467
|
if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
|
|
409
|
-
this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
468
|
+
this.mc.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
410
469
|
// unregister default to no log on every op after falling behind
|
|
411
470
|
// and register summary ack handler to re-register this handler
|
|
412
471
|
// after successful summary
|
|
413
472
|
this.summaryCollection.once(MessageType.SummaryAck, () => {
|
|
414
|
-
this.logger.sendTelemetryEvent({
|
|
473
|
+
this.mc.logger.sendTelemetryEvent({
|
|
474
|
+
eventName: "SummaryStatus:CaughtUp",
|
|
475
|
+
});
|
|
415
476
|
// we've caught up, so re-register the default action to monitor for
|
|
416
477
|
// falling behind, and unregister ourself
|
|
417
478
|
this.summaryCollection.on("default", defaultAction);
|
|
@@ -422,12 +483,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
422
483
|
this.summaryCollection.on("default", defaultAction);
|
|
423
484
|
// Create the SummaryManager and mark the initial state
|
|
424
485
|
this.summaryManager = new SummaryManager(this.summarizerClientElection, this, // IConnectedState
|
|
425
|
-
this.summaryCollection, this.logger, this.formRequestSummarizerFn(
|
|
486
|
+
this.summaryCollection, this.logger, this.formRequestSummarizerFn(loader), new Throttler(60 * 1000, // 60 sec delay window
|
|
426
487
|
30 * 1000, // 30 sec max delay
|
|
427
488
|
// throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
|
|
428
489
|
formExponentialFn({ coefficient: 20, initialDelay: 0 })), {
|
|
429
490
|
initialDelayMs: this.initialSummarizerDelayMs,
|
|
430
491
|
}, this.heuristicsDisabled);
|
|
492
|
+
this.summaryManager.on("summarize", (eventProps) => {
|
|
493
|
+
this.emit("summarize", eventProps);
|
|
494
|
+
});
|
|
431
495
|
this.summaryManager.start();
|
|
432
496
|
}
|
|
433
497
|
}
|
|
@@ -452,27 +516,44 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
452
516
|
this.replayPendingStates();
|
|
453
517
|
});
|
|
454
518
|
// logging hardware telemetry
|
|
455
|
-
logger.sendTelemetryEvent(
|
|
456
|
-
|
|
519
|
+
logger.sendTelemetryEvent({
|
|
520
|
+
eventName: "DeviceSpec",
|
|
521
|
+
...getDeviceSpec(),
|
|
522
|
+
});
|
|
523
|
+
this.mc.logger.sendTelemetryEvent({
|
|
524
|
+
eventName: "ContainerLoadStats",
|
|
525
|
+
...this.createContainerMetadata,
|
|
526
|
+
...this.dataStores.containerLoadStats,
|
|
527
|
+
summaryNumber: loadSummaryNumber,
|
|
528
|
+
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
529
|
+
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
530
|
+
gcVersion: metadata?.gcFeature,
|
|
531
|
+
options: JSON.stringify(runtimeOptions),
|
|
532
|
+
featureGates: JSON.stringify({
|
|
457
533
|
disableCompression,
|
|
458
534
|
disableOpReentryCheck,
|
|
459
535
|
disableChunking,
|
|
460
536
|
disableAttachReorder: this.disableAttachReorder,
|
|
461
537
|
disablePartialFlush,
|
|
462
538
|
idCompressorEnabled: this.idCompressorEnabled,
|
|
463
|
-
summaryStateUpdateMethod: this.summaryStateUpdateMethod,
|
|
464
539
|
closeSummarizerDelayOverride,
|
|
465
|
-
}),
|
|
466
|
-
|
|
540
|
+
}),
|
|
541
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
542
|
+
groupedBatchingEnabled: this.groupedBatchingEnabled,
|
|
543
|
+
});
|
|
544
|
+
ReportOpPerfTelemetry(this.clientId, this.deltaManager, this.logger);
|
|
467
545
|
BindBatchTracker(this, this.logger);
|
|
468
546
|
this.entryPoint = new LazyPromise(async () => {
|
|
469
|
-
if (this.
|
|
547
|
+
if (this.isSummarizerClient) {
|
|
470
548
|
assert(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
|
|
471
549
|
return this._summarizer;
|
|
472
550
|
}
|
|
473
|
-
return initializeEntryPoint
|
|
551
|
+
return initializeEntryPoint?.(this);
|
|
474
552
|
});
|
|
475
553
|
}
|
|
554
|
+
/**
|
|
555
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
556
|
+
*/
|
|
476
557
|
get IFluidRouter() {
|
|
477
558
|
return this;
|
|
478
559
|
}
|
|
@@ -518,22 +599,34 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
518
599
|
* This object should provide all the functionality that the Container is expected to provide to the loader layer.
|
|
519
600
|
*/
|
|
520
601
|
static async loadRuntime(params) {
|
|
521
|
-
|
|
522
|
-
const
|
|
602
|
+
const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
|
|
603
|
+
const initializeEntryPoint = params.initializeEntryPoint ??
|
|
604
|
+
(async (containerRuntime) => ({
|
|
605
|
+
get IFluidRouter() {
|
|
606
|
+
return this;
|
|
607
|
+
},
|
|
608
|
+
async request(req) {
|
|
609
|
+
return containerRuntime.request(req);
|
|
610
|
+
},
|
|
611
|
+
}));
|
|
523
612
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
524
613
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
525
614
|
const backCompatContext = context;
|
|
526
|
-
const passLogger =
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
615
|
+
const passLogger = backCompatContext.taggedLogger ??
|
|
616
|
+
// eslint-disable-next-line import/no-deprecated
|
|
617
|
+
new TaggedLoggerAdapter(backCompatContext.logger);
|
|
618
|
+
const logger = createChildLogger({
|
|
619
|
+
logger: passLogger,
|
|
620
|
+
properties: {
|
|
621
|
+
all: {
|
|
622
|
+
runtimeVersion: pkgVersion,
|
|
623
|
+
},
|
|
530
624
|
},
|
|
531
625
|
});
|
|
532
626
|
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = false, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
|
|
533
627
|
const registry = new FluidDataStoreRegistry(registryEntries);
|
|
534
628
|
const tryFetchBlob = async (blobName) => {
|
|
535
|
-
|
|
536
|
-
const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
|
|
629
|
+
const blobId = context.baseSnapshot?.blobs[blobName];
|
|
537
630
|
if (context.baseSnapshot && blobId) {
|
|
538
631
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
539
632
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
@@ -548,16 +641,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
548
641
|
tryFetchBlob(aliasBlobName),
|
|
549
642
|
tryFetchBlob(idCompressorBlobName),
|
|
550
643
|
]);
|
|
551
|
-
const loadExisting = existing === true || context.existing === true;
|
|
552
644
|
// read snapshot blobs needed for BlobManager to load
|
|
553
|
-
const blobManagerSnapshot = await BlobManager.load(
|
|
645
|
+
const blobManagerSnapshot = await BlobManager.load(context.baseSnapshot?.trees[blobsTreeName], async (id) => {
|
|
554
646
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
555
647
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
556
648
|
assert(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
|
|
557
649
|
return readAndParse(context.storage, id);
|
|
558
650
|
});
|
|
559
651
|
// Verify summary runtime sequence number matches protocol sequence number.
|
|
560
|
-
const runtimeSequenceNumber =
|
|
652
|
+
const runtimeSequenceNumber = metadata?.message?.sequenceNumber;
|
|
561
653
|
// When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
|
|
562
654
|
if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
|
|
563
655
|
const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
|
|
@@ -572,22 +664,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
572
664
|
logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
|
|
573
665
|
}
|
|
574
666
|
else {
|
|
575
|
-
// Call both close and dispose as closeFn implementation will no longer dispose runtime in future
|
|
576
667
|
context.closeFn(error);
|
|
577
|
-
(_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
|
|
578
668
|
}
|
|
579
669
|
}
|
|
580
670
|
}
|
|
581
|
-
const idCompressorEnabled =
|
|
671
|
+
const idCompressorEnabled = metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
|
|
582
672
|
let idCompressor;
|
|
583
673
|
if (idCompressorEnabled) {
|
|
584
674
|
const { IdCompressor, createSessionId } = await import("./id-compressor");
|
|
585
675
|
idCompressor =
|
|
586
676
|
serializedIdCompressor !== undefined
|
|
587
677
|
? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
|
|
588
|
-
:
|
|
678
|
+
: IdCompressor.create(logger);
|
|
589
679
|
}
|
|
590
|
-
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks
|
|
680
|
+
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
|
|
591
681
|
summaryOptions,
|
|
592
682
|
gcOptions,
|
|
593
683
|
loadSequenceNumberVerification,
|
|
@@ -598,8 +688,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
598
688
|
enableRuntimeIdCompressor,
|
|
599
689
|
enableOpReentryCheck,
|
|
600
690
|
enableGroupedBatching,
|
|
601
|
-
}, containerScope, logger,
|
|
691
|
+
}, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
|
|
602
692
|
initializeEntryPoint);
|
|
693
|
+
await runtime.blobManager.processStashedChanges();
|
|
603
694
|
// It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
|
|
604
695
|
// at 1, so we won't see a replayed saved op with a sequence number of 0.
|
|
605
696
|
await runtime.pendingStateManager.applyStashedOpsAt(0);
|
|
@@ -607,34 +698,16 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
607
698
|
await runtime.initializeBaseState();
|
|
608
699
|
return runtime;
|
|
609
700
|
}
|
|
610
|
-
get options() {
|
|
611
|
-
return this.context.options;
|
|
612
|
-
}
|
|
613
701
|
get clientId() {
|
|
614
|
-
return this.
|
|
615
|
-
}
|
|
616
|
-
get clientDetails() {
|
|
617
|
-
return this.context.clientDetails;
|
|
702
|
+
return this._getClientId();
|
|
618
703
|
}
|
|
619
704
|
get storage() {
|
|
620
705
|
return this._storage;
|
|
621
706
|
}
|
|
707
|
+
/** @deprecated - The functionality is no longer exposed publicly */
|
|
622
708
|
get reSubmitFn() {
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
}
|
|
626
|
-
get disposeFn() {
|
|
627
|
-
var _a;
|
|
628
|
-
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
629
|
-
return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
|
|
630
|
-
}
|
|
631
|
-
get closeFn() {
|
|
632
|
-
// Also call disposeFn to retain functionality of runtime being disposed on close
|
|
633
|
-
return (error) => {
|
|
634
|
-
var _a, _b;
|
|
635
|
-
this.context.closeFn(error);
|
|
636
|
-
(_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
|
|
637
|
-
};
|
|
709
|
+
return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
|
|
710
|
+
// Note: compatDetails is not included in this deprecated API
|
|
638
711
|
}
|
|
639
712
|
get flushMode() {
|
|
640
713
|
return this._flushMode;
|
|
@@ -646,7 +719,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
646
719
|
return this.registry;
|
|
647
720
|
}
|
|
648
721
|
get attachState() {
|
|
649
|
-
return this.
|
|
722
|
+
return this._getAttachState();
|
|
650
723
|
}
|
|
651
724
|
get IFluidHandleContext() {
|
|
652
725
|
return this.handleContext;
|
|
@@ -673,8 +746,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
673
746
|
}
|
|
674
747
|
/** clientId of parent (non-summarizing) container that owns summarizer container */
|
|
675
748
|
get summarizerClientId() {
|
|
676
|
-
|
|
677
|
-
return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
|
|
749
|
+
return this.summarizerClientElection?.electedClientId;
|
|
678
750
|
}
|
|
679
751
|
get disposed() {
|
|
680
752
|
return this._disposed;
|
|
@@ -711,12 +783,11 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
711
783
|
await this.garbageCollector.initializeBaseState();
|
|
712
784
|
}
|
|
713
785
|
dispose(error) {
|
|
714
|
-
var _a;
|
|
715
786
|
if (this._disposed) {
|
|
716
787
|
return;
|
|
717
788
|
}
|
|
718
789
|
this._disposed = true;
|
|
719
|
-
this.logger.sendTelemetryEvent({
|
|
790
|
+
this.mc.logger.sendTelemetryEvent({
|
|
720
791
|
eventName: "ContainerRuntimeDisposed",
|
|
721
792
|
isDirty: this.isDirty,
|
|
722
793
|
lastSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
@@ -726,7 +797,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
726
797
|
this.summaryManager.dispose();
|
|
727
798
|
}
|
|
728
799
|
this.garbageCollector.dispose();
|
|
729
|
-
|
|
800
|
+
this._summarizer?.dispose();
|
|
730
801
|
this.dataStores.dispose();
|
|
731
802
|
this.pendingStateManager.dispose();
|
|
732
803
|
this.emit("dispose");
|
|
@@ -735,6 +806,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
735
806
|
/**
|
|
736
807
|
* Notifies this object about the request made to the container.
|
|
737
808
|
* @param request - Request made to the handler.
|
|
809
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
738
810
|
*/
|
|
739
811
|
async request(request) {
|
|
740
812
|
try {
|
|
@@ -786,7 +858,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
786
858
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
787
859
|
// unintentionally modifying the url if that changes.
|
|
788
860
|
assert(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
|
|
789
|
-
return dataStore.
|
|
861
|
+
return dataStore.request(subRequest);
|
|
790
862
|
}
|
|
791
863
|
return create404Response(request);
|
|
792
864
|
}
|
|
@@ -801,19 +873,17 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
801
873
|
return this.entryPoint;
|
|
802
874
|
}
|
|
803
875
|
internalId(maybeAlias) {
|
|
804
|
-
|
|
805
|
-
return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
|
|
876
|
+
return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
|
|
806
877
|
}
|
|
807
878
|
async getDataStoreFromRequest(id, request) {
|
|
808
|
-
var _a, _b, _c;
|
|
809
879
|
const headerData = {};
|
|
810
|
-
if (typeof
|
|
880
|
+
if (typeof request.headers?.[RuntimeHeaders.wait] === "boolean") {
|
|
811
881
|
headerData.wait = request.headers[RuntimeHeaders.wait];
|
|
812
882
|
}
|
|
813
|
-
if (typeof
|
|
883
|
+
if (typeof request.headers?.[RuntimeHeaders.viaHandle] === "boolean") {
|
|
814
884
|
headerData.viaHandle = request.headers[RuntimeHeaders.viaHandle];
|
|
815
885
|
}
|
|
816
|
-
if (typeof
|
|
886
|
+
if (typeof request.headers?.[AllowTombstoneRequestHeaderKey] === "boolean") {
|
|
817
887
|
headerData.allowTombstone = request.headers[AllowTombstoneRequestHeaderKey];
|
|
818
888
|
}
|
|
819
889
|
await this.dataStores.waitIfPendingAlias(id);
|
|
@@ -823,22 +893,27 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
823
893
|
// Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
|
|
824
894
|
// the same as GC nodes id.
|
|
825
895
|
const urlWithoutQuery = trimLeadingAndTrailingSlashes(request.url.split("?")[0]);
|
|
826
|
-
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request
|
|
896
|
+
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request?.headers);
|
|
827
897
|
return dataStoreChannel;
|
|
828
898
|
}
|
|
829
899
|
/** Adds the container's metadata to the given summary tree. */
|
|
830
900
|
addMetadataToSummary(summaryTree) {
|
|
831
|
-
|
|
832
|
-
|
|
901
|
+
const metadata = {
|
|
902
|
+
...this.createContainerMetadata,
|
|
833
903
|
// Increment the summary number for the next summary that will be generated.
|
|
834
|
-
summaryNumber: this.nextSummaryNumber++,
|
|
904
|
+
summaryNumber: this.nextSummaryNumber++,
|
|
905
|
+
summaryFormatVersion: 1,
|
|
906
|
+
...this.garbageCollector.getMetadata(),
|
|
835
907
|
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
836
908
|
// last summary.
|
|
837
|
-
message:
|
|
909
|
+
message: extractSummaryMetadataMessage(this.deltaManager.lastMessage) ??
|
|
910
|
+
this.messageAtLastSummary,
|
|
911
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
912
|
+
idCompressorEnabled: this.idCompressorEnabled ? true : undefined,
|
|
913
|
+
};
|
|
838
914
|
addBlobToSummary(summaryTree, metadataBlobName, JSON.stringify(metadata));
|
|
839
915
|
}
|
|
840
916
|
addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
|
|
841
|
-
var _a;
|
|
842
917
|
this.addMetadataToSummary(summaryTree);
|
|
843
918
|
if (this.idCompressorEnabled) {
|
|
844
919
|
assert(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
|
|
@@ -854,7 +929,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
854
929
|
addBlobToSummary(summaryTree, aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
855
930
|
}
|
|
856
931
|
if (this.summarizerClientElection) {
|
|
857
|
-
const electedSummarizerContent = JSON.stringify(
|
|
932
|
+
const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
|
|
858
933
|
addBlobToSummary(summaryTree, electedSummarizerBlobName, electedSummarizerContent);
|
|
859
934
|
}
|
|
860
935
|
const blobManagerSummary = this.blobManager.summarize();
|
|
@@ -891,7 +966,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
891
966
|
this.mc.logger.sendTelemetryEvent({
|
|
892
967
|
eventName: "ReconnectsWithNoProgress",
|
|
893
968
|
attempts: this.consecutiveReconnects,
|
|
894
|
-
pendingMessages: this.
|
|
969
|
+
pendingMessages: this.pendingMessagesCount,
|
|
895
970
|
});
|
|
896
971
|
}
|
|
897
972
|
return this.consecutiveReconnects < this.maxConsecutiveReconnects;
|
|
@@ -901,7 +976,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
901
976
|
// in their own batches before the originating batch is sent.
|
|
902
977
|
// Therefore, receiving them while attempting to send the originating batch
|
|
903
978
|
// does not mean that the container is making any progress.
|
|
904
|
-
if (
|
|
979
|
+
if (message?.type !== ContainerMessageType.ChunkedOp) {
|
|
905
980
|
this.consecutiveReconnects = 0;
|
|
906
981
|
}
|
|
907
982
|
}
|
|
@@ -951,13 +1026,13 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
951
1026
|
*/
|
|
952
1027
|
parseOpContent(serializedContent) {
|
|
953
1028
|
assert(serializedContent !== undefined, 0x6d5 /* content must be defined */);
|
|
954
|
-
const
|
|
955
|
-
assert(
|
|
956
|
-
return { type
|
|
1029
|
+
const { type, contents, compatDetails } = JSON.parse(serializedContent);
|
|
1030
|
+
assert(type !== undefined, 0x6d6 /* incorrect op content format */);
|
|
1031
|
+
return { type, contents, compatDetails };
|
|
957
1032
|
}
|
|
958
1033
|
async applyStashedOp(op) {
|
|
959
1034
|
// Need to parse from string for back-compat
|
|
960
|
-
const { type, contents } = this.parseOpContent(op);
|
|
1035
|
+
const { type, contents, compatDetails } = this.parseOpContent(op);
|
|
961
1036
|
switch (type) {
|
|
962
1037
|
case ContainerMessageType.FluidDataStoreOp:
|
|
963
1038
|
return this.dataStores.applyStashedOp(contents);
|
|
@@ -973,8 +1048,22 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
973
1048
|
throw new Error("chunkedOp not expected here");
|
|
974
1049
|
case ContainerMessageType.Rejoin:
|
|
975
1050
|
throw new Error("rejoin not expected here");
|
|
976
|
-
default:
|
|
977
|
-
|
|
1051
|
+
default: {
|
|
1052
|
+
// This should be extremely rare for stashed ops.
|
|
1053
|
+
// It would require a newer runtime stashing ops and then an older one applying them,
|
|
1054
|
+
// e.g. if an app rolled back its container version
|
|
1055
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1056
|
+
if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
|
|
1057
|
+
const error = DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
|
|
1058
|
+
messageDetails: JSON.stringify({
|
|
1059
|
+
type,
|
|
1060
|
+
compatBehavior,
|
|
1061
|
+
}),
|
|
1062
|
+
});
|
|
1063
|
+
this.closeFn(error);
|
|
1064
|
+
throw error;
|
|
1065
|
+
}
|
|
1066
|
+
}
|
|
978
1067
|
}
|
|
979
1068
|
}
|
|
980
1069
|
setConnectionState(connected, clientId) {
|
|
@@ -986,24 +1075,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
986
1075
|
// Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
|
|
987
1076
|
return;
|
|
988
1077
|
}
|
|
989
|
-
// If attachment blobs were added while disconnected, we need to delay
|
|
990
|
-
// propagation of the "connected" event until we have uploaded them to
|
|
991
|
-
// ensure we don't submit ops referencing a blob that has not been uploaded
|
|
992
|
-
// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
|
|
993
|
-
const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
|
|
994
|
-
if (connecting && this.blobManager.hasPendingOfflineUploads) {
|
|
995
|
-
assert(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
|
|
996
|
-
assert(!!clientId, 0x393 /* Must have clientId when connecting */);
|
|
997
|
-
this.delayConnectClientId = clientId;
|
|
998
|
-
this.blobManager.onConnected().then(() => {
|
|
999
|
-
// make sure we didn't reconnect before the promise resolved
|
|
1000
|
-
if (this.delayConnectClientId === clientId && !this.disposed) {
|
|
1001
|
-
this.delayConnectClientId = undefined;
|
|
1002
|
-
this.setConnectionStateCore(connected, clientId);
|
|
1003
|
-
}
|
|
1004
|
-
}, (error) => this.closeFn(error));
|
|
1005
|
-
return;
|
|
1006
|
-
}
|
|
1007
1078
|
this.setConnectionStateCore(connected, clientId);
|
|
1008
1079
|
}
|
|
1009
1080
|
setConnectionStateCore(connected, clientId) {
|
|
@@ -1012,6 +1083,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1012
1083
|
// There might be no change of state due to Container calling this API after loading runtime.
|
|
1013
1084
|
const changeOfState = this._connected !== connected;
|
|
1014
1085
|
const reconnection = changeOfState && !connected;
|
|
1086
|
+
// We need to flush the ops currently collected by Outbox to preserve original order.
|
|
1087
|
+
// This flush NEEDS to happen before we set the ContainerRuntime to "connected".
|
|
1088
|
+
// We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
|
|
1089
|
+
if (changeOfState && connected) {
|
|
1090
|
+
this.flush();
|
|
1091
|
+
}
|
|
1015
1092
|
this._connected = connected;
|
|
1016
1093
|
if (!connected) {
|
|
1017
1094
|
this._perfSignalData.signalsLost = 0;
|
|
@@ -1028,7 +1105,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1028
1105
|
this.closeFn(DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
|
|
1029
1106
|
dataLoss: 1,
|
|
1030
1107
|
attempts: this.consecutiveReconnects,
|
|
1031
|
-
pendingMessages: this.
|
|
1108
|
+
pendingMessages: this.pendingMessagesCount,
|
|
1032
1109
|
}));
|
|
1033
1110
|
return;
|
|
1034
1111
|
}
|
|
@@ -1045,18 +1122,23 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1045
1122
|
}
|
|
1046
1123
|
process(messageArg, local) {
|
|
1047
1124
|
this.verifyNotClosed();
|
|
1048
|
-
// Whether or not the message
|
|
1125
|
+
// Whether or not the message appears to be a runtime message from an up-to-date client.
|
|
1049
1126
|
// It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
|
|
1050
1127
|
// or something different, like a system message.
|
|
1051
|
-
const
|
|
1128
|
+
const modernRuntimeMessage = messageArg.type === MessageType.Operation;
|
|
1052
1129
|
// Do shallow copy of message, as the processing flow will modify it.
|
|
1053
|
-
const messageCopy =
|
|
1130
|
+
const messageCopy = { ...messageArg };
|
|
1054
1131
|
for (const message of this.remoteMessageProcessor.process(messageCopy)) {
|
|
1055
|
-
this.processCore(message, local,
|
|
1132
|
+
this.processCore(message, local, modernRuntimeMessage);
|
|
1056
1133
|
}
|
|
1057
1134
|
}
|
|
1058
|
-
|
|
1059
|
-
|
|
1135
|
+
/**
|
|
1136
|
+
* Direct the message to the correct subsystem for processing, and implement other side effects
|
|
1137
|
+
* @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
|
|
1138
|
+
* @param local - Did this client send the op?
|
|
1139
|
+
* @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
|
|
1140
|
+
*/
|
|
1141
|
+
processCore(message, local, modernRuntimeMessage) {
|
|
1060
1142
|
// Surround the actual processing of the operation with messages to the schedule manager indicating
|
|
1061
1143
|
// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
|
|
1062
1144
|
// messages once a batch has been fully processed.
|
|
@@ -1064,7 +1146,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1064
1146
|
this._processedClientSequenceNumber = message.clientSequenceNumber;
|
|
1065
1147
|
try {
|
|
1066
1148
|
let localOpMetadata;
|
|
1067
|
-
if (local &&
|
|
1149
|
+
if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
|
|
1068
1150
|
localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
|
|
1069
1151
|
}
|
|
1070
1152
|
// If there are no more pending messages after processing a local message,
|
|
@@ -1072,45 +1154,8 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1072
1154
|
if (!this.hasPendingMessages()) {
|
|
1073
1155
|
this.updateDocumentDirtyState(false);
|
|
1074
1156
|
}
|
|
1075
|
-
|
|
1076
|
-
|
|
1077
|
-
case ContainerMessageType.Attach:
|
|
1078
|
-
this.dataStores.processAttachMessage(message, local);
|
|
1079
|
-
break;
|
|
1080
|
-
case ContainerMessageType.Alias:
|
|
1081
|
-
this.processAliasMessage(message, localOpMetadata, local);
|
|
1082
|
-
break;
|
|
1083
|
-
case ContainerMessageType.FluidDataStoreOp:
|
|
1084
|
-
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1085
|
-
break;
|
|
1086
|
-
case ContainerMessageType.BlobAttach:
|
|
1087
|
-
this.blobManager.processBlobAttachOp(message, local);
|
|
1088
|
-
break;
|
|
1089
|
-
case ContainerMessageType.IdAllocation:
|
|
1090
|
-
assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1091
|
-
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1092
|
-
break;
|
|
1093
|
-
case ContainerMessageType.ChunkedOp:
|
|
1094
|
-
case ContainerMessageType.Rejoin:
|
|
1095
|
-
break;
|
|
1096
|
-
default:
|
|
1097
|
-
if (runtimeMessage) {
|
|
1098
|
-
const error = DataProcessingError.create(
|
|
1099
|
-
// Former assert 0x3ce
|
|
1100
|
-
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1101
|
-
local,
|
|
1102
|
-
type: message.type,
|
|
1103
|
-
contentType: typeof message.contents,
|
|
1104
|
-
batch: (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch,
|
|
1105
|
-
compression: message.compression,
|
|
1106
|
-
});
|
|
1107
|
-
this.closeFn(error);
|
|
1108
|
-
throw error;
|
|
1109
|
-
}
|
|
1110
|
-
}
|
|
1111
|
-
if (runtimeMessage || this.groupedBatchingEnabled) {
|
|
1112
|
-
this.emit("op", message, runtimeMessage);
|
|
1113
|
-
}
|
|
1157
|
+
this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
|
|
1158
|
+
this.emit("op", message, modernRuntimeMessage);
|
|
1114
1159
|
this.scheduleManager.afterOpProcessing(undefined, message);
|
|
1115
1160
|
if (local) {
|
|
1116
1161
|
// If we have processed a local op, this means that the container is
|
|
@@ -1124,8 +1169,59 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1124
1169
|
throw e;
|
|
1125
1170
|
}
|
|
1126
1171
|
}
|
|
1127
|
-
|
|
1128
|
-
|
|
1172
|
+
/**
|
|
1173
|
+
* Assuming the given message is also a ContainerRuntimeMessage,
|
|
1174
|
+
* checks its type and dispatches the message to the appropriate handler in the runtime.
|
|
1175
|
+
* Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
|
|
1176
|
+
*/
|
|
1177
|
+
validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
|
|
1178
|
+
// Optimistically extract ContainerRuntimeMessage-specific props from the message
|
|
1179
|
+
const { type: maybeContainerMessageType, compatDetails } = message;
|
|
1180
|
+
switch (maybeContainerMessageType) {
|
|
1181
|
+
case ContainerMessageType.Attach:
|
|
1182
|
+
this.dataStores.processAttachMessage(message, local);
|
|
1183
|
+
break;
|
|
1184
|
+
case ContainerMessageType.Alias:
|
|
1185
|
+
this.dataStores.processAliasMessage(message, localOpMetadata, local);
|
|
1186
|
+
break;
|
|
1187
|
+
case ContainerMessageType.FluidDataStoreOp:
|
|
1188
|
+
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1189
|
+
break;
|
|
1190
|
+
case ContainerMessageType.BlobAttach:
|
|
1191
|
+
this.blobManager.processBlobAttachOp(message, local);
|
|
1192
|
+
break;
|
|
1193
|
+
case ContainerMessageType.IdAllocation:
|
|
1194
|
+
assert(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1195
|
+
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1196
|
+
break;
|
|
1197
|
+
case ContainerMessageType.ChunkedOp:
|
|
1198
|
+
case ContainerMessageType.Rejoin:
|
|
1199
|
+
break;
|
|
1200
|
+
default: {
|
|
1201
|
+
// If we didn't necessarily expect a runtime message type, then no worries - just return
|
|
1202
|
+
// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
|
|
1203
|
+
if (!expectRuntimeMessageType) {
|
|
1204
|
+
return;
|
|
1205
|
+
}
|
|
1206
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1207
|
+
if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
|
|
1208
|
+
const error = DataProcessingError.create(
|
|
1209
|
+
// Former assert 0x3ce
|
|
1210
|
+
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1211
|
+
local,
|
|
1212
|
+
messageDetails: JSON.stringify({
|
|
1213
|
+
type: message.type,
|
|
1214
|
+
contentType: typeof message.contents,
|
|
1215
|
+
compatBehavior,
|
|
1216
|
+
batch: message.metadata?.batch,
|
|
1217
|
+
compression: message.compression,
|
|
1218
|
+
}),
|
|
1219
|
+
});
|
|
1220
|
+
this.closeFn(error);
|
|
1221
|
+
throw error;
|
|
1222
|
+
}
|
|
1223
|
+
}
|
|
1224
|
+
}
|
|
1129
1225
|
}
|
|
1130
1226
|
/**
|
|
1131
1227
|
* Emits the Signal event and update the perf signal data.
|
|
@@ -1133,7 +1229,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1133
1229
|
*/
|
|
1134
1230
|
sendSignalTelemetryEvent(clientSignalSequenceNumber) {
|
|
1135
1231
|
const duration = Date.now() - this._perfSignalData.signalTimestamp;
|
|
1136
|
-
this.logger.sendPerformanceEvent({
|
|
1232
|
+
this.mc.logger.sendPerformanceEvent({
|
|
1137
1233
|
eventName: "SignalLatency",
|
|
1138
1234
|
duration,
|
|
1139
1235
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1156,7 +1252,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1156
1252
|
this._perfSignalData.trackingSignalSequenceNumber) {
|
|
1157
1253
|
this._perfSignalData.signalsLost++;
|
|
1158
1254
|
this._perfSignalData.trackingSignalSequenceNumber = undefined;
|
|
1159
|
-
this.logger.sendErrorEvent({
|
|
1255
|
+
this.mc.logger.sendErrorEvent({
|
|
1160
1256
|
eventName: "SignalLost",
|
|
1161
1257
|
type: envelope.contents.type,
|
|
1162
1258
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1180,6 +1276,12 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1180
1276
|
}
|
|
1181
1277
|
this.dataStores.processSignal(envelope.address, transformed, local);
|
|
1182
1278
|
}
|
|
1279
|
+
/**
|
|
1280
|
+
* Returns the runtime of the data store.
|
|
1281
|
+
* @param id - Id supplied during creating the data store.
|
|
1282
|
+
* @param wait - True if you want to wait for it.
|
|
1283
|
+
* @deprecated - Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
|
|
1284
|
+
*/
|
|
1183
1285
|
async getRootDataStore(id, wait = true) {
|
|
1184
1286
|
return this.getRootDataStoreChannel(id, wait);
|
|
1185
1287
|
}
|
|
@@ -1241,9 +1343,25 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1241
1343
|
}
|
|
1242
1344
|
return result;
|
|
1243
1345
|
}
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1346
|
+
/**
|
|
1347
|
+
* Returns the aliased data store's entryPoint, given the alias.
|
|
1348
|
+
* @param alias - The alias for the data store.
|
|
1349
|
+
* @returns The data store's entry point ({@link @fluidframework/core-interfaces#IFluidHandle}) if it exists and is aliased.
|
|
1350
|
+
* Returns undefined if no data store has been assigned the given alias.
|
|
1351
|
+
*/
|
|
1352
|
+
async getAliasedDataStoreEntryPoint(alias) {
|
|
1353
|
+
await this.dataStores.waitIfPendingAlias(alias);
|
|
1354
|
+
const internalId = this.internalId(alias);
|
|
1355
|
+
const context = await this.dataStores.getDataStoreIfAvailable(internalId, { wait: false });
|
|
1356
|
+
// If the data store is not available or not an alias, return undefined.
|
|
1357
|
+
if (context === undefined || !(await context.isRoot())) {
|
|
1358
|
+
return undefined;
|
|
1359
|
+
}
|
|
1360
|
+
const channel = await context.realize();
|
|
1361
|
+
if (channel.entryPoint === undefined) {
|
|
1362
|
+
throw new UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
|
|
1363
|
+
}
|
|
1364
|
+
return channel.entryPoint;
|
|
1247
1365
|
}
|
|
1248
1366
|
createDetachedRootDataStore(pkg, rootDataStoreId) {
|
|
1249
1367
|
if (rootDataStoreId.includes("/")) {
|
|
@@ -1254,16 +1372,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1254
1372
|
createDetachedDataStore(pkg) {
|
|
1255
1373
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1256
1374
|
}
|
|
1257
|
-
async
|
|
1258
|
-
const
|
|
1259
|
-
|
|
1260
|
-
.
|
|
1261
|
-
|
|
1375
|
+
async createDataStore(pkg) {
|
|
1376
|
+
const id = uuid();
|
|
1377
|
+
return channelToDataStore(await this.dataStores
|
|
1378
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id)
|
|
1379
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1262
1380
|
}
|
|
1263
|
-
|
|
1264
|
-
|
|
1381
|
+
/**
|
|
1382
|
+
* @deprecated 0.16 Issue #1537, #3631
|
|
1383
|
+
* @internal
|
|
1384
|
+
*/
|
|
1385
|
+
async _createDataStoreWithProps(pkg, props, id = uuid()) {
|
|
1386
|
+
return channelToDataStore(await this.dataStores
|
|
1265
1387
|
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
|
|
1266
|
-
.realize();
|
|
1388
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1267
1389
|
}
|
|
1268
1390
|
canSendOps() {
|
|
1269
1391
|
// Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
|
|
@@ -1277,11 +1399,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1277
1399
|
return this.flushMode !== FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
|
|
1278
1400
|
}
|
|
1279
1401
|
getQuorum() {
|
|
1280
|
-
return this.
|
|
1402
|
+
return this._quorum;
|
|
1281
1403
|
}
|
|
1282
1404
|
getAudience() {
|
|
1283
|
-
|
|
1284
|
-
return this.context.audience;
|
|
1405
|
+
return this._audience;
|
|
1285
1406
|
}
|
|
1286
1407
|
/**
|
|
1287
1408
|
* Returns true of container is dirty, i.e. there are some pending local changes that
|
|
@@ -1290,7 +1411,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1290
1411
|
get isDirty() {
|
|
1291
1412
|
return this.dirtyContainer;
|
|
1292
1413
|
}
|
|
1293
|
-
isContainerMessageDirtyable(type, contents) {
|
|
1414
|
+
isContainerMessageDirtyable({ type, contents }) {
|
|
1294
1415
|
// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
|
|
1295
1416
|
// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
|
|
1296
1417
|
if (type === ContainerMessageType.Attach) {
|
|
@@ -1330,11 +1451,11 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1330
1451
|
submitSignal(type, content) {
|
|
1331
1452
|
this.verifyNotClosed();
|
|
1332
1453
|
const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
|
|
1333
|
-
return this.
|
|
1454
|
+
return this.submitSignalFn(envelope);
|
|
1334
1455
|
}
|
|
1335
1456
|
submitDataStoreSignal(address, type, content) {
|
|
1336
1457
|
const envelope = this.createNewSignalEnvelope(address, type, content);
|
|
1337
|
-
return this.
|
|
1458
|
+
return this.submitSignalFn(envelope);
|
|
1338
1459
|
}
|
|
1339
1460
|
setAttachState(attachState) {
|
|
1340
1461
|
if (attachState === AttachState.Attaching) {
|
|
@@ -1367,22 +1488,17 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1367
1488
|
this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
|
|
1368
1489
|
return summarizeResult.summary;
|
|
1369
1490
|
}
|
|
1370
|
-
async getAbsoluteUrl(relativeUrl) {
|
|
1371
|
-
if (this.context.getAbsoluteUrl === undefined) {
|
|
1372
|
-
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
1373
|
-
}
|
|
1374
|
-
if (this.attachState !== AttachState.Attached) {
|
|
1375
|
-
return undefined;
|
|
1376
|
-
}
|
|
1377
|
-
return this.context.getAbsoluteUrl(relativeUrl);
|
|
1378
|
-
}
|
|
1379
1491
|
async summarizeInternal(fullTree, trackState, telemetryContext) {
|
|
1380
1492
|
const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
|
|
1381
1493
|
// Wrap data store summaries in .channels subtree.
|
|
1382
1494
|
wrapSummaryInChannelsTree(summarizeResult);
|
|
1383
1495
|
const pathPartsForChildren = [channelsTreeName];
|
|
1384
1496
|
this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
|
|
1385
|
-
return
|
|
1497
|
+
return {
|
|
1498
|
+
...summarizeResult,
|
|
1499
|
+
id: "",
|
|
1500
|
+
pathPartsForChildren,
|
|
1501
|
+
};
|
|
1386
1502
|
}
|
|
1387
1503
|
/**
|
|
1388
1504
|
* Returns a summary of the runtime at the current sequence number.
|
|
@@ -1400,16 +1516,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1400
1516
|
runSweep,
|
|
1401
1517
|
});
|
|
1402
1518
|
try {
|
|
1403
|
-
let gcStats;
|
|
1404
1519
|
if (runGC) {
|
|
1405
|
-
|
|
1520
|
+
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
|
|
1406
1521
|
}
|
|
1407
1522
|
const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
|
|
1408
1523
|
assert(summary.type === SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1409
|
-
return { stats, summary
|
|
1524
|
+
return { stats, summary };
|
|
1410
1525
|
}
|
|
1411
1526
|
finally {
|
|
1412
|
-
this.logger.sendTelemetryEvent({
|
|
1527
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1413
1528
|
eventName: "SummarizeTelemetry",
|
|
1414
1529
|
details: telemetryContext.serialize(),
|
|
1415
1530
|
});
|
|
@@ -1471,7 +1586,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1471
1586
|
/**
|
|
1472
1587
|
* After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
|
|
1473
1588
|
* @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
|
|
1474
|
-
* @returns
|
|
1589
|
+
* @returns The routes of nodes that were deleted.
|
|
1475
1590
|
*/
|
|
1476
1591
|
deleteSweepReadyNodes(sweepReadyRoutes) {
|
|
1477
1592
|
const { dataStoreRoutes, blobManagerRoutes } = this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
|
|
@@ -1491,21 +1606,19 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1491
1606
|
* Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
|
|
1492
1607
|
*/
|
|
1493
1608
|
getCurrentReferenceTimestampMs() {
|
|
1494
|
-
var _a, _b, _c;
|
|
1495
1609
|
// Use the timestamp of the last message seen by this client as that is server generated. If no messages have
|
|
1496
1610
|
// been processed, use the timestamp of the message from the last summary.
|
|
1497
|
-
return
|
|
1611
|
+
return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
|
|
1498
1612
|
}
|
|
1499
1613
|
/**
|
|
1500
1614
|
* Returns the type of the GC node. Currently, there are nodes that belong to the root ("/"), data stores or
|
|
1501
1615
|
* blob manager.
|
|
1502
1616
|
*/
|
|
1503
1617
|
getNodeType(nodePath) {
|
|
1504
|
-
var _a;
|
|
1505
1618
|
if (this.isBlobPath(nodePath)) {
|
|
1506
1619
|
return GCNodeType.Blob;
|
|
1507
1620
|
}
|
|
1508
|
-
return
|
|
1621
|
+
return this.dataStores.getGCNodeType(nodePath) ?? GCNodeType.Other;
|
|
1509
1622
|
}
|
|
1510
1623
|
/**
|
|
1511
1624
|
* Called by GC to retrieve the package path of the node with the given path. The node should belong to a
|
|
@@ -1535,7 +1648,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1535
1648
|
/**
|
|
1536
1649
|
* From a given list of routes, separate and return routes that belong to blob manager and data stores.
|
|
1537
1650
|
* @param routes - A list of routes that can belong to data stores or blob manager.
|
|
1538
|
-
* @returns
|
|
1651
|
+
* @returns Two route lists - One that contains routes for blob manager and another one that contains routes
|
|
1539
1652
|
* for data stores.
|
|
1540
1653
|
*/
|
|
1541
1654
|
getDataStoreAndBlobManagerRoutes(routes) {
|
|
@@ -1576,23 +1689,62 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1576
1689
|
* @param options - options controlling how the summary is generated or submitted
|
|
1577
1690
|
*/
|
|
1578
1691
|
async submitSummary(options) {
|
|
1579
|
-
|
|
1580
|
-
const { fullTree = false, refreshLatestAck, summaryLogger } = options;
|
|
1692
|
+
const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
|
|
1581
1693
|
// The summary number for this summary. This will be updated during the summary process, so get it now and
|
|
1582
1694
|
// use it for all events logged during this summary.
|
|
1583
1695
|
const summaryNumber = this.nextSummaryNumber;
|
|
1584
|
-
const summaryNumberLogger =
|
|
1585
|
-
|
|
1696
|
+
const summaryNumberLogger = createChildLogger({
|
|
1697
|
+
logger: summaryLogger,
|
|
1698
|
+
properties: {
|
|
1699
|
+
all: { summaryNumber },
|
|
1700
|
+
},
|
|
1586
1701
|
});
|
|
1587
1702
|
assert(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
|
|
1588
1703
|
let latestSnapshotVersionId;
|
|
1589
1704
|
if (refreshLatestAck) {
|
|
1590
|
-
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(
|
|
1705
|
+
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(createChildLogger({
|
|
1706
|
+
logger: summaryNumberLogger,
|
|
1707
|
+
properties: { all: { safeSummary: true } },
|
|
1708
|
+
}));
|
|
1591
1709
|
const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
|
|
1592
1710
|
latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
|
|
1593
1711
|
// We might need to catch up to the latest summary's reference sequence number before pausing.
|
|
1594
1712
|
await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
|
|
1595
1713
|
}
|
|
1714
|
+
// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
|
|
1715
|
+
// incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
|
|
1716
|
+
// within the timeout, check if it should be failed or can continue.
|
|
1717
|
+
if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
|
|
1718
|
+
const countBefore = this.pendingMessagesCount;
|
|
1719
|
+
// The timeout for waiting for pending ops can be overridden via configurations.
|
|
1720
|
+
const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
|
|
1721
|
+
defaultPendingOpsWaitTimeoutMs;
|
|
1722
|
+
await new Promise((resolve, reject) => {
|
|
1723
|
+
const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
|
|
1724
|
+
this.once("saved", () => {
|
|
1725
|
+
clearTimeout(timeoutId);
|
|
1726
|
+
resolve();
|
|
1727
|
+
});
|
|
1728
|
+
this.once("dispose", () => {
|
|
1729
|
+
clearTimeout(timeoutId);
|
|
1730
|
+
reject(new Error("Runtime is disposed while summarizing"));
|
|
1731
|
+
});
|
|
1732
|
+
});
|
|
1733
|
+
// Log that there are pending ops while summarizing. This will help us gather data on how often this
|
|
1734
|
+
// happens, whether we attempted to wait for these ops to be acked and what was the result.
|
|
1735
|
+
summaryNumberLogger.sendTelemetryEvent({
|
|
1736
|
+
eventName: "PendingOpsWhileSummarizing",
|
|
1737
|
+
saved: this.hasPendingMessages() ? false : true,
|
|
1738
|
+
timeout: pendingOpsTimeout,
|
|
1739
|
+
countBefore,
|
|
1740
|
+
countAfter: this.pendingMessagesCount,
|
|
1741
|
+
});
|
|
1742
|
+
// There could still be pending ops. Check if summary should fail or continue.
|
|
1743
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
|
|
1744
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1745
|
+
return pendingMessagesFailResult;
|
|
1746
|
+
}
|
|
1747
|
+
}
|
|
1596
1748
|
const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
|
|
1597
1749
|
let summaryRefSeqNum;
|
|
1598
1750
|
try {
|
|
@@ -1607,7 +1759,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1607
1759
|
this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
|
|
1608
1760
|
// Helper function to check whether we should still continue between each async step.
|
|
1609
1761
|
const checkContinue = () => {
|
|
1610
|
-
var _a;
|
|
1611
1762
|
// Do not check for loss of connectivity directly! Instead leave it up to
|
|
1612
1763
|
// RunWhileConnectedCoordinator to control policy in a single place.
|
|
1613
1764
|
// This will allow easier change of design if we chose to. For example, we may chose to allow
|
|
@@ -1630,7 +1781,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1630
1781
|
error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
|
|
1631
1782
|
};
|
|
1632
1783
|
}
|
|
1633
|
-
assert(summaryRefSeqNum ===
|
|
1784
|
+
assert(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
|
|
1634
1785
|
if (lastAck !== this.summaryCollection.latestAck) {
|
|
1635
1786
|
return {
|
|
1636
1787
|
continue: false,
|
|
@@ -1669,6 +1820,25 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1669
1820
|
error,
|
|
1670
1821
|
};
|
|
1671
1822
|
}
|
|
1823
|
+
// If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
|
|
1824
|
+
if (this.validateSummaryBeforeUpload) {
|
|
1825
|
+
// Validate that the summaries generated by summarize nodes is correct.
|
|
1826
|
+
const validateResult = this.summarizerNode.validateSummary();
|
|
1827
|
+
if (!validateResult.success) {
|
|
1828
|
+
const { success, ...loggingProps } = validateResult;
|
|
1829
|
+
const error = new RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
|
|
1830
|
+
return {
|
|
1831
|
+
stage: "base",
|
|
1832
|
+
referenceSequenceNumber: summaryRefSeqNum,
|
|
1833
|
+
minimumSequenceNumber,
|
|
1834
|
+
error,
|
|
1835
|
+
};
|
|
1836
|
+
}
|
|
1837
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
|
|
1838
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1839
|
+
return pendingMessagesFailResult;
|
|
1840
|
+
}
|
|
1841
|
+
}
|
|
1672
1842
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
1673
1843
|
// Now that we have generated the summary, update the message at last summary to the last message processed.
|
|
1674
1844
|
this.messageAtLastSummary = this.deltaManager.lastMessage;
|
|
@@ -1681,7 +1851,15 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1681
1851
|
const gcSummaryTreeStats = summaryTree.tree[gcTreeKey]
|
|
1682
1852
|
? calculateStats(summaryTree.tree[gcTreeKey])
|
|
1683
1853
|
: undefined;
|
|
1684
|
-
const summaryStats =
|
|
1854
|
+
const summaryStats = {
|
|
1855
|
+
dataStoreCount: this.dataStores.size,
|
|
1856
|
+
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
1857
|
+
gcStateUpdatedDataStoreCount: this.garbageCollector.updatedDSCountSinceLastSummary,
|
|
1858
|
+
gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
|
|
1859
|
+
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
1860
|
+
summaryNumber,
|
|
1861
|
+
...partialStats,
|
|
1862
|
+
};
|
|
1685
1863
|
const generateSummaryData = {
|
|
1686
1864
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1687
1865
|
minimumSequenceNumber,
|
|
@@ -1692,7 +1870,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1692
1870
|
};
|
|
1693
1871
|
continueResult = checkContinue();
|
|
1694
1872
|
if (!continueResult.continue) {
|
|
1695
|
-
return
|
|
1873
|
+
return { stage: "generate", ...generateSummaryData, error: continueResult.error };
|
|
1696
1874
|
}
|
|
1697
1875
|
// It may happen that the lastAck it not correct due to missing summaryAck in case of single commit
|
|
1698
1876
|
// summary. So if the previous summarizer closes just after submitting the summary and before
|
|
@@ -1700,7 +1878,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1700
1878
|
// latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
|
|
1701
1879
|
// the one fetched from storage as parent as that is the latest.
|
|
1702
1880
|
let summaryContext;
|
|
1703
|
-
if (
|
|
1881
|
+
if (lastAck?.summaryAck.contents.handle !== latestSnapshotVersionId &&
|
|
1704
1882
|
latestSnapshotVersionId !== undefined) {
|
|
1705
1883
|
summaryContext = {
|
|
1706
1884
|
proposalHandle: undefined,
|
|
@@ -1711,7 +1889,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1711
1889
|
else if (lastAck === undefined) {
|
|
1712
1890
|
summaryContext = {
|
|
1713
1891
|
proposalHandle: undefined,
|
|
1714
|
-
ackHandle:
|
|
1892
|
+
ackHandle: this.loadedFromVersionId,
|
|
1715
1893
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1716
1894
|
};
|
|
1717
1895
|
}
|
|
@@ -1727,7 +1905,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1727
1905
|
handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
|
|
1728
1906
|
}
|
|
1729
1907
|
catch (error) {
|
|
1730
|
-
return
|
|
1908
|
+
return { stage: "generate", ...generateSummaryData, error };
|
|
1731
1909
|
}
|
|
1732
1910
|
const parent = summaryContext.ackHandle;
|
|
1733
1911
|
const summaryMessage = {
|
|
@@ -1737,27 +1915,42 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1737
1915
|
message,
|
|
1738
1916
|
parents: parent ? [parent] : [],
|
|
1739
1917
|
};
|
|
1740
|
-
const uploadData =
|
|
1918
|
+
const uploadData = {
|
|
1919
|
+
...generateSummaryData,
|
|
1920
|
+
handle,
|
|
1921
|
+
uploadDuration: trace.trace().duration,
|
|
1922
|
+
};
|
|
1741
1923
|
continueResult = checkContinue();
|
|
1742
1924
|
if (!continueResult.continue) {
|
|
1743
|
-
return
|
|
1925
|
+
return { stage: "upload", ...uploadData, error: continueResult.error };
|
|
1744
1926
|
}
|
|
1745
1927
|
let clientSequenceNumber;
|
|
1746
1928
|
try {
|
|
1747
1929
|
clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
|
|
1748
1930
|
}
|
|
1749
1931
|
catch (error) {
|
|
1750
|
-
return
|
|
1932
|
+
return { stage: "upload", ...uploadData, error };
|
|
1933
|
+
}
|
|
1934
|
+
const submitData = {
|
|
1935
|
+
stage: "submit",
|
|
1936
|
+
...uploadData,
|
|
1937
|
+
clientSequenceNumber,
|
|
1938
|
+
submitOpDuration: trace.trace().duration,
|
|
1939
|
+
};
|
|
1940
|
+
try {
|
|
1941
|
+
// If validateSummaryBeforeUpload is false, the summary should be validated in this step.
|
|
1942
|
+
this.summarizerNode.completeSummary(handle, !this.validateSummaryBeforeUpload /* validate */);
|
|
1943
|
+
}
|
|
1944
|
+
catch (error) {
|
|
1945
|
+
return { stage: "upload", ...uploadData, error };
|
|
1751
1946
|
}
|
|
1752
|
-
const submitData = Object.assign(Object.assign({ stage: "submit" }, uploadData), { clientSequenceNumber, submitOpDuration: trace.trace().duration });
|
|
1753
|
-
this.summarizerNode.completeSummary(handle);
|
|
1754
1947
|
return submitData;
|
|
1755
1948
|
}
|
|
1756
1949
|
finally {
|
|
1757
1950
|
// Cleanup wip summary in case of failure
|
|
1758
1951
|
this.summarizerNode.clearSummary();
|
|
1759
1952
|
// ! This needs to happen before we resume inbound queues to ensure heuristics are tracked correctly
|
|
1760
|
-
|
|
1953
|
+
this._summarizer?.recordSummaryAttempt?.(summaryRefSeqNum);
|
|
1761
1954
|
// Restart the delta manager
|
|
1762
1955
|
this.deltaManager.inbound.resume();
|
|
1763
1956
|
if (shouldPauseInboundSignal) {
|
|
@@ -1765,8 +1958,55 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1765
1958
|
}
|
|
1766
1959
|
}
|
|
1767
1960
|
}
|
|
1961
|
+
/**
|
|
1962
|
+
* This helper is called during summarization. If there are pending ops, it will return a failed summarize result
|
|
1963
|
+
* (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
|
|
1964
|
+
* @param logger - The logger to be used for sending telemetry.
|
|
1965
|
+
* @param referenceSequenceNumber - The reference sequence number of the summary attempt.
|
|
1966
|
+
* @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
|
|
1967
|
+
* @param finalAttempt - Whether this is the final summary attempt.
|
|
1968
|
+
* @param beforeSummaryGeneration - Whether this is called before summary generation or after.
|
|
1969
|
+
* @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
|
|
1970
|
+
*/
|
|
1971
|
+
async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
|
|
1972
|
+
if (!this.hasPendingMessages()) {
|
|
1973
|
+
return;
|
|
1974
|
+
}
|
|
1975
|
+
// If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
|
|
1976
|
+
// This is a fallback to make progress in documents where there are consistently pending ops in
|
|
1977
|
+
// the summarizer.
|
|
1978
|
+
if (finalAttempt &&
|
|
1979
|
+
this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
|
|
1980
|
+
const error = DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
|
|
1981
|
+
logger.sendErrorEvent({
|
|
1982
|
+
eventName: "SkipFailingIncorrectSummary",
|
|
1983
|
+
referenceSequenceNumber,
|
|
1984
|
+
minimumSequenceNumber,
|
|
1985
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
1986
|
+
}, error);
|
|
1987
|
+
}
|
|
1988
|
+
else {
|
|
1989
|
+
// The retry delay when there are pending ops can be overridden via config so that we can adjust it
|
|
1990
|
+
// based on telemetry while we decide on a stable number.
|
|
1991
|
+
const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
|
|
1992
|
+
defaultPendingOpsRetryDelayMs;
|
|
1993
|
+
const error = new RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
|
|
1994
|
+
count: this.pendingMessagesCount,
|
|
1995
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
1996
|
+
});
|
|
1997
|
+
return {
|
|
1998
|
+
stage: "base",
|
|
1999
|
+
referenceSequenceNumber,
|
|
2000
|
+
minimumSequenceNumber,
|
|
2001
|
+
error,
|
|
2002
|
+
};
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
get pendingMessagesCount() {
|
|
2006
|
+
return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
|
|
2007
|
+
}
|
|
1768
2008
|
hasPendingMessages() {
|
|
1769
|
-
return this.
|
|
2009
|
+
return this.pendingMessagesCount !== 0;
|
|
1770
2010
|
}
|
|
1771
2011
|
updateDocumentDirtyState(dirty) {
|
|
1772
2012
|
if (this.attachState !== AttachState.Attached) {
|
|
@@ -1782,7 +2022,6 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1782
2022
|
this.dirtyContainer = dirty;
|
|
1783
2023
|
if (this.emitDirtyDocumentEvent) {
|
|
1784
2024
|
this.emit(dirty ? "dirty" : "saved");
|
|
1785
|
-
this.context.updateDirtyContainerState(dirty);
|
|
1786
2025
|
}
|
|
1787
2026
|
}
|
|
1788
2027
|
submitDataStoreOp(id, contents, localOpMetadata = undefined) {
|
|
@@ -1790,21 +2029,20 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1790
2029
|
address: id,
|
|
1791
2030
|
contents,
|
|
1792
2031
|
};
|
|
1793
|
-
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2032
|
+
this.submit({ type: ContainerMessageType.FluidDataStoreOp, contents: envelope }, localOpMetadata);
|
|
1794
2033
|
}
|
|
1795
2034
|
submitDataStoreAliasOp(contents, localOpMetadata) {
|
|
1796
2035
|
const aliasMessage = contents;
|
|
1797
2036
|
if (!isDataStoreAliasMessage(aliasMessage)) {
|
|
1798
2037
|
throw new UsageError("malformedDataStoreAliasMessage");
|
|
1799
2038
|
}
|
|
1800
|
-
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2039
|
+
this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
|
|
1801
2040
|
}
|
|
1802
|
-
async uploadBlob(blob) {
|
|
2041
|
+
async uploadBlob(blob, signal) {
|
|
1803
2042
|
this.verifyNotClosed();
|
|
1804
|
-
return this.blobManager.createBlob(blob);
|
|
2043
|
+
return this.blobManager.createBlob(blob, signal);
|
|
1805
2044
|
}
|
|
1806
2045
|
maybeSubmitIdAllocationOp(type) {
|
|
1807
|
-
var _a, _b;
|
|
1808
2046
|
if (type !== ContainerMessageType.IdAllocation) {
|
|
1809
2047
|
let idAllocationBatchMessage;
|
|
1810
2048
|
let idRange;
|
|
@@ -1812,7 +2050,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1812
2050
|
assert(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
|
|
1813
2051
|
idRange = this.idCompressor.takeNextCreationRange();
|
|
1814
2052
|
// Don't include the idRange if there weren't any Ids allocated
|
|
1815
|
-
idRange =
|
|
2053
|
+
idRange = idRange?.ids !== undefined ? idRange : undefined;
|
|
1816
2054
|
}
|
|
1817
2055
|
if (idRange !== undefined) {
|
|
1818
2056
|
const idAllocationMessage = {
|
|
@@ -1823,7 +2061,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1823
2061
|
contents: JSON.stringify(idAllocationMessage),
|
|
1824
2062
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
1825
2063
|
metadata: undefined,
|
|
1826
|
-
localOpMetadata:
|
|
2064
|
+
localOpMetadata: this.idCompressor?.serialize(true),
|
|
1827
2065
|
type: ContainerMessageType.IdAllocation,
|
|
1828
2066
|
};
|
|
1829
2067
|
}
|
|
@@ -1832,20 +2070,21 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1832
2070
|
}
|
|
1833
2071
|
}
|
|
1834
2072
|
}
|
|
1835
|
-
submit(
|
|
2073
|
+
submit(containerRuntimeMessage, localOpMetadata = undefined, metadata = undefined) {
|
|
1836
2074
|
this.verifyNotClosed();
|
|
1837
2075
|
this.verifyCanSubmitOps();
|
|
1838
2076
|
// There should be no ops in detached container state!
|
|
1839
2077
|
assert(this.attachState !== AttachState.Detached, 0x132 /* "sending ops in detached container" */);
|
|
1840
|
-
const serializedContent = JSON.stringify(
|
|
2078
|
+
const serializedContent = JSON.stringify(containerRuntimeMessage);
|
|
1841
2079
|
// Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
|
|
1842
2080
|
// container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
|
|
1843
2081
|
if (this.innerDeltaManager.readOnlyInfo.readonly) {
|
|
1844
|
-
this.logger.sendTelemetryEvent({
|
|
2082
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1845
2083
|
eventName: "SubmitOpInReadonly",
|
|
1846
2084
|
connected: this.connected,
|
|
1847
2085
|
});
|
|
1848
2086
|
}
|
|
2087
|
+
const type = containerRuntimeMessage.type;
|
|
1849
2088
|
const message = {
|
|
1850
2089
|
contents: serializedContent,
|
|
1851
2090
|
type,
|
|
@@ -1884,6 +2123,10 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1884
2123
|
this.disableAttachReorder !== true) {
|
|
1885
2124
|
this.outbox.submitAttach(message);
|
|
1886
2125
|
}
|
|
2126
|
+
else if (type === ContainerMessageType.BlobAttach) {
|
|
2127
|
+
// BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
|
|
2128
|
+
this.outbox.submitBlobAttach(message);
|
|
2129
|
+
}
|
|
1887
2130
|
else {
|
|
1888
2131
|
this.outbox.submit(message);
|
|
1889
2132
|
}
|
|
@@ -1898,7 +2141,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1898
2141
|
this.closeFn(error);
|
|
1899
2142
|
throw error;
|
|
1900
2143
|
}
|
|
1901
|
-
if (this.isContainerMessageDirtyable(
|
|
2144
|
+
if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
|
|
1902
2145
|
this.updateDocumentDirtyState(true);
|
|
1903
2146
|
}
|
|
1904
2147
|
}
|
|
@@ -1941,9 +2184,9 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1941
2184
|
// System message should not be sent in the middle of the batch.
|
|
1942
2185
|
assert(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
|
|
1943
2186
|
// back-compat: ADO #1385: Make this call unconditional in the future
|
|
1944
|
-
return this.
|
|
1945
|
-
? this.
|
|
1946
|
-
: this.
|
|
2187
|
+
return this.submitSummaryFn !== undefined
|
|
2188
|
+
? this.submitSummaryFn(contents, referenceSequenceNumber)
|
|
2189
|
+
: this.submitFn(MessageType.Summarize, contents, false);
|
|
1947
2190
|
}
|
|
1948
2191
|
/**
|
|
1949
2192
|
* Throw an error if the runtime is closed. Methods that are expected to potentially
|
|
@@ -1960,7 +2203,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1960
2203
|
if (this.opReentryCallsToReport > 0) {
|
|
1961
2204
|
this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" },
|
|
1962
2205
|
// We need to capture the call stack in order to inspect the source of this usage pattern
|
|
1963
|
-
new UsageError(errorMessage));
|
|
2206
|
+
getLongStack(() => new UsageError(errorMessage)));
|
|
1964
2207
|
this.opReentryCallsToReport--;
|
|
1965
2208
|
}
|
|
1966
2209
|
// Creating ops while processing ops can lead
|
|
@@ -1980,34 +2223,43 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
1980
2223
|
}
|
|
1981
2224
|
}
|
|
1982
2225
|
}
|
|
1983
|
-
|
|
2226
|
+
reSubmitBatch(batch) {
|
|
2227
|
+
this.orderSequentially(() => {
|
|
2228
|
+
for (const message of batch) {
|
|
2229
|
+
this.reSubmit(message);
|
|
2230
|
+
}
|
|
2231
|
+
});
|
|
2232
|
+
this.flush();
|
|
2233
|
+
}
|
|
2234
|
+
reSubmit(message) {
|
|
1984
2235
|
// Need to parse from string for back-compat
|
|
1985
|
-
const
|
|
1986
|
-
this.reSubmitCore(
|
|
2236
|
+
const containerRuntimeMessage = this.parseOpContent(message.content);
|
|
2237
|
+
this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
|
|
1987
2238
|
}
|
|
1988
2239
|
/**
|
|
1989
2240
|
* Finds the right store and asks it to resubmit the message. This typically happens when we
|
|
1990
2241
|
* reconnect and there are pending messages.
|
|
1991
|
-
* @param
|
|
2242
|
+
* @param message - The original ContainerRuntimeMessage.
|
|
1992
2243
|
* @param localOpMetadata - The local metadata associated with the original message.
|
|
1993
2244
|
*/
|
|
1994
|
-
reSubmitCore(
|
|
1995
|
-
|
|
2245
|
+
reSubmitCore(message, localOpMetadata, opMetadata) {
|
|
2246
|
+
const contents = message.contents;
|
|
2247
|
+
switch (message.type) {
|
|
1996
2248
|
case ContainerMessageType.FluidDataStoreOp:
|
|
1997
2249
|
// For Operations, call resubmitDataStoreOp which will find the right store
|
|
1998
2250
|
// and trigger resubmission on it.
|
|
1999
|
-
this.dataStores.resubmitDataStoreOp(
|
|
2251
|
+
this.dataStores.resubmitDataStoreOp(contents, localOpMetadata);
|
|
2000
2252
|
break;
|
|
2001
2253
|
case ContainerMessageType.Attach:
|
|
2002
2254
|
case ContainerMessageType.Alias:
|
|
2003
|
-
this.submit(
|
|
2255
|
+
this.submit(message, localOpMetadata);
|
|
2004
2256
|
break;
|
|
2005
2257
|
case ContainerMessageType.IdAllocation:
|
|
2006
2258
|
// Remove the stashedState from the op if it's a stashed op
|
|
2007
|
-
if (
|
|
2008
|
-
delete
|
|
2259
|
+
if (contents.stashedState !== undefined) {
|
|
2260
|
+
delete contents.stashedState;
|
|
2009
2261
|
}
|
|
2010
|
-
this.submit(
|
|
2262
|
+
this.submit(message, localOpMetadata);
|
|
2011
2263
|
break;
|
|
2012
2264
|
case ContainerMessageType.ChunkedOp:
|
|
2013
2265
|
throw new Error(`chunkedOp not expected here`);
|
|
@@ -2015,10 +2267,29 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2015
2267
|
this.blobManager.reSubmit(opMetadata);
|
|
2016
2268
|
break;
|
|
2017
2269
|
case ContainerMessageType.Rejoin:
|
|
2018
|
-
this.submit(
|
|
2270
|
+
this.submit(message);
|
|
2019
2271
|
break;
|
|
2020
|
-
default:
|
|
2021
|
-
|
|
2272
|
+
default: {
|
|
2273
|
+
// This case should be very rare - it would imply an op was stashed from a
|
|
2274
|
+
// future version of runtime code and now is being applied on an older version
|
|
2275
|
+
const compatBehavior = message.compatDetails?.behavior;
|
|
2276
|
+
if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
|
|
2277
|
+
this.logger.sendTelemetryEvent({
|
|
2278
|
+
eventName: "resubmitUnrecognizedMessageTypeAllowed",
|
|
2279
|
+
messageDetails: { type: message.type, compatBehavior },
|
|
2280
|
+
});
|
|
2281
|
+
}
|
|
2282
|
+
else {
|
|
2283
|
+
const error = DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
|
|
2284
|
+
messageDetails: JSON.stringify({
|
|
2285
|
+
type: message.type,
|
|
2286
|
+
compatBehavior,
|
|
2287
|
+
}),
|
|
2288
|
+
});
|
|
2289
|
+
this.closeFn(error);
|
|
2290
|
+
throw error;
|
|
2291
|
+
}
|
|
2292
|
+
}
|
|
2022
2293
|
}
|
|
2023
2294
|
}
|
|
2024
2295
|
rollback(content, localOpMetadata) {
|
|
@@ -2031,6 +2302,7 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2031
2302
|
this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
|
|
2032
2303
|
break;
|
|
2033
2304
|
default:
|
|
2305
|
+
// Don't check message.compatDetails because this is for rolling back a local op so the type will be known
|
|
2034
2306
|
throw new Error(`Can't rollback ${type}`);
|
|
2035
2307
|
}
|
|
2036
2308
|
}
|
|
@@ -2048,29 +2320,22 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2048
2320
|
/** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
|
|
2049
2321
|
async refreshLatestSummaryAck(options) {
|
|
2050
2322
|
const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
|
|
2323
|
+
// proposalHandle is always passed from RunningSummarizer.
|
|
2324
|
+
assert(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
|
|
2051
2325
|
const readAndParseBlob = async (id) => readAndParse(this.storage, id);
|
|
2052
|
-
|
|
2053
|
-
|
|
2054
|
-
|
|
2055
|
-
|
|
2056
|
-
|
|
2326
|
+
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
|
|
2327
|
+
/**
|
|
2328
|
+
* When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
|
|
2329
|
+
* current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
|
|
2330
|
+
* it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
|
|
2331
|
+
* and then close as the current main client is likely to be re-elected as the parent summarizer again.
|
|
2332
|
+
*/
|
|
2333
|
+
if (!result.isSummaryTracked && result.isSummaryNewer) {
|
|
2334
|
+
const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2057
2335
|
eventName: "RefreshLatestSummaryAckFetch",
|
|
2058
2336
|
ackHandle,
|
|
2059
2337
|
targetSequenceNumber: summaryRefSeq,
|
|
2060
|
-
}, readAndParseBlob);
|
|
2061
|
-
/**
|
|
2062
|
-
* back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
|
|
2063
|
-
* getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
|
|
2064
|
-
* these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
|
|
2065
|
-
* change that started fetching latest snapshot always.
|
|
2066
|
-
*/
|
|
2067
|
-
if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
|
|
2068
|
-
fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2069
|
-
eventName: "RefreshLatestSummaryAckFetchBackCompat",
|
|
2070
|
-
ackHandle,
|
|
2071
|
-
targetSequenceNumber: summaryRefSeq,
|
|
2072
|
-
}, readAndParseBlob, ackHandle);
|
|
2073
|
-
}
|
|
2338
|
+
}, readAndParseBlob, null);
|
|
2074
2339
|
/**
|
|
2075
2340
|
* If the fetched snapshot is older than the one for which the ack was received, close the container.
|
|
2076
2341
|
* This should never happen because an ack should be sent after the latest summary is updated in the server.
|
|
@@ -2087,20 +2352,14 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2087
2352
|
summaryRefSeq,
|
|
2088
2353
|
fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2089
2354
|
});
|
|
2090
|
-
this.
|
|
2355
|
+
this.disposeFn(error);
|
|
2091
2356
|
throw error;
|
|
2092
2357
|
}
|
|
2093
|
-
|
|
2094
|
-
|
|
2095
|
-
|
|
2096
|
-
return {
|
|
2097
|
-
snapshotTree: fetchResult.snapshotTree,
|
|
2098
|
-
snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2099
|
-
};
|
|
2100
|
-
};
|
|
2101
|
-
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2358
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
|
|
2359
|
+
return;
|
|
2360
|
+
}
|
|
2102
2361
|
// Notify the garbage collector so it can update its latest summary state.
|
|
2103
|
-
await this.garbageCollector.refreshLatestSummary(
|
|
2362
|
+
await this.garbageCollector.refreshLatestSummary(result);
|
|
2104
2363
|
}
|
|
2105
2364
|
/**
|
|
2106
2365
|
* Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
|
|
@@ -2110,24 +2369,31 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2110
2369
|
*/
|
|
2111
2370
|
async refreshLatestSummaryAckFromServer(summaryLogger) {
|
|
2112
2371
|
const readAndParseBlob = async (id) => readAndParse(this.storage, id);
|
|
2113
|
-
const {
|
|
2372
|
+
const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2114
2373
|
eventName: "RefreshLatestSummaryFromServerFetch",
|
|
2115
|
-
}, readAndParseBlob);
|
|
2116
|
-
|
|
2117
|
-
snapshotTree,
|
|
2118
|
-
snapshotRefSeq: latestSnapshotRefSeq,
|
|
2119
|
-
};
|
|
2120
|
-
const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2121
|
-
// Notify the garbage collector so it can update its latest summary state.
|
|
2122
|
-
await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
|
|
2374
|
+
}, readAndParseBlob, null);
|
|
2375
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
|
|
2123
2376
|
return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
|
|
2124
2377
|
}
|
|
2125
|
-
async
|
|
2126
|
-
|
|
2378
|
+
async closeStaleSummarizer(codePath) {
|
|
2379
|
+
this.mc.logger.sendTelemetryEvent({
|
|
2380
|
+
eventName: "ClosingSummarizerOnSummaryStale",
|
|
2381
|
+
codePath,
|
|
2382
|
+
message: "Stopping fetch from storage",
|
|
2383
|
+
closeSummarizerDelayMs: this.closeSummarizerDelayMs,
|
|
2384
|
+
}, new GenericError("Restarting summarizer instead of refreshing"));
|
|
2385
|
+
// Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2386
|
+
await delay(this.closeSummarizerDelayMs);
|
|
2387
|
+
this._summarizer?.stop("latestSummaryStateStale");
|
|
2388
|
+
this.disposeFn();
|
|
2127
2389
|
}
|
|
2390
|
+
/**
|
|
2391
|
+
* Downloads snapshot from storage with the given versionId or latest if versionId is null.
|
|
2392
|
+
* By default, it also closes the container after downloading the snapshot. However, this may be
|
|
2393
|
+
* overridden via options.
|
|
2394
|
+
*/
|
|
2128
2395
|
async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
|
|
2129
|
-
|
|
2130
|
-
const snapshotResults = await PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2396
|
+
return PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2131
2397
|
const stats = {};
|
|
2132
2398
|
const trace = Trace.start();
|
|
2133
2399
|
const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? FetchSource.noCache : undefined);
|
|
@@ -2146,33 +2412,65 @@ export class ContainerRuntime extends TypedEventEmitter {
|
|
|
2146
2412
|
latestSnapshotRefSeq,
|
|
2147
2413
|
};
|
|
2148
2414
|
});
|
|
2149
|
-
// We choose to close the summarizer after the snapshot cache is updated to avoid
|
|
2150
|
-
// situations which the main client (which is likely to be re-elected as the leader again)
|
|
2151
|
-
// loads the summarizer from cache.
|
|
2152
|
-
if (this.summaryStateUpdateMethod === "restart") {
|
|
2153
|
-
const error = new GenericError("Restarting summarizer instead of refreshing");
|
|
2154
|
-
this.mc.logger.sendTelemetryEvent(Object.assign(Object.assign({}, event), { eventName: "ClosingSummarizerOnSummaryStale", codePath: event.eventName, message: "Stopping fetch from storage", versionId: versionId != null ? versionId : undefined, closeSummarizerDelayMs: this.closeSummarizerDelayMs }), error);
|
|
2155
|
-
// Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2156
|
-
await delay(this.closeSummarizerDelayMs);
|
|
2157
|
-
(_a = this._summarizer) === null || _a === void 0 ? void 0 : _a.stop("latestSummaryStateStale");
|
|
2158
|
-
this.closeFn();
|
|
2159
|
-
throw error;
|
|
2160
|
-
}
|
|
2161
|
-
return snapshotResults;
|
|
2162
2415
|
}
|
|
2163
2416
|
notifyAttaching() { } // do nothing (deprecated method)
|
|
2164
|
-
getPendingLocalState() {
|
|
2165
|
-
|
|
2166
|
-
|
|
2417
|
+
async getPendingLocalState(props) {
|
|
2418
|
+
return PerformanceEvent.timedExecAsync(this.mc.logger, {
|
|
2419
|
+
eventName: "getPendingLocalState",
|
|
2420
|
+
notifyImminentClosure: props?.notifyImminentClosure,
|
|
2421
|
+
}, async (event) => {
|
|
2422
|
+
this.verifyNotClosed();
|
|
2423
|
+
const waitBlobsToAttach = props?.notifyImminentClosure;
|
|
2424
|
+
if (this._orderSequentiallyCalls !== 0) {
|
|
2425
|
+
throw new UsageError("can't get state during orderSequentially");
|
|
2426
|
+
}
|
|
2427
|
+
const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
|
|
2428
|
+
const pending = this.pendingStateManager.getLocalState();
|
|
2429
|
+
if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
|
|
2430
|
+
return; // no pending state to save
|
|
2431
|
+
}
|
|
2432
|
+
// Flush pending batch.
|
|
2433
|
+
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2434
|
+
// to close current batch.
|
|
2435
|
+
this.flush();
|
|
2436
|
+
const pendingState = {
|
|
2437
|
+
pending,
|
|
2438
|
+
pendingAttachmentBlobs,
|
|
2439
|
+
};
|
|
2440
|
+
event.end({
|
|
2441
|
+
attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
|
|
2442
|
+
pendingOpsSize: pending?.pendingStates.length,
|
|
2443
|
+
});
|
|
2444
|
+
return pendingState;
|
|
2445
|
+
});
|
|
2446
|
+
}
|
|
2447
|
+
summarizeOnDemand(options) {
|
|
2448
|
+
if (this.isSummarizerClient) {
|
|
2449
|
+
return this.summarizer.summarizeOnDemand(options);
|
|
2450
|
+
}
|
|
2451
|
+
else if (this.summaryManager !== undefined) {
|
|
2452
|
+
return this.summaryManager.summarizeOnDemand(options);
|
|
2453
|
+
}
|
|
2454
|
+
else {
|
|
2455
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2456
|
+
// disableSummaries is turned on. We are throwing instead of returning a failure here,
|
|
2457
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2458
|
+
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2459
|
+
}
|
|
2460
|
+
}
|
|
2461
|
+
enqueueSummarize(options) {
|
|
2462
|
+
if (this.isSummarizerClient) {
|
|
2463
|
+
return this.summarizer.enqueueSummarize(options);
|
|
2464
|
+
}
|
|
2465
|
+
else if (this.summaryManager !== undefined) {
|
|
2466
|
+
return this.summaryManager.enqueueSummarize(options);
|
|
2467
|
+
}
|
|
2468
|
+
else {
|
|
2469
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2470
|
+
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
2471
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2472
|
+
throw new UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2167
2473
|
}
|
|
2168
|
-
// Flush pending batch.
|
|
2169
|
-
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2170
|
-
// to close current batch.
|
|
2171
|
-
this.flush();
|
|
2172
|
-
return {
|
|
2173
|
-
pending: this.pendingStateManager.getLocalState(),
|
|
2174
|
-
pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
|
|
2175
|
-
};
|
|
2176
2474
|
}
|
|
2177
2475
|
/**
|
|
2178
2476
|
* * Forms a function that will request a Summarizer.
|