@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +4 -3
- package/dist/batchTracker.d.ts +3 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +6 -5
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +15 -18
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +212 -171
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +33 -17
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +172 -35
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +722 -425
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +15 -7
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +87 -90
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +10 -10
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.js +2 -2
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +23 -7
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +125 -82
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerProxyBase.d.ts +35 -0
- package/dist/deltaManagerProxyBase.d.ts.map +1 -0
- package/dist/deltaManagerProxyBase.js +77 -0
- package/dist/deltaManagerProxyBase.js.map +1 -0
- package/dist/deltaManagerSummarizerProxy.d.ts +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/dist/deltaManagerSummarizerProxy.js +4 -2
- package/dist/deltaManagerSummarizerProxy.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +10 -10
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/error.d.ts +14 -0
- package/dist/error.d.ts.map +1 -0
- package/dist/error.js +21 -0
- package/dist/error.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +10 -9
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +65 -56
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +18 -14
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +17 -5
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +14 -15
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +0 -8
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +11 -24
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +4 -7
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +19 -58
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +1 -1
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +45 -35
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +4 -4
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +3 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.js +26 -68
- package/dist/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/dist/id-compressor/finalSpace.d.ts +29 -0
- package/dist/id-compressor/finalSpace.d.ts.map +1 -0
- package/dist/id-compressor/finalSpace.js +62 -0
- package/dist/id-compressor/finalSpace.js.map +1 -0
- package/dist/id-compressor/idCompressor.d.ts +25 -250
- package/dist/id-compressor/idCompressor.d.ts.map +1 -1
- package/dist/id-compressor/idCompressor.js +390 -1153
- package/dist/id-compressor/idCompressor.js.map +1 -1
- package/dist/id-compressor/identifiers.d.ts +32 -0
- package/dist/id-compressor/identifiers.d.ts.map +1 -0
- package/dist/id-compressor/identifiers.js +15 -0
- package/dist/id-compressor/identifiers.js.map +1 -0
- package/dist/id-compressor/index.d.ts +5 -6
- package/dist/id-compressor/index.d.ts.map +1 -1
- package/dist/id-compressor/index.js +20 -26
- package/dist/id-compressor/index.js.map +1 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +22 -0
- package/dist/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/dist/id-compressor/persistanceUtilities.js +43 -0
- package/dist/id-compressor/persistanceUtilities.js.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js +80 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/dist/id-compressor/sessions.d.ts +115 -0
- package/dist/id-compressor/sessions.d.ts.map +1 -0
- package/dist/id-compressor/sessions.js +305 -0
- package/dist/id-compressor/sessions.js.map +1 -0
- package/dist/id-compressor/utilities.d.ts +49 -0
- package/dist/id-compressor/utilities.d.ts.map +1 -0
- package/dist/id-compressor/utilities.js +166 -0
- package/dist/id-compressor/utilities.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/metadata.d.ts +18 -0
- package/dist/metadata.d.ts.map +1 -0
- package/dist/metadata.js +7 -0
- package/dist/metadata.js.map +1 -0
- package/dist/opLifecycle/batchManager.d.ts +2 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +15 -7
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +11 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +2 -2
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +12 -7
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -2
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +30 -21
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +19 -13
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +2 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +24 -19
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +39 -6
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +138 -61
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +22 -8
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.js +1 -2
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +25 -10
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +101 -64
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +43 -33
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summary/index.d.ts +4 -4
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +3 -1
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +3 -3
- package/dist/summary/orderedClientElection.d.ts.map +1 -1
- package/dist/summary/orderedClientElection.js +26 -27
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +3 -3
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/runningSummarizer.d.ts +31 -10
- package/dist/summary/runningSummarizer.d.ts.map +1 -1
- package/dist/summary/runningSummarizer.js +271 -139
- package/dist/summary/runningSummarizer.js.map +1 -1
- package/dist/summary/summarizer.d.ts +8 -7
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +79 -78
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerClientElection.d.ts +2 -2
- package/dist/summary/summarizerClientElection.d.ts.map +1 -1
- package/dist/summary/summarizerClientElection.js +7 -11
- package/dist/summary/summarizerClientElection.js.map +1 -1
- package/dist/summary/summarizerHeuristics.js +10 -14
- package/dist/summary/summarizerHeuristics.js.map +1 -1
- package/dist/summary/summarizerNode/index.d.ts +1 -1
- package/dist/summary/summarizerNode/index.d.ts.map +1 -1
- package/dist/summary/summarizerNode/index.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.js +144 -149
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +74 -123
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +44 -24
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +2 -2
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +16 -13
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +4 -0
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +8 -5
- package/dist/summary/summaryFormat.js.map +1 -1
- package/dist/summary/summaryGenerator.d.ts +21 -6
- package/dist/summary/summaryGenerator.d.ts.map +1 -1
- package/dist/summary/summaryGenerator.js +117 -54
- package/dist/summary/summaryGenerator.js.map +1 -1
- package/dist/summary/summaryManager.d.ts +8 -7
- package/dist/summary/summaryManager.d.ts.map +1 -1
- package/dist/summary/summaryManager.js +38 -28
- package/dist/summary/summaryManager.js.map +1 -1
- package/lib/batchTracker.d.ts +3 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +5 -4
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +15 -18
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +187 -146
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +23 -7
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +172 -35
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +678 -380
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +13 -5
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +49 -52
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +3 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +23 -7
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +107 -64
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerProxyBase.d.ts +35 -0
- package/lib/deltaManagerProxyBase.d.ts.map +1 -0
- package/lib/deltaManagerProxyBase.js +73 -0
- package/lib/deltaManagerProxyBase.js.map +1 -0
- package/lib/deltaManagerSummarizerProxy.d.ts +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/lib/deltaManagerSummarizerProxy.js +3 -1
- package/lib/deltaManagerSummarizerProxy.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +7 -7
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/error.d.ts +14 -0
- package/lib/error.d.ts.map +1 -0
- package/lib/error.js +17 -0
- package/lib/error.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +10 -9
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +61 -52
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +16 -12
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +17 -5
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +13 -14
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +0 -8
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +5 -17
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +4 -7
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +20 -59
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +1 -1
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +46 -36
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -2
- package/lib/gc/index.js.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.js +25 -66
- package/lib/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/lib/id-compressor/finalSpace.d.ts +29 -0
- package/lib/id-compressor/finalSpace.d.ts.map +1 -0
- package/lib/id-compressor/finalSpace.js +58 -0
- package/lib/id-compressor/finalSpace.js.map +1 -0
- package/lib/id-compressor/idCompressor.d.ts +25 -250
- package/lib/id-compressor/idCompressor.d.ts.map +1 -1
- package/lib/id-compressor/idCompressor.js +385 -1142
- package/lib/id-compressor/idCompressor.js.map +1 -1
- package/lib/id-compressor/identifiers.d.ts +32 -0
- package/lib/id-compressor/identifiers.d.ts.map +1 -0
- package/lib/id-compressor/identifiers.js +11 -0
- package/lib/id-compressor/identifiers.js.map +1 -0
- package/lib/id-compressor/index.d.ts +5 -6
- package/lib/id-compressor/index.d.ts.map +1 -1
- package/lib/id-compressor/index.js +5 -6
- package/lib/id-compressor/index.js.map +1 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +22 -0
- package/lib/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/lib/id-compressor/persistanceUtilities.js +34 -0
- package/lib/id-compressor/persistanceUtilities.js.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js +76 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/lib/id-compressor/sessions.d.ts +115 -0
- package/lib/id-compressor/sessions.d.ts.map +1 -0
- package/lib/id-compressor/sessions.js +290 -0
- package/lib/id-compressor/sessions.js.map +1 -0
- package/lib/id-compressor/utilities.d.ts +49 -0
- package/lib/id-compressor/utilities.d.ts.map +1 -0
- package/lib/id-compressor/utilities.js +148 -0
- package/lib/id-compressor/utilities.js.map +1 -0
- package/lib/index.d.ts +3 -3
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -2
- package/lib/index.js.map +1 -1
- package/lib/metadata.d.ts +18 -0
- package/lib/metadata.d.ts.map +1 -0
- package/lib/metadata.js +6 -0
- package/lib/metadata.js.map +1 -0
- package/lib/opLifecycle/batchManager.d.ts +2 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +15 -7
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +11 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +2 -2
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +10 -5
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -2
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +22 -13
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +17 -11
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +2 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +15 -10
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +39 -6
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +132 -56
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +23 -9
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.js +1 -2
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +25 -10
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +90 -53
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +25 -15
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summary/index.d.ts +4 -4
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +2 -2
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +3 -3
- package/lib/summary/orderedClientElection.d.ts.map +1 -1
- package/lib/summary/orderedClientElection.js +21 -22
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/runningSummarizer.d.ts +31 -10
- package/lib/summary/runningSummarizer.d.ts.map +1 -1
- package/lib/summary/runningSummarizer.js +265 -133
- package/lib/summary/runningSummarizer.js.map +1 -1
- package/lib/summary/summarizer.d.ts +8 -7
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +75 -74
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerClientElection.d.ts +2 -2
- package/lib/summary/summarizerClientElection.d.ts.map +1 -1
- package/lib/summary/summarizerClientElection.js +6 -10
- package/lib/summary/summarizerClientElection.js.map +1 -1
- package/lib/summary/summarizerHeuristics.js +9 -13
- package/lib/summary/summarizerHeuristics.js.map +1 -1
- package/lib/summary/summarizerNode/index.d.ts +1 -1
- package/lib/summary/summarizerNode/index.d.ts.map +1 -1
- package/lib/summary/summarizerNode/index.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.js +132 -137
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +70 -119
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +44 -24
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +2 -2
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +9 -6
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +4 -0
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +7 -4
- package/lib/summary/summaryFormat.js.map +1 -1
- package/lib/summary/summaryGenerator.d.ts +21 -6
- package/lib/summary/summaryGenerator.d.ts.map +1 -1
- package/lib/summary/summaryGenerator.js +109 -47
- package/lib/summary/summaryGenerator.js.map +1 -1
- package/lib/summary/summaryManager.d.ts +8 -7
- package/lib/summary/summaryManager.d.ts.map +1 -1
- package/lib/summary/summaryManager.js +35 -25
- package/lib/summary/summaryManager.js.map +1 -1
- package/package.json +30 -32
- package/src/batchTracker.ts +7 -5
- package/src/blobManager.ts +235 -172
- package/src/connectionTelemetry.ts +19 -5
- package/src/containerRuntime.ts +853 -431
- package/src/dataStore.ts +12 -4
- package/src/dataStoreContext.ts +49 -46
- package/src/dataStoreContexts.ts +4 -4
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +119 -80
- package/src/deltaManagerProxyBase.ts +111 -0
- package/src/deltaManagerSummarizerProxy.ts +4 -1
- package/src/deltaScheduler.ts +7 -11
- package/src/error.ts +18 -0
- package/src/gc/garbageCollection.md +53 -5
- package/src/gc/garbageCollection.ts +58 -51
- package/src/gc/gcConfigs.ts +4 -2
- package/src/gc/gcDefinitions.ts +17 -21
- package/src/gc/gcEarlyAdoption.md +145 -0
- package/src/gc/gcHelpers.ts +1 -12
- package/src/gc/gcSummaryStateTracker.ts +19 -65
- package/src/gc/gcTelemetry.ts +15 -13
- package/src/gc/gcUnreferencedStateTracker.ts +1 -1
- package/src/gc/index.ts +2 -4
- package/src/id-compressor/appendOnlySortedMap.ts +26 -87
- package/src/id-compressor/finalSpace.ts +67 -0
- package/src/id-compressor/idCompressor.ts +458 -1682
- package/src/id-compressor/identifiers.ts +42 -0
- package/src/id-compressor/index.ts +11 -20
- package/src/id-compressor/persistanceUtilities.ts +58 -0
- package/src/id-compressor/sessionSpaceNormalizer.ts +83 -0
- package/src/id-compressor/sessions.ts +405 -0
- package/src/id-compressor/utilities.ts +187 -0
- package/src/index.ts +9 -2
- package/src/metadata.ts +19 -0
- package/src/opLifecycle/README.md +20 -0
- package/src/opLifecycle/batchManager.ts +9 -1
- package/src/opLifecycle/definitions.ts +11 -0
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +6 -5
- package/src/opLifecycle/opDecompressor.ts +47 -17
- package/src/opLifecycle/opGroupingManager.ts +18 -8
- package/src/opLifecycle/opSplitter.ts +10 -7
- package/src/opLifecycle/outbox.ts +177 -72
- package/src/opLifecycle/remoteMessageProcessor.ts +32 -9
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +123 -78
- package/src/scheduleManager.ts +22 -11
- package/src/summary/index.ts +7 -4
- package/src/summary/orderedClientElection.ts +10 -6
- package/src/summary/runWhileConnectedCoordinator.ts +1 -1
- package/src/summary/runningSummarizer.ts +291 -163
- package/src/summary/summarizer.ts +27 -16
- package/src/summary/summarizerClientElection.ts +2 -2
- package/src/summary/summarizerHeuristics.ts +1 -1
- package/src/summary/summarizerNode/index.ts +2 -2
- package/src/summary/summarizerNode/summarizerNode.ts +142 -184
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +27 -35
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +72 -148
- package/src/summary/summarizerTypes.ts +49 -24
- package/src/summary/summaryCollection.ts +9 -4
- package/src/summary/summaryFormat.ts +9 -2
- package/src/summary/summaryGenerator.ts +72 -49
- package/src/summary/summaryManager.ts +44 -16
- package/dist/id-compressor/idRange.d.ts +0 -11
- package/dist/id-compressor/idRange.d.ts.map +0 -1
- package/dist/id-compressor/idRange.js +0 -29
- package/dist/id-compressor/idRange.js.map +0 -1
- package/dist/id-compressor/numericUuid.d.ts +0 -59
- package/dist/id-compressor/numericUuid.d.ts.map +0 -1
- package/dist/id-compressor/numericUuid.js +0 -325
- package/dist/id-compressor/numericUuid.js.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/dist/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.js +0 -488
- package/dist/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/dist/id-compressor/utils.d.ts +0 -57
- package/dist/id-compressor/utils.d.ts.map +0 -1
- package/dist/id-compressor/utils.js +0 -90
- package/dist/id-compressor/utils.js.map +0 -1
- package/dist/id-compressor/uuidUtilities.d.ts +0 -30
- package/dist/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/dist/id-compressor/uuidUtilities.js +0 -106
- package/dist/id-compressor/uuidUtilities.js.map +0 -1
- package/lib/id-compressor/idRange.d.ts +0 -11
- package/lib/id-compressor/idRange.d.ts.map +0 -1
- package/lib/id-compressor/idRange.js +0 -25
- package/lib/id-compressor/idRange.js.map +0 -1
- package/lib/id-compressor/numericUuid.d.ts +0 -59
- package/lib/id-compressor/numericUuid.d.ts.map +0 -1
- package/lib/id-compressor/numericUuid.js +0 -315
- package/lib/id-compressor/numericUuid.js.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/lib/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.js +0 -484
- package/lib/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/lib/id-compressor/utils.d.ts +0 -57
- package/lib/id-compressor/utils.d.ts.map +0 -1
- package/lib/id-compressor/utils.js +0 -79
- package/lib/id-compressor/utils.js.map +0 -1
- package/lib/id-compressor/uuidUtilities.d.ts +0 -30
- package/lib/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/lib/id-compressor/uuidUtilities.js +0 -98
- package/lib/id-compressor/uuidUtilities.js.map +0 -1
- package/src/id-compressor/idRange.ts +0 -35
- package/src/id-compressor/numericUuid.ts +0 -383
- package/src/id-compressor/sessionIdNormalizer.ts +0 -609
- package/src/id-compressor/utils.ts +0 -114
- package/src/id-compressor/uuidUtilities.ts +0 -123
package/dist/containerRuntime.js
CHANGED
|
@@ -19,13 +19,13 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
19
19
|
return result;
|
|
20
20
|
};
|
|
21
21
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
22
|
-
exports.ContainerRuntime = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.TombstoneResponseHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
|
|
22
|
+
exports.ContainerRuntime = exports.makeLegacySendBatchFn = exports.getDeviceSpec = exports.agentSchedulerId = exports.isRuntimeMessage = exports.RuntimeMessage = exports.defaultPendingOpsRetryDelayMs = exports.defaultPendingOpsWaitTimeoutMs = exports.CompressionAlgorithms = exports.defaultRuntimeHeaderData = exports.InactiveResponseHeaderKey = exports.TombstoneResponseHeaderKey = exports.AllowInactiveRequestHeaderKey = exports.AllowTombstoneRequestHeaderKey = exports.RuntimeHeaders = exports.DefaultSummaryConfiguration = exports.ContainerMessageType = void 0;
|
|
23
23
|
const container_definitions_1 = require("@fluidframework/container-definitions");
|
|
24
|
-
const
|
|
24
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
25
|
+
const client_utils_1 = require("@fluid-internal/client-utils");
|
|
25
26
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
26
27
|
const driver_definitions_1 = require("@fluidframework/driver-definitions");
|
|
27
28
|
const driver_utils_1 = require("@fluidframework/driver-utils");
|
|
28
|
-
const container_utils_1 = require("@fluidframework/container-utils");
|
|
29
29
|
const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
30
30
|
const runtime_definitions_1 = require("@fluidframework/runtime-definitions");
|
|
31
31
|
const runtime_utils_1 = require("@fluidframework/runtime-utils");
|
|
@@ -66,6 +66,18 @@ var ContainerMessageType;
|
|
|
66
66
|
*/
|
|
67
67
|
ContainerMessageType["IdAllocation"] = "idAllocation";
|
|
68
68
|
})(ContainerMessageType = exports.ContainerMessageType || (exports.ContainerMessageType = {}));
|
|
69
|
+
/**
|
|
70
|
+
* Utility to implement compat behaviors given an unknown message type
|
|
71
|
+
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|
|
72
|
+
*
|
|
73
|
+
* @param _unknownContainerRuntimeMessageType - Typed as never, to ensure all known types have been
|
|
74
|
+
* handled before calling this function (e.g. in a switch statement).
|
|
75
|
+
* @param compatBehavior - Typed redundantly with CompatModeBehavior to ensure handling is added when updating that type
|
|
76
|
+
*/
|
|
77
|
+
function compatBehaviorAllowsMessageType(_unknownContainerRuntimeMessageType, compatBehavior) {
|
|
78
|
+
// undefined defaults to same behavior as "FailToProcess"
|
|
79
|
+
return compatBehavior === "Ignore";
|
|
80
|
+
}
|
|
69
81
|
exports.DefaultSummaryConfiguration = {
|
|
70
82
|
state: "enabled",
|
|
71
83
|
minIdleTime: 0,
|
|
@@ -92,8 +104,12 @@ var RuntimeHeaders;
|
|
|
92
104
|
})(RuntimeHeaders = exports.RuntimeHeaders || (exports.RuntimeHeaders = {}));
|
|
93
105
|
/** True if a tombstoned object should be returned without erroring */
|
|
94
106
|
exports.AllowTombstoneRequestHeaderKey = "allowTombstone"; // Belongs in the enum above, but avoiding the breaking change
|
|
107
|
+
/** [IRRELEVANT IF throwOnInactiveLoad OPTION NOT SET] True if an inactive object should be returned without erroring */
|
|
108
|
+
exports.AllowInactiveRequestHeaderKey = "allowInactive"; // Belongs in the enum above, but avoiding the breaking change
|
|
95
109
|
/** Tombstone error responses will have this header set to true */
|
|
96
110
|
exports.TombstoneResponseHeaderKey = "isTombstoned";
|
|
111
|
+
/** Inactive error responses will have this header set to true */
|
|
112
|
+
exports.InactiveResponseHeaderKey = "isInactive";
|
|
97
113
|
/** Default values for Runtime Headers */
|
|
98
114
|
exports.defaultRuntimeHeaderData = {
|
|
99
115
|
wait: true,
|
|
@@ -120,12 +136,16 @@ const defaultCompressionConfig = {
|
|
|
120
136
|
compressionAlgorithm: CompressionAlgorithms.lz4,
|
|
121
137
|
};
|
|
122
138
|
const defaultChunkSizeInBytes = 204800;
|
|
139
|
+
/** The default time to wait for pending ops to be processed during summarization */
|
|
140
|
+
exports.defaultPendingOpsWaitTimeoutMs = 1000;
|
|
141
|
+
/** The default time to delay a summarization retry attempt when there are pending ops */
|
|
142
|
+
exports.defaultPendingOpsRetryDelayMs = 1000;
|
|
123
143
|
/**
|
|
124
144
|
* Instead of refreshing from latest because we do not have 100% confidence in the state
|
|
125
145
|
* of the current system, we should close the summarizer and let it recover.
|
|
126
146
|
* This delay's goal is to prevent tight restart loops
|
|
127
147
|
*/
|
|
128
|
-
const defaultCloseSummarizerDelayMs =
|
|
148
|
+
const defaultCloseSummarizerDelayMs = 5000; // 5 seconds
|
|
129
149
|
/**
|
|
130
150
|
* @deprecated - use ContainerRuntimeMessage instead
|
|
131
151
|
*/
|
|
@@ -162,23 +182,40 @@ function getDeviceSpec() {
|
|
|
162
182
|
};
|
|
163
183
|
}
|
|
164
184
|
}
|
|
165
|
-
catch
|
|
185
|
+
catch { }
|
|
166
186
|
return {};
|
|
167
187
|
}
|
|
168
188
|
exports.getDeviceSpec = getDeviceSpec;
|
|
189
|
+
/**
|
|
190
|
+
* Older loader doesn't have a submitBatchFn member, this is the older way of submitting a batch.
|
|
191
|
+
* Rather than exposing the submitFn (now deprecated) and IDeltaManager (dangerous to hand out) to the Outbox,
|
|
192
|
+
* we can provide a partially-applied function to keep those items private to the ContainerRuntime.
|
|
193
|
+
*/
|
|
194
|
+
const makeLegacySendBatchFn = (submitFn, deltaManager) => (batch) => {
|
|
195
|
+
for (const message of batch.content) {
|
|
196
|
+
submitFn(protocol_definitions_1.MessageType.Operation,
|
|
197
|
+
// For back-compat (submitFn only works on deserialized content)
|
|
198
|
+
message.contents === undefined ? undefined : JSON.parse(message.contents), true, // batch
|
|
199
|
+
message.metadata);
|
|
200
|
+
}
|
|
201
|
+
deltaManager.flush();
|
|
202
|
+
};
|
|
203
|
+
exports.makeLegacySendBatchFn = makeLegacySendBatchFn;
|
|
169
204
|
/**
|
|
170
205
|
* Represents the runtime of the container. Contains helper functions/state of the container.
|
|
171
206
|
* It will define the store level mappings.
|
|
172
207
|
*/
|
|
173
|
-
class ContainerRuntime extends
|
|
208
|
+
class ContainerRuntime extends client_utils_1.TypedEventEmitter {
|
|
174
209
|
/**
|
|
175
210
|
* @internal
|
|
176
211
|
*/
|
|
177
|
-
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration
|
|
178
|
-
|
|
179
|
-
|
|
212
|
+
constructor(context, registry, metadata, electedSummarizerData, chunks, dataStoreAliasMap, runtimeOptions, containerScope, logger, existing, blobManagerSnapshot, _storage, idCompressor, requestHandler, summaryConfiguration = {
|
|
213
|
+
// the defaults
|
|
214
|
+
...exports.DefaultSummaryConfiguration,
|
|
215
|
+
// the runtime configuration overrides
|
|
216
|
+
...runtimeOptions.summaryOptions?.summaryConfigOverrides,
|
|
217
|
+
}, initializeEntryPoint) {
|
|
180
218
|
super();
|
|
181
|
-
this.context = context;
|
|
182
219
|
this.registry = registry;
|
|
183
220
|
this.runtimeOptions = runtimeOptions;
|
|
184
221
|
this.containerScope = containerScope;
|
|
@@ -207,51 +244,56 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
207
244
|
signalTimestamp: 0,
|
|
208
245
|
trackingSignalSequenceNumber: undefined,
|
|
209
246
|
};
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
223
|
-
|
|
224
|
-
this.
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
else if (this.summaryManager !== undefined) {
|
|
229
|
-
return this.summaryManager.enqueueSummarize(...args);
|
|
247
|
+
const { options, clientDetails, connected, baseSnapshot, submitFn, submitBatchFn, submitSummaryFn, submitSignalFn, disposeFn, closeFn, deltaManager, quorum, audience, loader, pendingLocalState, supportedFeatures, } = context;
|
|
248
|
+
this.innerDeltaManager = deltaManager;
|
|
249
|
+
this.deltaManager = new deltaManagerSummarizerProxy_1.DeltaManagerSummarizerProxy(this.innerDeltaManager);
|
|
250
|
+
// Here we could wrap/intercept on these functions to block/modify outgoing messages if needed.
|
|
251
|
+
// This makes ContainerRuntime the final gatekeeper for outgoing messages.
|
|
252
|
+
this.submitFn = submitFn;
|
|
253
|
+
this.submitBatchFn = submitBatchFn;
|
|
254
|
+
this.submitSummaryFn = submitSummaryFn;
|
|
255
|
+
this.submitSignalFn = submitSignalFn;
|
|
256
|
+
this.options = options;
|
|
257
|
+
this.clientDetails = clientDetails;
|
|
258
|
+
this.isSummarizerClient = this.clientDetails.type === summary_1.summarizerClientType;
|
|
259
|
+
this.loadedFromVersionId = context.getLoadedFromVersion()?.id;
|
|
260
|
+
this._getClientId = () => context.clientId;
|
|
261
|
+
this._getAttachState = () => context.attachState;
|
|
262
|
+
this.getAbsoluteUrl = async (relativeUrl) => {
|
|
263
|
+
if (context.getAbsoluteUrl === undefined) {
|
|
264
|
+
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
230
265
|
}
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
234
|
-
// because it is a misuse of the API rather than an expected failure.
|
|
235
|
-
throw new container_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
266
|
+
if (this.attachState !== container_definitions_1.AttachState.Attached) {
|
|
267
|
+
return undefined;
|
|
236
268
|
}
|
|
269
|
+
return context.getAbsoluteUrl(relativeUrl);
|
|
237
270
|
};
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
this.
|
|
271
|
+
// TODO: Consider that the Container could just listen to these events itself, or even more appropriately maybe the
|
|
272
|
+
// customer should observe dirty state on the runtime (the owner of dirty state) directly, rather than on the IContainer.
|
|
273
|
+
this.on("dirty", () => context.updateDirtyContainerState(true));
|
|
274
|
+
this.on("saved", () => context.updateDirtyContainerState(false));
|
|
275
|
+
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
276
|
+
this.disposeFn = disposeFn ?? closeFn;
|
|
277
|
+
// In cases of summarizer, we want to dispose instead since consumer doesn't interact with this container
|
|
278
|
+
this.closeFn = this.isSummarizerClient ? this.disposeFn : closeFn;
|
|
279
|
+
this.mc = (0, telemetry_utils_1.createChildMonitoringContext)({
|
|
280
|
+
logger: this.logger,
|
|
281
|
+
namespace: "ContainerRuntime",
|
|
282
|
+
});
|
|
241
283
|
let loadSummaryNumber;
|
|
242
284
|
// Get the container creation metadata. For new container, we initialize these. For existing containers,
|
|
243
285
|
// get the values from the metadata blob.
|
|
244
286
|
if (existing) {
|
|
245
287
|
this.createContainerMetadata = {
|
|
246
|
-
createContainerRuntimeVersion: metadata
|
|
247
|
-
createContainerTimestamp: metadata
|
|
288
|
+
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
289
|
+
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
248
290
|
};
|
|
249
291
|
// summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
|
|
250
292
|
// the count is reset to 0.
|
|
251
|
-
loadSummaryNumber =
|
|
293
|
+
loadSummaryNumber = metadata?.summaryNumber ?? 0;
|
|
252
294
|
// Enabling the IdCompressor is a one-way operation and we only want to
|
|
253
295
|
// allow new containers to turn it on
|
|
254
|
-
this.idCompressorEnabled =
|
|
296
|
+
this.idCompressorEnabled = metadata?.idCompressorEnabled ?? false;
|
|
255
297
|
}
|
|
256
298
|
else {
|
|
257
299
|
this.createContainerMetadata = {
|
|
@@ -260,24 +302,27 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
260
302
|
};
|
|
261
303
|
loadSummaryNumber = 0;
|
|
262
304
|
this.idCompressorEnabled =
|
|
263
|
-
|
|
305
|
+
this.mc.config.getBoolean("Fluid.ContainerRuntime.IdCompressorEnabled") ??
|
|
306
|
+
idCompressor !== undefined;
|
|
264
307
|
}
|
|
265
308
|
this.nextSummaryNumber = loadSummaryNumber + 1;
|
|
266
|
-
this.messageAtLastSummary = metadata
|
|
267
|
-
|
|
268
|
-
|
|
309
|
+
this.messageAtLastSummary = metadata?.message;
|
|
310
|
+
// Note that we only need to pull the *initial* connected state from the context.
|
|
311
|
+
// Later updates come through calls to setConnectionState.
|
|
312
|
+
this._connected = connected;
|
|
313
|
+
this.gcTombstoneEnforcementAllowed = (0, gc_1.shouldAllowGcTombstoneEnforcement)(metadata?.gcFeatureMatrix?.tombstoneGeneration /* persisted */, this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName] /* current */);
|
|
269
314
|
this.mc.logger.sendTelemetryEvent({
|
|
270
315
|
eventName: "GCFeatureMatrix",
|
|
271
|
-
metadataValue: JSON.stringify(metadata
|
|
316
|
+
metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
|
|
272
317
|
inputs: JSON.stringify({
|
|
273
318
|
gcOptions_gcTombstoneGeneration: this.runtimeOptions.gcOptions[gc_1.gcTombstoneGenerationOptionName],
|
|
274
319
|
}),
|
|
275
320
|
});
|
|
276
|
-
this.telemetryDocumentId =
|
|
321
|
+
this.telemetryDocumentId = metadata?.telemetryDocumentId ?? (0, uuid_1.v4)();
|
|
277
322
|
this.disableAttachReorder = this.mc.config.getBoolean("Fluid.ContainerRuntime.disableAttachOpReorder");
|
|
278
323
|
const disableChunking = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionChunkingDisabled");
|
|
279
324
|
const opGroupingManager = new opLifecycle_1.OpGroupingManager(this.groupedBatchingEnabled);
|
|
280
|
-
const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.
|
|
325
|
+
const opSplitter = new opLifecycle_1.OpSplitter(chunks, this.submitBatchFn, disableChunking === true ? Number.POSITIVE_INFINITY : runtimeOptions.chunkSizeInBytes, runtimeOptions.maxBatchSizeInBytes, this.mc.logger);
|
|
281
326
|
this.remoteMessageProcessor = new opLifecycle_1.RemoteMessageProcessor(opSplitter, new opLifecycle_1.OpDecompressor(this.mc.logger), opGroupingManager);
|
|
282
327
|
this.handleContext = new containerHandleContext_1.ContainerFluidHandleContext("", this);
|
|
283
328
|
if (this.summaryConfiguration.state === "enabled") {
|
|
@@ -296,9 +341,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
296
341
|
this.idCompressor = idCompressor;
|
|
297
342
|
}
|
|
298
343
|
this.maxConsecutiveReconnects =
|
|
299
|
-
|
|
344
|
+
this.mc.config.getNumber(maxConsecutiveReconnectsKey) ??
|
|
345
|
+
this.defaultMaxConsecutiveReconnects;
|
|
300
346
|
if (runtimeOptions.flushMode === runtime_definitions_1.FlushModeExperimental.Async &&
|
|
301
|
-
|
|
347
|
+
supportedFeatures?.get("referenceSequenceNumbers") !== true) {
|
|
302
348
|
// The loader does not support reference sequence numbers, falling back on FlushMode.TurnBased
|
|
303
349
|
this.mc.logger.sendErrorEvent({ eventName: "FlushModeFallback" });
|
|
304
350
|
this._flushMode = runtime_definitions_1.FlushMode.TurnBased;
|
|
@@ -306,40 +352,39 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
306
352
|
else {
|
|
307
353
|
this._flushMode = runtimeOptions.flushMode;
|
|
308
354
|
}
|
|
309
|
-
const pendingRuntimeState =
|
|
310
|
-
const maxSnapshotCacheDurationMs =
|
|
355
|
+
const pendingRuntimeState = pendingLocalState;
|
|
356
|
+
const maxSnapshotCacheDurationMs = this._storage?.policies?.maximumCacheDurationMs;
|
|
311
357
|
if (maxSnapshotCacheDurationMs !== undefined &&
|
|
312
358
|
maxSnapshotCacheDurationMs > 5 * 24 * 60 * 60 * 1000) {
|
|
313
359
|
// This is a runtime enforcement of what's already explicit in the policy's type itself,
|
|
314
360
|
// which dictates the value is either undefined or exactly 5 days in ms.
|
|
315
361
|
// As long as the actual value is less than 5 days, the assumptions GC makes here are valid.
|
|
316
|
-
throw new
|
|
362
|
+
throw new telemetry_utils_1.UsageError("Driver's maximumCacheDurationMs policy cannot exceed 5 days");
|
|
317
363
|
}
|
|
318
364
|
this.garbageCollector = gc_1.GarbageCollector.create({
|
|
319
365
|
runtime: this,
|
|
320
366
|
gcOptions: this.runtimeOptions.gcOptions,
|
|
321
|
-
baseSnapshot
|
|
367
|
+
baseSnapshot,
|
|
322
368
|
baseLogger: this.mc.logger,
|
|
323
369
|
existing,
|
|
324
370
|
metadata,
|
|
325
371
|
createContainerMetadata: this.createContainerMetadata,
|
|
326
|
-
isSummarizerClient: this.
|
|
372
|
+
isSummarizerClient: this.isSummarizerClient,
|
|
327
373
|
getNodePackagePath: async (nodePath) => this.getGCNodePackagePath(nodePath),
|
|
328
|
-
getLastSummaryTimestampMs: () =>
|
|
374
|
+
getLastSummaryTimestampMs: () => this.messageAtLastSummary?.timestamp,
|
|
329
375
|
readAndParseBlob: async (id) => (0, driver_utils_1.readAndParse)(this.storage, id),
|
|
330
|
-
getContainerDiagnosticId: () => this.context.id,
|
|
331
376
|
// GC runs in summarizer client and needs access to the real (non-proxy) active information. The proxy
|
|
332
377
|
// delta manager would always return false for summarizer client.
|
|
333
378
|
activeConnection: () => this.innerDeltaManager.active,
|
|
334
379
|
});
|
|
335
380
|
const loadedFromSequenceNumber = this.deltaManager.initialSequenceNumber;
|
|
336
|
-
this.summarizerNode = (0, summary_1.createRootSummarizerNodeWithGC)(telemetry_utils_1.
|
|
381
|
+
this.summarizerNode = (0, summary_1.createRootSummarizerNodeWithGC)((0, telemetry_utils_1.createChildLogger)({ logger: this.logger, namespace: "SummarizerNode" }),
|
|
337
382
|
// Summarize function to call when summarize is called. Summarizer node always tracks summary state.
|
|
338
383
|
async (fullTree, trackState, telemetryContext) => this.summarizeInternal(fullTree, trackState, telemetryContext),
|
|
339
384
|
// Latest change sequence number, no changes since summary applied yet
|
|
340
385
|
loadedFromSequenceNumber,
|
|
341
386
|
// Summary reference sequence number, undefined if no summary yet
|
|
342
|
-
|
|
387
|
+
baseSnapshot !== undefined ? loadedFromSequenceNumber : undefined, {
|
|
343
388
|
// Must set to false to prevent sending summary handle which would be pointing to
|
|
344
389
|
// a summary with an older protocol state.
|
|
345
390
|
canReuseHandle: false,
|
|
@@ -353,27 +398,28 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
353
398
|
async (fullGC) => this.getGCDataInternal(fullGC),
|
|
354
399
|
// Function to get the GC details from the base snapshot we loaded from.
|
|
355
400
|
async () => this.garbageCollector.getBaseGCDetails());
|
|
356
|
-
if (
|
|
357
|
-
this.summarizerNode.updateBaseSummaryState(
|
|
401
|
+
if (baseSnapshot) {
|
|
402
|
+
this.summarizerNode.updateBaseSummaryState(baseSnapshot);
|
|
358
403
|
}
|
|
359
|
-
this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(
|
|
404
|
+
this.dataStores = new dataStores_1.DataStores((0, dataStores_1.getSummaryForDatastores)(baseSnapshot, metadata), this, (attachMsg) => this.submit({ type: ContainerMessageType.Attach, contents: attachMsg }), (id, createParam) => (summarizeInternal, getGCDataFn) => this.summarizerNode.createChild(summarizeInternal, id, createParam, undefined, getGCDataFn), (id) => this.summarizerNode.deleteChild(id), this.mc.logger, (path, timestampMs, packagePath) => this.garbageCollector.nodeUpdated(path, "Changed", timestampMs, packagePath), (path) => this.garbageCollector.isNodeDeleted(path), new Map(dataStoreAliasMap));
|
|
360
405
|
this.blobManager = new blobManager_1.BlobManager(this.handleContext, blobManagerSnapshot, () => this.storage, (localId, blobId) => {
|
|
361
406
|
if (!this.disposed) {
|
|
362
|
-
this.submit(ContainerMessageType.BlobAttach, undefined, undefined, {
|
|
407
|
+
this.submit({ type: ContainerMessageType.BlobAttach, contents: undefined }, undefined, {
|
|
363
408
|
localId,
|
|
364
409
|
blobId,
|
|
365
410
|
});
|
|
366
411
|
}
|
|
367
|
-
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState
|
|
368
|
-
this.scheduleManager = new scheduleManager_1.ScheduleManager(
|
|
412
|
+
}, (blobPath) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"), (blobPath) => this.garbageCollector.isNodeDeleted(blobPath), this, pendingRuntimeState?.pendingAttachmentBlobs, (error) => this.closeFn(error));
|
|
413
|
+
this.scheduleManager = new scheduleManager_1.ScheduleManager(this.innerDeltaManager, this, () => this.clientId, (0, telemetry_utils_1.createChildLogger)({ logger: this.logger, namespace: "ScheduleManager" }));
|
|
369
414
|
this.pendingStateManager = new pendingStateManager_1.PendingStateManager({
|
|
370
415
|
applyStashedOp: this.applyStashedOp.bind(this),
|
|
371
416
|
clientId: () => this.clientId,
|
|
372
417
|
close: this.closeFn,
|
|
373
418
|
connected: () => this.connected,
|
|
374
419
|
reSubmit: this.reSubmit.bind(this),
|
|
375
|
-
|
|
376
|
-
|
|
420
|
+
reSubmitBatch: this.reSubmitBatch.bind(this),
|
|
421
|
+
isActiveConnection: () => this.innerDeltaManager.active,
|
|
422
|
+
}, pendingRuntimeState?.pending, this.logger);
|
|
377
423
|
const disableCompression = this.mc.config.getBoolean("Fluid.ContainerRuntime.CompressionDisabled");
|
|
378
424
|
const compressionOptions = disableCompression === true
|
|
379
425
|
? {
|
|
@@ -382,16 +428,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
382
428
|
}
|
|
383
429
|
: runtimeOptions.compressionOptions;
|
|
384
430
|
const disablePartialFlush = this.mc.config.getBoolean("Fluid.ContainerRuntime.DisablePartialFlush");
|
|
431
|
+
const legacySendBatchFn = (0, exports.makeLegacySendBatchFn)(this.submitFn, this.innerDeltaManager);
|
|
385
432
|
this.outbox = new opLifecycle_1.Outbox({
|
|
386
433
|
shouldSend: () => this.canSendOps(),
|
|
387
434
|
pendingStateManager: this.pendingStateManager,
|
|
388
|
-
|
|
435
|
+
submitBatchFn: this.submitBatchFn,
|
|
436
|
+
legacySendBatchFn,
|
|
389
437
|
compressor: new opLifecycle_1.OpCompressor(this.mc.logger),
|
|
390
438
|
splitter: opSplitter,
|
|
391
439
|
config: {
|
|
392
440
|
compressionOptions,
|
|
393
441
|
maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
|
|
394
442
|
disablePartialFlush: disablePartialFlush === true,
|
|
443
|
+
enableGroupedBatching: this.groupedBatchingEnabled,
|
|
395
444
|
},
|
|
396
445
|
logger: this.mc.logger,
|
|
397
446
|
groupingManager: opGroupingManager,
|
|
@@ -399,43 +448,54 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
399
448
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
400
449
|
clientSequenceNumber: this._processedClientSequenceNumber,
|
|
401
450
|
}),
|
|
451
|
+
reSubmit: this.reSubmit.bind(this),
|
|
452
|
+
opReentrancy: () => this.ensureNoDataModelChangesCalls > 0,
|
|
453
|
+
closeContainer: this.closeFn,
|
|
402
454
|
});
|
|
403
|
-
this.
|
|
455
|
+
this._quorum = quorum;
|
|
456
|
+
this._quorum.on("removeMember", (clientId) => {
|
|
404
457
|
this.remoteMessageProcessor.clearPartialMessagesFor(clientId);
|
|
405
458
|
});
|
|
406
|
-
|
|
459
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
460
|
+
this._audience = audience;
|
|
407
461
|
const closeSummarizerDelayOverride = this.mc.config.getNumber("Fluid.ContainerRuntime.Test.CloseSummarizerDelayOverrideMs");
|
|
408
|
-
this.closeSummarizerDelayMs = closeSummarizerDelayOverride
|
|
462
|
+
this.closeSummarizerDelayMs = closeSummarizerDelayOverride ?? defaultCloseSummarizerDelayMs;
|
|
463
|
+
this.validateSummaryBeforeUpload =
|
|
464
|
+
this.mc.config.getBoolean("Fluid.Summarizer.ValidateSummaryBeforeUpload") ?? false;
|
|
409
465
|
this.summaryCollection = new summary_1.SummaryCollection(this.deltaManager, this.logger);
|
|
410
466
|
this.dirtyContainer =
|
|
411
|
-
this.
|
|
412
|
-
|
|
413
|
-
this.context.updateDirtyContainerState(this.dirtyContainer);
|
|
467
|
+
this.attachState !== container_definitions_1.AttachState.Attached || this.hasPendingMessages();
|
|
468
|
+
context.updateDirtyContainerState(this.dirtyContainer);
|
|
414
469
|
if (this.summariesDisabled) {
|
|
415
470
|
this.mc.logger.sendTelemetryEvent({ eventName: "SummariesDisabled" });
|
|
416
471
|
}
|
|
417
472
|
else {
|
|
418
|
-
const orderedClientLogger =
|
|
419
|
-
|
|
420
|
-
|
|
473
|
+
const orderedClientLogger = (0, telemetry_utils_1.createChildLogger)({
|
|
474
|
+
logger: this.logger,
|
|
475
|
+
namespace: "OrderedClientElection",
|
|
476
|
+
});
|
|
477
|
+
const orderedClientCollection = new summary_1.OrderedClientCollection(orderedClientLogger, this.innerDeltaManager, this._quorum);
|
|
478
|
+
const orderedClientElectionForSummarizer = new summary_1.OrderedClientElection(orderedClientLogger, orderedClientCollection, electedSummarizerData ?? this.innerDeltaManager.lastSequenceNumber, summary_1.SummarizerClientElection.isClientEligible);
|
|
421
479
|
this.summarizerClientElection = new summary_1.SummarizerClientElection(orderedClientLogger, this.summaryCollection, orderedClientElectionForSummarizer, this.maxOpsSinceLastSummary);
|
|
422
|
-
if (this.
|
|
480
|
+
if (this.isSummarizerClient) {
|
|
423
481
|
this._summarizer = new summary_1.Summarizer(this /* ISummarizerRuntime */, () => this.summaryConfiguration, this /* ISummarizerInternalsProvider */, this.handleContext, this.summaryCollection, async (runtime) => summary_1.RunWhileConnectedCoordinator.create(runtime,
|
|
424
482
|
// Summarization runs in summarizer client and needs access to the real (non-proxy) active
|
|
425
483
|
// information. The proxy delta manager would always return false for summarizer client.
|
|
426
484
|
() => this.innerDeltaManager.active));
|
|
427
485
|
}
|
|
428
|
-
else if (summary_1.SummarizerClientElection.clientDetailsPermitElection(this.
|
|
486
|
+
else if (summary_1.SummarizerClientElection.clientDetailsPermitElection(this.clientDetails)) {
|
|
429
487
|
// Only create a SummaryManager and SummarizerClientElection
|
|
430
488
|
// if summaries are enabled and we are not the summarizer client.
|
|
431
489
|
const defaultAction = () => {
|
|
432
490
|
if (this.summaryCollection.opsSinceLastAck > this.maxOpsSinceLastSummary) {
|
|
433
|
-
this.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
491
|
+
this.mc.logger.sendTelemetryEvent({ eventName: "SummaryStatus:Behind" });
|
|
434
492
|
// unregister default to no log on every op after falling behind
|
|
435
493
|
// and register summary ack handler to re-register this handler
|
|
436
494
|
// after successful summary
|
|
437
495
|
this.summaryCollection.once(protocol_definitions_1.MessageType.SummaryAck, () => {
|
|
438
|
-
this.logger.sendTelemetryEvent({
|
|
496
|
+
this.mc.logger.sendTelemetryEvent({
|
|
497
|
+
eventName: "SummaryStatus:CaughtUp",
|
|
498
|
+
});
|
|
439
499
|
// we've caught up, so re-register the default action to monitor for
|
|
440
500
|
// falling behind, and unregister ourself
|
|
441
501
|
this.summaryCollection.on("default", defaultAction);
|
|
@@ -446,12 +506,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
446
506
|
this.summaryCollection.on("default", defaultAction);
|
|
447
507
|
// Create the SummaryManager and mark the initial state
|
|
448
508
|
this.summaryManager = new summary_1.SummaryManager(this.summarizerClientElection, this, // IConnectedState
|
|
449
|
-
this.summaryCollection, this.logger, this.formRequestSummarizerFn(
|
|
509
|
+
this.summaryCollection, this.logger, this.formRequestSummarizerFn(loader), new throttler_1.Throttler(60 * 1000, // 60 sec delay window
|
|
450
510
|
30 * 1000, // 30 sec max delay
|
|
451
511
|
// throttling function increases exponentially (0ms, 40ms, 80ms, 160ms, etc)
|
|
452
512
|
(0, throttler_1.formExponentialFn)({ coefficient: 20, initialDelay: 0 })), {
|
|
453
513
|
initialDelayMs: this.initialSummarizerDelayMs,
|
|
454
514
|
}, this.heuristicsDisabled);
|
|
515
|
+
this.summaryManager.on("summarize", (eventProps) => {
|
|
516
|
+
this.emit("summarize", eventProps);
|
|
517
|
+
});
|
|
455
518
|
this.summaryManager.start();
|
|
456
519
|
}
|
|
457
520
|
}
|
|
@@ -459,7 +522,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
459
522
|
// we accumulate ops while being in read-only state.
|
|
460
523
|
// once user gets write permissions and we have active connection, flush all pending ops.
|
|
461
524
|
// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
|
|
462
|
-
(0,
|
|
525
|
+
(0, core_utils_1.assert)(readonly === this.innerDeltaManager.readOnlyInfo.readonly, 0x124 /* "inconsistent readonly property/event state" */);
|
|
463
526
|
// We need to be very careful with when we (re)send pending ops, to ensure that we only send ops
|
|
464
527
|
// when we either never send an op, or attempted to send it but we know for sure it was not
|
|
465
528
|
// sequenced by server and will never be sequenced (i.e. was lost)
|
|
@@ -472,31 +535,48 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
472
535
|
// can rely on same safety mechanism and resend ops only when we establish new connection.
|
|
473
536
|
// This is applicable for read-only permissions (event is raised before connection is properly registered),
|
|
474
537
|
// but it's an extra requirement for Container.forceReadonly() API
|
|
475
|
-
(0,
|
|
538
|
+
(0, core_utils_1.assert)(!readonly || !this.connected, 0x125 /* "Unsafe to transition to read-only state!" */);
|
|
476
539
|
this.replayPendingStates();
|
|
477
540
|
});
|
|
478
541
|
// logging hardware telemetry
|
|
479
|
-
logger.sendTelemetryEvent(
|
|
480
|
-
|
|
542
|
+
logger.sendTelemetryEvent({
|
|
543
|
+
eventName: "DeviceSpec",
|
|
544
|
+
...getDeviceSpec(),
|
|
545
|
+
});
|
|
546
|
+
this.mc.logger.sendTelemetryEvent({
|
|
547
|
+
eventName: "ContainerLoadStats",
|
|
548
|
+
...this.createContainerMetadata,
|
|
549
|
+
...this.dataStores.containerLoadStats,
|
|
550
|
+
summaryNumber: loadSummaryNumber,
|
|
551
|
+
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
552
|
+
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
553
|
+
gcVersion: metadata?.gcFeature,
|
|
554
|
+
options: JSON.stringify(runtimeOptions),
|
|
555
|
+
featureGates: JSON.stringify({
|
|
481
556
|
disableCompression,
|
|
482
557
|
disableOpReentryCheck,
|
|
483
558
|
disableChunking,
|
|
484
559
|
disableAttachReorder: this.disableAttachReorder,
|
|
485
560
|
disablePartialFlush,
|
|
486
561
|
idCompressorEnabled: this.idCompressorEnabled,
|
|
487
|
-
summaryStateUpdateMethod: this.summaryStateUpdateMethod,
|
|
488
562
|
closeSummarizerDelayOverride,
|
|
489
|
-
}),
|
|
490
|
-
|
|
563
|
+
}),
|
|
564
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
565
|
+
groupedBatchingEnabled: this.groupedBatchingEnabled,
|
|
566
|
+
});
|
|
567
|
+
(0, connectionTelemetry_1.ReportOpPerfTelemetry)(this.clientId, this.deltaManager, this.logger);
|
|
491
568
|
(0, batchTracker_1.BindBatchTracker)(this, this.logger);
|
|
492
|
-
this.entryPoint = new
|
|
493
|
-
if (this.
|
|
494
|
-
(0,
|
|
569
|
+
this.entryPoint = new core_utils_1.LazyPromise(async () => {
|
|
570
|
+
if (this.isSummarizerClient) {
|
|
571
|
+
(0, core_utils_1.assert)(this._summarizer !== undefined, 0x5bf /* Summarizer object is undefined in a summarizer client */);
|
|
495
572
|
return this._summarizer;
|
|
496
573
|
}
|
|
497
|
-
return initializeEntryPoint
|
|
574
|
+
return initializeEntryPoint?.(this);
|
|
498
575
|
});
|
|
499
576
|
}
|
|
577
|
+
/**
|
|
578
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
579
|
+
*/
|
|
500
580
|
get IFluidRouter() {
|
|
501
581
|
return this;
|
|
502
582
|
}
|
|
@@ -542,26 +622,38 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
542
622
|
* This object should provide all the functionality that the Container is expected to provide to the loader layer.
|
|
543
623
|
*/
|
|
544
624
|
static async loadRuntime(params) {
|
|
545
|
-
|
|
546
|
-
const
|
|
625
|
+
const { context, registryEntries, existing, requestHandler, runtimeOptions = {}, containerScope = {}, containerRuntimeCtor = ContainerRuntime, } = params;
|
|
626
|
+
const initializeEntryPoint = params.initializeEntryPoint ??
|
|
627
|
+
(async (containerRuntime) => ({
|
|
628
|
+
get IFluidRouter() {
|
|
629
|
+
return this;
|
|
630
|
+
},
|
|
631
|
+
async request(req) {
|
|
632
|
+
return containerRuntime.request(req);
|
|
633
|
+
},
|
|
634
|
+
}));
|
|
547
635
|
// If taggedLogger exists, use it. Otherwise, wrap the vanilla logger:
|
|
548
636
|
// back-compat: Remove the TaggedLoggerAdapter fallback once all the host are using loader > 0.45
|
|
549
637
|
const backCompatContext = context;
|
|
550
|
-
const passLogger =
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
638
|
+
const passLogger = backCompatContext.taggedLogger ??
|
|
639
|
+
// eslint-disable-next-line import/no-deprecated
|
|
640
|
+
new telemetry_utils_1.TaggedLoggerAdapter(backCompatContext.logger);
|
|
641
|
+
const logger = (0, telemetry_utils_1.createChildLogger)({
|
|
642
|
+
logger: passLogger,
|
|
643
|
+
properties: {
|
|
644
|
+
all: {
|
|
645
|
+
runtimeVersion: packageVersion_1.pkgVersion,
|
|
646
|
+
},
|
|
554
647
|
},
|
|
555
648
|
});
|
|
556
649
|
const { summaryOptions = {}, gcOptions = {}, loadSequenceNumberVerification = "close", flushMode = defaultFlushMode, compressionOptions = defaultCompressionConfig, maxBatchSizeInBytes = defaultMaxBatchSizeInBytes, enableRuntimeIdCompressor = false, chunkSizeInBytes = defaultChunkSizeInBytes, enableOpReentryCheck = false, enableGroupedBatching = false, } = runtimeOptions;
|
|
557
650
|
const registry = new dataStoreRegistry_1.FluidDataStoreRegistry(registryEntries);
|
|
558
651
|
const tryFetchBlob = async (blobName) => {
|
|
559
|
-
|
|
560
|
-
const blobId = (_a = context.baseSnapshot) === null || _a === void 0 ? void 0 : _a.blobs[blobName];
|
|
652
|
+
const blobId = context.baseSnapshot?.blobs[blobName];
|
|
561
653
|
if (context.baseSnapshot && blobId) {
|
|
562
654
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
563
655
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
564
|
-
(0,
|
|
656
|
+
(0, core_utils_1.assert)(context.storage !== undefined, 0x1f5 /* "Attached state should have storage" */);
|
|
565
657
|
return (0, driver_utils_1.readAndParse)(context.storage, blobId);
|
|
566
658
|
}
|
|
567
659
|
};
|
|
@@ -572,16 +664,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
572
664
|
tryFetchBlob(summary_1.aliasBlobName),
|
|
573
665
|
tryFetchBlob(summary_1.idCompressorBlobName),
|
|
574
666
|
]);
|
|
575
|
-
const loadExisting = existing === true || context.existing === true;
|
|
576
667
|
// read snapshot blobs needed for BlobManager to load
|
|
577
|
-
const blobManagerSnapshot = await blobManager_1.BlobManager.load(
|
|
668
|
+
const blobManagerSnapshot = await blobManager_1.BlobManager.load(context.baseSnapshot?.trees[summary_1.blobsTreeName], async (id) => {
|
|
578
669
|
// IContainerContext storage api return type still has undefined in 0.39 package version.
|
|
579
670
|
// So once we release 0.40 container-defn package we can remove this check.
|
|
580
|
-
(0,
|
|
671
|
+
(0, core_utils_1.assert)(context.storage !== undefined, 0x256 /* "storage undefined in attached container" */);
|
|
581
672
|
return (0, driver_utils_1.readAndParse)(context.storage, id);
|
|
582
673
|
});
|
|
583
674
|
// Verify summary runtime sequence number matches protocol sequence number.
|
|
584
|
-
const runtimeSequenceNumber =
|
|
675
|
+
const runtimeSequenceNumber = metadata?.message?.sequenceNumber;
|
|
585
676
|
// When we load with pending state, we reuse an old snapshot so we don't expect these numbers to match
|
|
586
677
|
if (!context.pendingLocalState && runtimeSequenceNumber !== undefined) {
|
|
587
678
|
const protocolSequenceNumber = context.deltaManager.initialSequenceNumber;
|
|
@@ -589,29 +680,27 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
589
680
|
if (loadSequenceNumberVerification !== "bypass" &&
|
|
590
681
|
runtimeSequenceNumber !== protocolSequenceNumber) {
|
|
591
682
|
// "Load from summary, runtime metadata sequenceNumber !== initialSequenceNumber"
|
|
592
|
-
const error = new
|
|
683
|
+
const error = new telemetry_utils_1.DataCorruptionError(
|
|
593
684
|
// pre-0.58 error message: SummaryMetadataMismatch
|
|
594
685
|
"Summary metadata mismatch", { runtimeVersion: packageVersion_1.pkgVersion, runtimeSequenceNumber, protocolSequenceNumber });
|
|
595
686
|
if (loadSequenceNumberVerification === "log") {
|
|
596
687
|
logger.sendErrorEvent({ eventName: "SequenceNumberMismatch" }, error);
|
|
597
688
|
}
|
|
598
689
|
else {
|
|
599
|
-
// Call both close and dispose as closeFn implementation will no longer dispose runtime in future
|
|
600
690
|
context.closeFn(error);
|
|
601
|
-
(_d = context.disposeFn) === null || _d === void 0 ? void 0 : _d.call(context, error);
|
|
602
691
|
}
|
|
603
692
|
}
|
|
604
693
|
}
|
|
605
|
-
const idCompressorEnabled =
|
|
694
|
+
const idCompressorEnabled = metadata?.idCompressorEnabled ?? runtimeOptions.enableRuntimeIdCompressor ?? false;
|
|
606
695
|
let idCompressor;
|
|
607
696
|
if (idCompressorEnabled) {
|
|
608
697
|
const { IdCompressor, createSessionId } = await Promise.resolve().then(() => __importStar(require("./id-compressor")));
|
|
609
698
|
idCompressor =
|
|
610
699
|
serializedIdCompressor !== undefined
|
|
611
700
|
? IdCompressor.deserialize(serializedIdCompressor, createSessionId())
|
|
612
|
-
:
|
|
701
|
+
: IdCompressor.create(logger);
|
|
613
702
|
}
|
|
614
|
-
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks
|
|
703
|
+
const runtime = new containerRuntimeCtor(context, registry, metadata, electedSummarizerData, chunks ?? [], aliases ?? [], {
|
|
615
704
|
summaryOptions,
|
|
616
705
|
gcOptions,
|
|
617
706
|
loadSequenceNumberVerification,
|
|
@@ -622,8 +711,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
622
711
|
enableRuntimeIdCompressor,
|
|
623
712
|
enableOpReentryCheck,
|
|
624
713
|
enableGroupedBatching,
|
|
625
|
-
}, containerScope, logger,
|
|
714
|
+
}, containerScope, logger, existing, blobManagerSnapshot, context.storage, idCompressor, requestHandler, undefined, // summaryConfiguration
|
|
626
715
|
initializeEntryPoint);
|
|
716
|
+
await runtime.blobManager.processStashedChanges();
|
|
627
717
|
// It's possible to have ops with a reference sequence number of 0. Op sequence numbers start
|
|
628
718
|
// at 1, so we won't see a replayed saved op with a sequence number of 0.
|
|
629
719
|
await runtime.pendingStateManager.applyStashedOpsAt(0);
|
|
@@ -631,34 +721,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
631
721
|
await runtime.initializeBaseState();
|
|
632
722
|
return runtime;
|
|
633
723
|
}
|
|
634
|
-
get options() {
|
|
635
|
-
return this.context.options;
|
|
636
|
-
}
|
|
637
724
|
get clientId() {
|
|
638
|
-
return this.
|
|
639
|
-
}
|
|
640
|
-
get clientDetails() {
|
|
641
|
-
return this.context.clientDetails;
|
|
725
|
+
return this._getClientId();
|
|
642
726
|
}
|
|
643
727
|
get storage() {
|
|
644
728
|
return this._storage;
|
|
645
729
|
}
|
|
730
|
+
/** @deprecated - The functionality is no longer exposed publicly */
|
|
646
731
|
get reSubmitFn() {
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
}
|
|
650
|
-
get disposeFn() {
|
|
651
|
-
var _a;
|
|
652
|
-
// In old loaders without dispose functionality, closeFn is equivalent but will also switch container to readonly mode
|
|
653
|
-
return (_a = this.context.disposeFn) !== null && _a !== void 0 ? _a : this.context.closeFn;
|
|
654
|
-
}
|
|
655
|
-
get closeFn() {
|
|
656
|
-
// Also call disposeFn to retain functionality of runtime being disposed on close
|
|
657
|
-
return (error) => {
|
|
658
|
-
var _a, _b;
|
|
659
|
-
this.context.closeFn(error);
|
|
660
|
-
(_b = (_a = this.context).disposeFn) === null || _b === void 0 ? void 0 : _b.call(_a, error);
|
|
661
|
-
};
|
|
732
|
+
return (type, contents, localOpMetadata, opMetadata) => this.reSubmitCore({ type, contents }, localOpMetadata, opMetadata);
|
|
733
|
+
// Note: compatDetails is not included in this deprecated API
|
|
662
734
|
}
|
|
663
735
|
get flushMode() {
|
|
664
736
|
return this._flushMode;
|
|
@@ -670,7 +742,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
670
742
|
return this.registry;
|
|
671
743
|
}
|
|
672
744
|
get attachState() {
|
|
673
|
-
return this.
|
|
745
|
+
return this._getAttachState();
|
|
674
746
|
}
|
|
675
747
|
get IFluidHandleContext() {
|
|
676
748
|
return this.handleContext;
|
|
@@ -697,14 +769,13 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
697
769
|
}
|
|
698
770
|
/** clientId of parent (non-summarizing) container that owns summarizer container */
|
|
699
771
|
get summarizerClientId() {
|
|
700
|
-
|
|
701
|
-
return (_a = this.summarizerClientElection) === null || _a === void 0 ? void 0 : _a.electedClientId;
|
|
772
|
+
return this.summarizerClientElection?.electedClientId;
|
|
702
773
|
}
|
|
703
774
|
get disposed() {
|
|
704
775
|
return this._disposed;
|
|
705
776
|
}
|
|
706
777
|
get summarizer() {
|
|
707
|
-
(0,
|
|
778
|
+
(0, core_utils_1.assert)(this._summarizer !== undefined, 0x257 /* "This is not summarizing container" */);
|
|
708
779
|
return this._summarizer;
|
|
709
780
|
}
|
|
710
781
|
isSummariesDisabled() {
|
|
@@ -735,12 +806,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
735
806
|
await this.garbageCollector.initializeBaseState();
|
|
736
807
|
}
|
|
737
808
|
dispose(error) {
|
|
738
|
-
var _a;
|
|
739
809
|
if (this._disposed) {
|
|
740
810
|
return;
|
|
741
811
|
}
|
|
742
812
|
this._disposed = true;
|
|
743
|
-
this.logger.sendTelemetryEvent({
|
|
813
|
+
this.mc.logger.sendTelemetryEvent({
|
|
744
814
|
eventName: "ContainerRuntimeDisposed",
|
|
745
815
|
isDirty: this.isDirty,
|
|
746
816
|
lastSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
@@ -750,7 +820,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
750
820
|
this.summaryManager.dispose();
|
|
751
821
|
}
|
|
752
822
|
this.garbageCollector.dispose();
|
|
753
|
-
|
|
823
|
+
this._summarizer?.dispose();
|
|
754
824
|
this.dataStores.dispose();
|
|
755
825
|
this.pendingStateManager.dispose();
|
|
756
826
|
this.emit("dispose");
|
|
@@ -759,6 +829,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
759
829
|
/**
|
|
760
830
|
* Notifies this object about the request made to the container.
|
|
761
831
|
* @param request - Request made to the handler.
|
|
832
|
+
* @deprecated - Will be removed in future major release. Migrate all usage of IFluidRouter to the "entryPoint" pattern. Refer to Removing-IFluidRouter.md
|
|
762
833
|
*/
|
|
763
834
|
async request(request) {
|
|
764
835
|
try {
|
|
@@ -809,8 +880,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
809
880
|
const subRequest = requestParser.createSubRequest(1);
|
|
810
881
|
// We always expect createSubRequest to include a leading slash, but asserting here to protect against
|
|
811
882
|
// unintentionally modifying the url if that changes.
|
|
812
|
-
(0,
|
|
813
|
-
return dataStore.
|
|
883
|
+
(0, core_utils_1.assert)(subRequest.url.startsWith("/"), 0x126 /* "Expected createSubRequest url to include a leading slash" */);
|
|
884
|
+
return dataStore.request(subRequest);
|
|
814
885
|
}
|
|
815
886
|
return (0, runtime_utils_1.create404Response)(request);
|
|
816
887
|
}
|
|
@@ -825,19 +896,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
825
896
|
return this.entryPoint;
|
|
826
897
|
}
|
|
827
898
|
internalId(maybeAlias) {
|
|
828
|
-
|
|
829
|
-
return (_a = this.dataStores.aliases.get(maybeAlias)) !== null && _a !== void 0 ? _a : maybeAlias;
|
|
899
|
+
return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
|
|
830
900
|
}
|
|
831
901
|
async getDataStoreFromRequest(id, request) {
|
|
832
|
-
var _a, _b, _c;
|
|
833
902
|
const headerData = {};
|
|
834
|
-
if (typeof
|
|
903
|
+
if (typeof request.headers?.[RuntimeHeaders.wait] === "boolean") {
|
|
835
904
|
headerData.wait = request.headers[RuntimeHeaders.wait];
|
|
836
905
|
}
|
|
837
|
-
if (typeof
|
|
906
|
+
if (typeof request.headers?.[RuntimeHeaders.viaHandle] === "boolean") {
|
|
838
907
|
headerData.viaHandle = request.headers[RuntimeHeaders.viaHandle];
|
|
839
908
|
}
|
|
840
|
-
if (typeof
|
|
909
|
+
if (typeof request.headers?.[exports.AllowTombstoneRequestHeaderKey] === "boolean") {
|
|
841
910
|
headerData.allowTombstone = request.headers[exports.AllowTombstoneRequestHeaderKey];
|
|
842
911
|
}
|
|
843
912
|
await this.dataStores.waitIfPendingAlias(id);
|
|
@@ -847,25 +916,30 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
847
916
|
// Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
|
|
848
917
|
// the same as GC nodes id.
|
|
849
918
|
const urlWithoutQuery = (0, gc_1.trimLeadingAndTrailingSlashes)(request.url.split("?")[0]);
|
|
850
|
-
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request
|
|
919
|
+
this.garbageCollector.nodeUpdated(`/${urlWithoutQuery}`, "Loaded", undefined /* timestampMs */, dataStoreContext.packagePath, request?.headers);
|
|
851
920
|
return dataStoreChannel;
|
|
852
921
|
}
|
|
853
922
|
/** Adds the container's metadata to the given summary tree. */
|
|
854
923
|
addMetadataToSummary(summaryTree) {
|
|
855
|
-
|
|
856
|
-
|
|
924
|
+
const metadata = {
|
|
925
|
+
...this.createContainerMetadata,
|
|
857
926
|
// Increment the summary number for the next summary that will be generated.
|
|
858
|
-
summaryNumber: this.nextSummaryNumber++,
|
|
927
|
+
summaryNumber: this.nextSummaryNumber++,
|
|
928
|
+
summaryFormatVersion: 1,
|
|
929
|
+
...this.garbageCollector.getMetadata(),
|
|
859
930
|
// The last message processed at the time of summary. If there are no new messages, use the message from the
|
|
860
931
|
// last summary.
|
|
861
|
-
message: (
|
|
932
|
+
message: (0, summary_1.extractSummaryMetadataMessage)(this.deltaManager.lastMessage) ??
|
|
933
|
+
this.messageAtLastSummary,
|
|
934
|
+
telemetryDocumentId: this.telemetryDocumentId,
|
|
935
|
+
idCompressorEnabled: this.idCompressorEnabled ? true : undefined,
|
|
936
|
+
};
|
|
862
937
|
(0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.metadataBlobName, JSON.stringify(metadata));
|
|
863
938
|
}
|
|
864
939
|
addContainerStateToSummary(summaryTree, fullTree, trackState, telemetryContext) {
|
|
865
|
-
var _a;
|
|
866
940
|
this.addMetadataToSummary(summaryTree);
|
|
867
941
|
if (this.idCompressorEnabled) {
|
|
868
|
-
(0,
|
|
942
|
+
(0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67a /* IdCompressor should be defined if enabled */);
|
|
869
943
|
const idCompressorState = JSON.stringify(this.idCompressor.serialize(false));
|
|
870
944
|
(0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.idCompressorBlobName, idCompressorState);
|
|
871
945
|
}
|
|
@@ -878,7 +952,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
878
952
|
(0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
879
953
|
}
|
|
880
954
|
if (this.summarizerClientElection) {
|
|
881
|
-
const electedSummarizerContent = JSON.stringify(
|
|
955
|
+
const electedSummarizerContent = JSON.stringify(this.summarizerClientElection?.serialize());
|
|
882
956
|
(0, runtime_utils_1.addBlobToSummary)(summaryTree, summary_1.electedSummarizerBlobName, electedSummarizerContent);
|
|
883
957
|
}
|
|
884
958
|
const blobManagerSummary = this.blobManager.summarize();
|
|
@@ -915,7 +989,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
915
989
|
this.mc.logger.sendTelemetryEvent({
|
|
916
990
|
eventName: "ReconnectsWithNoProgress",
|
|
917
991
|
attempts: this.consecutiveReconnects,
|
|
918
|
-
pendingMessages: this.
|
|
992
|
+
pendingMessages: this.pendingMessagesCount,
|
|
919
993
|
});
|
|
920
994
|
}
|
|
921
995
|
return this.consecutiveReconnects < this.maxConsecutiveReconnects;
|
|
@@ -925,7 +999,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
925
999
|
// in their own batches before the originating batch is sent.
|
|
926
1000
|
// Therefore, receiving them while attempting to send the originating batch
|
|
927
1001
|
// does not mean that the container is making any progress.
|
|
928
|
-
if (
|
|
1002
|
+
if (message?.type !== ContainerMessageType.ChunkedOp) {
|
|
929
1003
|
this.consecutiveReconnects = 0;
|
|
930
1004
|
}
|
|
931
1005
|
}
|
|
@@ -940,7 +1014,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
940
1014
|
// Save the old state, reset to false, disable event emit
|
|
941
1015
|
const oldState = this.dirtyContainer;
|
|
942
1016
|
this.dirtyContainer = false;
|
|
943
|
-
(0,
|
|
1017
|
+
(0, core_utils_1.assert)(this.emitDirtyDocumentEvent, 0x127 /* "dirty document event not set on replay" */);
|
|
944
1018
|
this.emitDirtyDocumentEvent = false;
|
|
945
1019
|
let newState;
|
|
946
1020
|
try {
|
|
@@ -974,21 +1048,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
974
1048
|
* ! Note: this format needs to be in-line with what is set in the "ContainerRuntime.submit(...)" method
|
|
975
1049
|
*/
|
|
976
1050
|
parseOpContent(serializedContent) {
|
|
977
|
-
(0,
|
|
978
|
-
const
|
|
979
|
-
(0,
|
|
980
|
-
return { type
|
|
1051
|
+
(0, core_utils_1.assert)(serializedContent !== undefined, 0x6d5 /* content must be defined */);
|
|
1052
|
+
const { type, contents, compatDetails } = JSON.parse(serializedContent);
|
|
1053
|
+
(0, core_utils_1.assert)(type !== undefined, 0x6d6 /* incorrect op content format */);
|
|
1054
|
+
return { type, contents, compatDetails };
|
|
981
1055
|
}
|
|
982
1056
|
async applyStashedOp(op) {
|
|
983
1057
|
// Need to parse from string for back-compat
|
|
984
|
-
const { type, contents } = this.parseOpContent(op);
|
|
1058
|
+
const { type, contents, compatDetails } = this.parseOpContent(op);
|
|
985
1059
|
switch (type) {
|
|
986
1060
|
case ContainerMessageType.FluidDataStoreOp:
|
|
987
1061
|
return this.dataStores.applyStashedOp(contents);
|
|
988
1062
|
case ContainerMessageType.Attach:
|
|
989
1063
|
return this.dataStores.applyStashedAttachOp(contents);
|
|
990
1064
|
case ContainerMessageType.IdAllocation:
|
|
991
|
-
(0,
|
|
1065
|
+
(0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67b /* IdCompressor should be defined if enabled */);
|
|
992
1066
|
return this.applyStashedIdAllocationOp(contents);
|
|
993
1067
|
case ContainerMessageType.Alias:
|
|
994
1068
|
case ContainerMessageType.BlobAttach:
|
|
@@ -997,8 +1071,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
997
1071
|
throw new Error("chunkedOp not expected here");
|
|
998
1072
|
case ContainerMessageType.Rejoin:
|
|
999
1073
|
throw new Error("rejoin not expected here");
|
|
1000
|
-
default:
|
|
1001
|
-
|
|
1074
|
+
default: {
|
|
1075
|
+
// This should be extremely rare for stashed ops.
|
|
1076
|
+
// It would require a newer runtime stashing ops and then an older one applying them,
|
|
1077
|
+
// e.g. if an app rolled back its container version
|
|
1078
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1079
|
+
if (!compatBehaviorAllowsMessageType(type, compatBehavior)) {
|
|
1080
|
+
const error = telemetry_utils_1.DataProcessingError.create("Stashed runtime message of unknown type", "applyStashedOp", undefined /* sequencedMessage */, {
|
|
1081
|
+
messageDetails: JSON.stringify({
|
|
1082
|
+
type,
|
|
1083
|
+
compatBehavior,
|
|
1084
|
+
}),
|
|
1085
|
+
});
|
|
1086
|
+
this.closeFn(error);
|
|
1087
|
+
throw error;
|
|
1088
|
+
}
|
|
1089
|
+
}
|
|
1002
1090
|
}
|
|
1003
1091
|
}
|
|
1004
1092
|
setConnectionState(connected, clientId) {
|
|
@@ -1010,32 +1098,20 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1010
1098
|
// Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
|
|
1011
1099
|
return;
|
|
1012
1100
|
}
|
|
1013
|
-
// If attachment blobs were added while disconnected, we need to delay
|
|
1014
|
-
// propagation of the "connected" event until we have uploaded them to
|
|
1015
|
-
// ensure we don't submit ops referencing a blob that has not been uploaded
|
|
1016
|
-
// Note that the inner (non-proxy) delta manager is needed here to get the readonly information.
|
|
1017
|
-
const connecting = connected && !this._connected && !this.innerDeltaManager.readOnlyInfo.readonly;
|
|
1018
|
-
if (connecting && this.blobManager.hasPendingOfflineUploads) {
|
|
1019
|
-
(0, common_utils_1.assert)(!this.delayConnectClientId, 0x392 /* Connect event delay must be canceled before subsequent connect event */);
|
|
1020
|
-
(0, common_utils_1.assert)(!!clientId, 0x393 /* Must have clientId when connecting */);
|
|
1021
|
-
this.delayConnectClientId = clientId;
|
|
1022
|
-
this.blobManager.onConnected().then(() => {
|
|
1023
|
-
// make sure we didn't reconnect before the promise resolved
|
|
1024
|
-
if (this.delayConnectClientId === clientId && !this.disposed) {
|
|
1025
|
-
this.delayConnectClientId = undefined;
|
|
1026
|
-
this.setConnectionStateCore(connected, clientId);
|
|
1027
|
-
}
|
|
1028
|
-
}, (error) => this.closeFn(error));
|
|
1029
|
-
return;
|
|
1030
|
-
}
|
|
1031
1101
|
this.setConnectionStateCore(connected, clientId);
|
|
1032
1102
|
}
|
|
1033
1103
|
setConnectionStateCore(connected, clientId) {
|
|
1034
|
-
(0,
|
|
1104
|
+
(0, core_utils_1.assert)(!this.delayConnectClientId, 0x394 /* connect event delay must be cleared before propagating connect event */);
|
|
1035
1105
|
this.verifyNotClosed();
|
|
1036
1106
|
// There might be no change of state due to Container calling this API after loading runtime.
|
|
1037
1107
|
const changeOfState = this._connected !== connected;
|
|
1038
1108
|
const reconnection = changeOfState && !connected;
|
|
1109
|
+
// We need to flush the ops currently collected by Outbox to preserve original order.
|
|
1110
|
+
// This flush NEEDS to happen before we set the ContainerRuntime to "connected".
|
|
1111
|
+
// We want these ops to get to the PendingStateManager without sending to service and have them return to the Outbox upon calling "replayPendingStates".
|
|
1112
|
+
if (changeOfState && connected) {
|
|
1113
|
+
this.flush();
|
|
1114
|
+
}
|
|
1039
1115
|
this._connected = connected;
|
|
1040
1116
|
if (!connected) {
|
|
1041
1117
|
this._perfSignalData.signalsLost = 0;
|
|
@@ -1043,16 +1119,16 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1043
1119
|
this._perfSignalData.trackingSignalSequenceNumber = undefined;
|
|
1044
1120
|
}
|
|
1045
1121
|
else {
|
|
1046
|
-
(0,
|
|
1122
|
+
(0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x3cd /* Connection is possible only if container exists in storage */);
|
|
1047
1123
|
}
|
|
1048
1124
|
// Fail while disconnected
|
|
1049
1125
|
if (reconnection) {
|
|
1050
1126
|
this.consecutiveReconnects++;
|
|
1051
1127
|
if (!this.shouldContinueReconnecting()) {
|
|
1052
|
-
this.closeFn(
|
|
1128
|
+
this.closeFn(telemetry_utils_1.DataProcessingError.create("Runtime detected too many reconnects with no progress syncing local ops.", "setConnectionState", undefined, {
|
|
1053
1129
|
dataLoss: 1,
|
|
1054
1130
|
attempts: this.consecutiveReconnects,
|
|
1055
|
-
pendingMessages: this.
|
|
1131
|
+
pendingMessages: this.pendingMessagesCount,
|
|
1056
1132
|
}));
|
|
1057
1133
|
return;
|
|
1058
1134
|
}
|
|
@@ -1069,18 +1145,23 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1069
1145
|
}
|
|
1070
1146
|
process(messageArg, local) {
|
|
1071
1147
|
this.verifyNotClosed();
|
|
1072
|
-
// Whether or not the message
|
|
1148
|
+
// Whether or not the message appears to be a runtime message from an up-to-date client.
|
|
1073
1149
|
// It may be a legacy runtime message (ie already unpacked and ContainerMessageType)
|
|
1074
1150
|
// or something different, like a system message.
|
|
1075
|
-
const
|
|
1151
|
+
const modernRuntimeMessage = messageArg.type === protocol_definitions_1.MessageType.Operation;
|
|
1076
1152
|
// Do shallow copy of message, as the processing flow will modify it.
|
|
1077
|
-
const messageCopy =
|
|
1153
|
+
const messageCopy = { ...messageArg };
|
|
1078
1154
|
for (const message of this.remoteMessageProcessor.process(messageCopy)) {
|
|
1079
|
-
this.processCore(message, local,
|
|
1155
|
+
this.processCore(message, local, modernRuntimeMessage);
|
|
1080
1156
|
}
|
|
1081
1157
|
}
|
|
1082
|
-
|
|
1083
|
-
|
|
1158
|
+
/**
|
|
1159
|
+
* Direct the message to the correct subsystem for processing, and implement other side effects
|
|
1160
|
+
* @param message - The unpacked message. Likely a ContainerRuntimeMessage, but could also be a system op
|
|
1161
|
+
* @param local - Did this client send the op?
|
|
1162
|
+
* @param modernRuntimeMessage - Does this appear like a current ContainerRuntimeMessage?
|
|
1163
|
+
*/
|
|
1164
|
+
processCore(message, local, modernRuntimeMessage) {
|
|
1084
1165
|
// Surround the actual processing of the operation with messages to the schedule manager indicating
|
|
1085
1166
|
// the beginning and end. This allows it to emit appropriate events and/or pause the processing of new
|
|
1086
1167
|
// messages once a batch has been fully processed.
|
|
@@ -1088,7 +1169,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1088
1169
|
this._processedClientSequenceNumber = message.clientSequenceNumber;
|
|
1089
1170
|
try {
|
|
1090
1171
|
let localOpMetadata;
|
|
1091
|
-
if (local &&
|
|
1172
|
+
if (local && modernRuntimeMessage && message.type !== ContainerMessageType.ChunkedOp) {
|
|
1092
1173
|
localOpMetadata = this.pendingStateManager.processPendingLocalMessage(message);
|
|
1093
1174
|
}
|
|
1094
1175
|
// If there are no more pending messages after processing a local message,
|
|
@@ -1096,45 +1177,8 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1096
1177
|
if (!this.hasPendingMessages()) {
|
|
1097
1178
|
this.updateDocumentDirtyState(false);
|
|
1098
1179
|
}
|
|
1099
|
-
|
|
1100
|
-
|
|
1101
|
-
case ContainerMessageType.Attach:
|
|
1102
|
-
this.dataStores.processAttachMessage(message, local);
|
|
1103
|
-
break;
|
|
1104
|
-
case ContainerMessageType.Alias:
|
|
1105
|
-
this.processAliasMessage(message, localOpMetadata, local);
|
|
1106
|
-
break;
|
|
1107
|
-
case ContainerMessageType.FluidDataStoreOp:
|
|
1108
|
-
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1109
|
-
break;
|
|
1110
|
-
case ContainerMessageType.BlobAttach:
|
|
1111
|
-
this.blobManager.processBlobAttachOp(message, local);
|
|
1112
|
-
break;
|
|
1113
|
-
case ContainerMessageType.IdAllocation:
|
|
1114
|
-
(0, common_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1115
|
-
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1116
|
-
break;
|
|
1117
|
-
case ContainerMessageType.ChunkedOp:
|
|
1118
|
-
case ContainerMessageType.Rejoin:
|
|
1119
|
-
break;
|
|
1120
|
-
default:
|
|
1121
|
-
if (runtimeMessage) {
|
|
1122
|
-
const error = container_utils_1.DataProcessingError.create(
|
|
1123
|
-
// Former assert 0x3ce
|
|
1124
|
-
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1125
|
-
local,
|
|
1126
|
-
type: message.type,
|
|
1127
|
-
contentType: typeof message.contents,
|
|
1128
|
-
batch: (_a = message.metadata) === null || _a === void 0 ? void 0 : _a.batch,
|
|
1129
|
-
compression: message.compression,
|
|
1130
|
-
});
|
|
1131
|
-
this.closeFn(error);
|
|
1132
|
-
throw error;
|
|
1133
|
-
}
|
|
1134
|
-
}
|
|
1135
|
-
if (runtimeMessage || this.groupedBatchingEnabled) {
|
|
1136
|
-
this.emit("op", message, runtimeMessage);
|
|
1137
|
-
}
|
|
1180
|
+
this.validateAndProcessRuntimeMessage(message, localOpMetadata, local, modernRuntimeMessage);
|
|
1181
|
+
this.emit("op", message, modernRuntimeMessage);
|
|
1138
1182
|
this.scheduleManager.afterOpProcessing(undefined, message);
|
|
1139
1183
|
if (local) {
|
|
1140
1184
|
// If we have processed a local op, this means that the container is
|
|
@@ -1148,8 +1192,59 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1148
1192
|
throw e;
|
|
1149
1193
|
}
|
|
1150
1194
|
}
|
|
1151
|
-
|
|
1152
|
-
|
|
1195
|
+
/**
|
|
1196
|
+
* Assuming the given message is also a ContainerRuntimeMessage,
|
|
1197
|
+
* checks its type and dispatches the message to the appropriate handler in the runtime.
|
|
1198
|
+
* Throws a DataProcessingError if the message doesn't conform to the ContainerRuntimeMessage type.
|
|
1199
|
+
*/
|
|
1200
|
+
validateAndProcessRuntimeMessage(message, localOpMetadata, local, expectRuntimeMessageType) {
|
|
1201
|
+
// Optimistically extract ContainerRuntimeMessage-specific props from the message
|
|
1202
|
+
const { type: maybeContainerMessageType, compatDetails } = message;
|
|
1203
|
+
switch (maybeContainerMessageType) {
|
|
1204
|
+
case ContainerMessageType.Attach:
|
|
1205
|
+
this.dataStores.processAttachMessage(message, local);
|
|
1206
|
+
break;
|
|
1207
|
+
case ContainerMessageType.Alias:
|
|
1208
|
+
this.dataStores.processAliasMessage(message, localOpMetadata, local);
|
|
1209
|
+
break;
|
|
1210
|
+
case ContainerMessageType.FluidDataStoreOp:
|
|
1211
|
+
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1212
|
+
break;
|
|
1213
|
+
case ContainerMessageType.BlobAttach:
|
|
1214
|
+
this.blobManager.processBlobAttachOp(message, local);
|
|
1215
|
+
break;
|
|
1216
|
+
case ContainerMessageType.IdAllocation:
|
|
1217
|
+
(0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67c /* IdCompressor should be defined if enabled */);
|
|
1218
|
+
this.idCompressor.finalizeCreationRange(message.contents);
|
|
1219
|
+
break;
|
|
1220
|
+
case ContainerMessageType.ChunkedOp:
|
|
1221
|
+
case ContainerMessageType.Rejoin:
|
|
1222
|
+
break;
|
|
1223
|
+
default: {
|
|
1224
|
+
// If we didn't necessarily expect a runtime message type, then no worries - just return
|
|
1225
|
+
// e.g. this case applies to system ops, or legacy ops that would have fallen into the above cases anyway.
|
|
1226
|
+
if (!expectRuntimeMessageType) {
|
|
1227
|
+
return;
|
|
1228
|
+
}
|
|
1229
|
+
const compatBehavior = compatDetails?.behavior;
|
|
1230
|
+
if (!compatBehaviorAllowsMessageType(maybeContainerMessageType, compatBehavior)) {
|
|
1231
|
+
const error = telemetry_utils_1.DataProcessingError.create(
|
|
1232
|
+
// Former assert 0x3ce
|
|
1233
|
+
"Runtime message of unknown type", "OpProcessing", message, {
|
|
1234
|
+
local,
|
|
1235
|
+
messageDetails: JSON.stringify({
|
|
1236
|
+
type: message.type,
|
|
1237
|
+
contentType: typeof message.contents,
|
|
1238
|
+
compatBehavior,
|
|
1239
|
+
batch: message.metadata?.batch,
|
|
1240
|
+
compression: message.compression,
|
|
1241
|
+
}),
|
|
1242
|
+
});
|
|
1243
|
+
this.closeFn(error);
|
|
1244
|
+
throw error;
|
|
1245
|
+
}
|
|
1246
|
+
}
|
|
1247
|
+
}
|
|
1153
1248
|
}
|
|
1154
1249
|
/**
|
|
1155
1250
|
* Emits the Signal event and update the perf signal data.
|
|
@@ -1157,7 +1252,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1157
1252
|
*/
|
|
1158
1253
|
sendSignalTelemetryEvent(clientSignalSequenceNumber) {
|
|
1159
1254
|
const duration = Date.now() - this._perfSignalData.signalTimestamp;
|
|
1160
|
-
this.logger.sendPerformanceEvent({
|
|
1255
|
+
this.mc.logger.sendPerformanceEvent({
|
|
1161
1256
|
eventName: "SignalLatency",
|
|
1162
1257
|
duration,
|
|
1163
1258
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1180,7 +1275,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1180
1275
|
this._perfSignalData.trackingSignalSequenceNumber) {
|
|
1181
1276
|
this._perfSignalData.signalsLost++;
|
|
1182
1277
|
this._perfSignalData.trackingSignalSequenceNumber = undefined;
|
|
1183
|
-
this.logger.sendErrorEvent({
|
|
1278
|
+
this.mc.logger.sendErrorEvent({
|
|
1184
1279
|
eventName: "SignalLost",
|
|
1185
1280
|
type: envelope.contents.type,
|
|
1186
1281
|
signalsLost: this._perfSignalData.signalsLost,
|
|
@@ -1204,6 +1299,12 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1204
1299
|
}
|
|
1205
1300
|
this.dataStores.processSignal(envelope.address, transformed, local);
|
|
1206
1301
|
}
|
|
1302
|
+
/**
|
|
1303
|
+
* Returns the runtime of the data store.
|
|
1304
|
+
* @param id - Id supplied during creating the data store.
|
|
1305
|
+
* @param wait - True if you want to wait for it.
|
|
1306
|
+
* @deprecated - Use getAliasedDataStoreEntryPoint instead to get an aliased data store's entry point.
|
|
1307
|
+
*/
|
|
1207
1308
|
async getRootDataStore(id, wait = true) {
|
|
1208
1309
|
return this.getRootDataStoreChannel(id, wait);
|
|
1209
1310
|
}
|
|
@@ -1211,7 +1312,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1211
1312
|
await this.dataStores.waitIfPendingAlias(id);
|
|
1212
1313
|
const internalId = this.internalId(id);
|
|
1213
1314
|
const context = await this.dataStores.getDataStore(internalId, { wait });
|
|
1214
|
-
(0,
|
|
1315
|
+
(0, core_utils_1.assert)(await context.isRoot(), 0x12b /* "did not get root data store" */);
|
|
1215
1316
|
return context.realize();
|
|
1216
1317
|
}
|
|
1217
1318
|
/**
|
|
@@ -1219,9 +1320,9 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1219
1320
|
* This method is expected to be called at the end of a batch.
|
|
1220
1321
|
*/
|
|
1221
1322
|
flush() {
|
|
1222
|
-
(0,
|
|
1323
|
+
(0, core_utils_1.assert)(this._orderSequentiallyCalls === 0, 0x24c /* "Cannot call `flush()` from `orderSequentially`'s callback" */);
|
|
1223
1324
|
this.outbox.flush();
|
|
1224
|
-
(0,
|
|
1325
|
+
(0, core_utils_1.assert)(this.outbox.isEmpty, 0x3cf /* reentrancy */);
|
|
1225
1326
|
}
|
|
1226
1327
|
orderSequentially(callback) {
|
|
1227
1328
|
let checkpoint;
|
|
@@ -1244,7 +1345,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1244
1345
|
}
|
|
1245
1346
|
catch (err) {
|
|
1246
1347
|
const error2 = (0, telemetry_utils_1.wrapError)(err, (message) => {
|
|
1247
|
-
return
|
|
1348
|
+
return telemetry_utils_1.DataProcessingError.create(`RollbackError: ${message}`, "checkpointRollback", undefined);
|
|
1248
1349
|
});
|
|
1249
1350
|
this.closeFn(error2);
|
|
1250
1351
|
throw error2;
|
|
@@ -1252,7 +1353,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1252
1353
|
}
|
|
1253
1354
|
else {
|
|
1254
1355
|
// pre-0.58 error message: orderSequentiallyCallbackException
|
|
1255
|
-
this.closeFn(new
|
|
1356
|
+
this.closeFn(new telemetry_utils_1.GenericError("orderSequentially callback exception", error));
|
|
1256
1357
|
}
|
|
1257
1358
|
throw error; // throw the original error for the consumer of the runtime
|
|
1258
1359
|
}
|
|
@@ -1265,29 +1366,49 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1265
1366
|
}
|
|
1266
1367
|
return result;
|
|
1267
1368
|
}
|
|
1268
|
-
|
|
1269
|
-
|
|
1270
|
-
|
|
1369
|
+
/**
|
|
1370
|
+
* Returns the aliased data store's entryPoint, given the alias.
|
|
1371
|
+
* @param alias - The alias for the data store.
|
|
1372
|
+
* @returns The data store's entry point ({@link @fluidframework/core-interfaces#IFluidHandle}) if it exists and is aliased.
|
|
1373
|
+
* Returns undefined if no data store has been assigned the given alias.
|
|
1374
|
+
*/
|
|
1375
|
+
async getAliasedDataStoreEntryPoint(alias) {
|
|
1376
|
+
await this.dataStores.waitIfPendingAlias(alias);
|
|
1377
|
+
const internalId = this.internalId(alias);
|
|
1378
|
+
const context = await this.dataStores.getDataStoreIfAvailable(internalId, { wait: false });
|
|
1379
|
+
// If the data store is not available or not an alias, return undefined.
|
|
1380
|
+
if (context === undefined || !(await context.isRoot())) {
|
|
1381
|
+
return undefined;
|
|
1382
|
+
}
|
|
1383
|
+
const channel = await context.realize();
|
|
1384
|
+
if (channel.entryPoint === undefined) {
|
|
1385
|
+
throw new telemetry_utils_1.UsageError("entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint");
|
|
1386
|
+
}
|
|
1387
|
+
return channel.entryPoint;
|
|
1271
1388
|
}
|
|
1272
1389
|
createDetachedRootDataStore(pkg, rootDataStoreId) {
|
|
1273
1390
|
if (rootDataStoreId.includes("/")) {
|
|
1274
|
-
throw new
|
|
1391
|
+
throw new telemetry_utils_1.UsageError(`Id cannot contain slashes: '${rootDataStoreId}'`);
|
|
1275
1392
|
}
|
|
1276
1393
|
return this.dataStores.createDetachedDataStoreCore(pkg, true, rootDataStoreId);
|
|
1277
1394
|
}
|
|
1278
1395
|
createDetachedDataStore(pkg) {
|
|
1279
1396
|
return this.dataStores.createDetachedDataStoreCore(pkg, false);
|
|
1280
1397
|
}
|
|
1281
|
-
async
|
|
1282
|
-
const
|
|
1283
|
-
|
|
1284
|
-
.
|
|
1285
|
-
|
|
1398
|
+
async createDataStore(pkg) {
|
|
1399
|
+
const id = (0, uuid_1.v4)();
|
|
1400
|
+
return (0, dataStore_1.channelToDataStore)(await this.dataStores
|
|
1401
|
+
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id)
|
|
1402
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1286
1403
|
}
|
|
1287
|
-
|
|
1288
|
-
|
|
1404
|
+
/**
|
|
1405
|
+
* @deprecated 0.16 Issue #1537, #3631
|
|
1406
|
+
* @internal
|
|
1407
|
+
*/
|
|
1408
|
+
async _createDataStoreWithProps(pkg, props, id = (0, uuid_1.v4)()) {
|
|
1409
|
+
return (0, dataStore_1.channelToDataStore)(await this.dataStores
|
|
1289
1410
|
._createFluidDataStoreContext(Array.isArray(pkg) ? pkg : [pkg], id, props)
|
|
1290
|
-
.realize();
|
|
1411
|
+
.realize(), id, this, this.dataStores, this.mc.logger);
|
|
1291
1412
|
}
|
|
1292
1413
|
canSendOps() {
|
|
1293
1414
|
// Note that the real (non-proxy) delta manager is needed here to get the readonly info. This is because
|
|
@@ -1301,11 +1422,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1301
1422
|
return this.flushMode !== runtime_definitions_1.FlushMode.Immediate || this._orderSequentiallyCalls !== 0;
|
|
1302
1423
|
}
|
|
1303
1424
|
getQuorum() {
|
|
1304
|
-
return this.
|
|
1425
|
+
return this._quorum;
|
|
1305
1426
|
}
|
|
1306
1427
|
getAudience() {
|
|
1307
|
-
|
|
1308
|
-
return this.context.audience;
|
|
1428
|
+
return this._audience;
|
|
1309
1429
|
}
|
|
1310
1430
|
/**
|
|
1311
1431
|
* Returns true of container is dirty, i.e. there are some pending local changes that
|
|
@@ -1314,7 +1434,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1314
1434
|
get isDirty() {
|
|
1315
1435
|
return this.dirtyContainer;
|
|
1316
1436
|
}
|
|
1317
|
-
isContainerMessageDirtyable(type, contents) {
|
|
1437
|
+
isContainerMessageDirtyable({ type, contents }) {
|
|
1318
1438
|
// For legacy purposes, exclude the old built-in AgentScheduler from dirty consideration as a special-case.
|
|
1319
1439
|
// Ultimately we should have no special-cases from the ContainerRuntime's perspective.
|
|
1320
1440
|
if (type === ContainerMessageType.Attach) {
|
|
@@ -1354,18 +1474,18 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1354
1474
|
submitSignal(type, content) {
|
|
1355
1475
|
this.verifyNotClosed();
|
|
1356
1476
|
const envelope = this.createNewSignalEnvelope(undefined /* address */, type, content);
|
|
1357
|
-
return this.
|
|
1477
|
+
return this.submitSignalFn(envelope);
|
|
1358
1478
|
}
|
|
1359
1479
|
submitDataStoreSignal(address, type, content) {
|
|
1360
1480
|
const envelope = this.createNewSignalEnvelope(address, type, content);
|
|
1361
|
-
return this.
|
|
1481
|
+
return this.submitSignalFn(envelope);
|
|
1362
1482
|
}
|
|
1363
1483
|
setAttachState(attachState) {
|
|
1364
1484
|
if (attachState === container_definitions_1.AttachState.Attaching) {
|
|
1365
|
-
(0,
|
|
1485
|
+
(0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attaching, 0x12d /* "Container Context should already be in attaching state" */);
|
|
1366
1486
|
}
|
|
1367
1487
|
else {
|
|
1368
|
-
(0,
|
|
1488
|
+
(0, core_utils_1.assert)(this.attachState === container_definitions_1.AttachState.Attached, 0x12e /* "Container Context should already be in attached state" */);
|
|
1369
1489
|
this.emit("attached");
|
|
1370
1490
|
}
|
|
1371
1491
|
if (attachState === container_definitions_1.AttachState.Attached && !this.hasPendingMessages()) {
|
|
@@ -1391,22 +1511,17 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1391
1511
|
this.addContainerStateToSummary(summarizeResult, true /* fullTree */, false /* trackState */, telemetryContext);
|
|
1392
1512
|
return summarizeResult.summary;
|
|
1393
1513
|
}
|
|
1394
|
-
async getAbsoluteUrl(relativeUrl) {
|
|
1395
|
-
if (this.context.getAbsoluteUrl === undefined) {
|
|
1396
|
-
throw new Error("Driver does not implement getAbsoluteUrl");
|
|
1397
|
-
}
|
|
1398
|
-
if (this.attachState !== container_definitions_1.AttachState.Attached) {
|
|
1399
|
-
return undefined;
|
|
1400
|
-
}
|
|
1401
|
-
return this.context.getAbsoluteUrl(relativeUrl);
|
|
1402
|
-
}
|
|
1403
1514
|
async summarizeInternal(fullTree, trackState, telemetryContext) {
|
|
1404
1515
|
const summarizeResult = await this.dataStores.summarize(fullTree, trackState, telemetryContext);
|
|
1405
1516
|
// Wrap data store summaries in .channels subtree.
|
|
1406
1517
|
(0, summary_1.wrapSummaryInChannelsTree)(summarizeResult);
|
|
1407
1518
|
const pathPartsForChildren = [runtime_definitions_1.channelsTreeName];
|
|
1408
1519
|
this.addContainerStateToSummary(summarizeResult, fullTree, trackState, telemetryContext);
|
|
1409
|
-
return
|
|
1520
|
+
return {
|
|
1521
|
+
...summarizeResult,
|
|
1522
|
+
id: "",
|
|
1523
|
+
pathPartsForChildren,
|
|
1524
|
+
};
|
|
1410
1525
|
}
|
|
1411
1526
|
/**
|
|
1412
1527
|
* Returns a summary of the runtime at the current sequence number.
|
|
@@ -1424,16 +1539,15 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1424
1539
|
runSweep,
|
|
1425
1540
|
});
|
|
1426
1541
|
try {
|
|
1427
|
-
let gcStats;
|
|
1428
1542
|
if (runGC) {
|
|
1429
|
-
|
|
1543
|
+
await this.collectGarbage({ logger: summaryLogger, runSweep, fullGC }, telemetryContext);
|
|
1430
1544
|
}
|
|
1431
1545
|
const { stats, summary } = await this.summarizerNode.summarize(fullTree, trackState, telemetryContext);
|
|
1432
|
-
(0,
|
|
1433
|
-
return { stats, summary
|
|
1546
|
+
(0, core_utils_1.assert)(summary.type === protocol_definitions_1.SummaryType.Tree, 0x12f /* "Container Runtime's summarize should always return a tree" */);
|
|
1547
|
+
return { stats, summary };
|
|
1434
1548
|
}
|
|
1435
1549
|
finally {
|
|
1436
|
-
this.logger.sendTelemetryEvent({
|
|
1550
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1437
1551
|
eventName: "SummarizeTelemetry",
|
|
1438
1552
|
details: telemetryContext.serialize(),
|
|
1439
1553
|
});
|
|
@@ -1495,7 +1609,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1495
1609
|
/**
|
|
1496
1610
|
* After GC has run and identified nodes that are sweep ready, this is called to delete the sweep ready nodes.
|
|
1497
1611
|
* @param sweepReadyRoutes - The routes of nodes that are sweep ready and should be deleted.
|
|
1498
|
-
* @returns
|
|
1612
|
+
* @returns The routes of nodes that were deleted.
|
|
1499
1613
|
*/
|
|
1500
1614
|
deleteSweepReadyNodes(sweepReadyRoutes) {
|
|
1501
1615
|
const { dataStoreRoutes, blobManagerRoutes } = this.getDataStoreAndBlobManagerRoutes(sweepReadyRoutes);
|
|
@@ -1515,21 +1629,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1515
1629
|
* Returns a server generated referenced timestamp to be used to track unreferenced nodes by GC.
|
|
1516
1630
|
*/
|
|
1517
1631
|
getCurrentReferenceTimestampMs() {
|
|
1518
|
-
var _a, _b, _c;
|
|
1519
1632
|
// Use the timestamp of the last message seen by this client as that is server generated. If no messages have
|
|
1520
1633
|
// been processed, use the timestamp of the message from the last summary.
|
|
1521
|
-
return
|
|
1634
|
+
return this.deltaManager.lastMessage?.timestamp ?? this.messageAtLastSummary?.timestamp;
|
|
1522
1635
|
}
|
|
1523
1636
|
/**
|
|
1524
1637
|
* Returns the type of the GC node. Currently, there are nodes that belong to the root ("/"), data stores or
|
|
1525
1638
|
* blob manager.
|
|
1526
1639
|
*/
|
|
1527
1640
|
getNodeType(nodePath) {
|
|
1528
|
-
var _a;
|
|
1529
1641
|
if (this.isBlobPath(nodePath)) {
|
|
1530
1642
|
return gc_1.GCNodeType.Blob;
|
|
1531
1643
|
}
|
|
1532
|
-
return
|
|
1644
|
+
return this.dataStores.getGCNodeType(nodePath) ?? gc_1.GCNodeType.Other;
|
|
1533
1645
|
}
|
|
1534
1646
|
/**
|
|
1535
1647
|
* Called by GC to retrieve the package path of the node with the given path. The node should belong to a
|
|
@@ -1543,7 +1655,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1543
1655
|
case gc_1.GCNodeType.SubDataStore:
|
|
1544
1656
|
return this.dataStores.getDataStorePackagePath(nodePath);
|
|
1545
1657
|
default:
|
|
1546
|
-
(0,
|
|
1658
|
+
(0, core_utils_1.assert)(false, 0x2de /* "Package path requested for unsupported node type." */);
|
|
1547
1659
|
}
|
|
1548
1660
|
}
|
|
1549
1661
|
/**
|
|
@@ -1559,7 +1671,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1559
1671
|
/**
|
|
1560
1672
|
* From a given list of routes, separate and return routes that belong to blob manager and data stores.
|
|
1561
1673
|
* @param routes - A list of routes that can belong to data stores or blob manager.
|
|
1562
|
-
* @returns
|
|
1674
|
+
* @returns Two route lists - One that contains routes for blob manager and another one that contains routes
|
|
1563
1675
|
* for data stores.
|
|
1564
1676
|
*/
|
|
1565
1677
|
getDataStoreAndBlobManagerRoutes(routes) {
|
|
@@ -1600,23 +1712,62 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1600
1712
|
* @param options - options controlling how the summary is generated or submitted
|
|
1601
1713
|
*/
|
|
1602
1714
|
async submitSummary(options) {
|
|
1603
|
-
|
|
1604
|
-
const { fullTree = false, refreshLatestAck, summaryLogger } = options;
|
|
1715
|
+
const { fullTree = false, finalAttempt = false, refreshLatestAck, summaryLogger } = options;
|
|
1605
1716
|
// The summary number for this summary. This will be updated during the summary process, so get it now and
|
|
1606
1717
|
// use it for all events logged during this summary.
|
|
1607
1718
|
const summaryNumber = this.nextSummaryNumber;
|
|
1608
|
-
const summaryNumberLogger = telemetry_utils_1.
|
|
1609
|
-
|
|
1719
|
+
const summaryNumberLogger = (0, telemetry_utils_1.createChildLogger)({
|
|
1720
|
+
logger: summaryLogger,
|
|
1721
|
+
properties: {
|
|
1722
|
+
all: { summaryNumber },
|
|
1723
|
+
},
|
|
1610
1724
|
});
|
|
1611
|
-
(0,
|
|
1725
|
+
(0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d1 /* Can't trigger summary in the middle of a batch */);
|
|
1612
1726
|
let latestSnapshotVersionId;
|
|
1613
1727
|
if (refreshLatestAck) {
|
|
1614
|
-
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer(telemetry_utils_1.
|
|
1728
|
+
const latestSnapshotInfo = await this.refreshLatestSummaryAckFromServer((0, telemetry_utils_1.createChildLogger)({
|
|
1729
|
+
logger: summaryNumberLogger,
|
|
1730
|
+
properties: { all: { safeSummary: true } },
|
|
1731
|
+
}));
|
|
1615
1732
|
const latestSnapshotRefSeq = latestSnapshotInfo.latestSnapshotRefSeq;
|
|
1616
1733
|
latestSnapshotVersionId = latestSnapshotInfo.latestSnapshotVersionId;
|
|
1617
1734
|
// We might need to catch up to the latest summary's reference sequence number before pausing.
|
|
1618
1735
|
await this.waitForDeltaManagerToCatchup(latestSnapshotRefSeq, summaryNumberLogger);
|
|
1619
1736
|
}
|
|
1737
|
+
// If there are pending (unacked ops), the summary will not be eventual consistent and it may even be
|
|
1738
|
+
// incorrect. So, wait for the container to be saved with a timeout. If the container is not saved
|
|
1739
|
+
// within the timeout, check if it should be failed or can continue.
|
|
1740
|
+
if (this.validateSummaryBeforeUpload && this.hasPendingMessages()) {
|
|
1741
|
+
const countBefore = this.pendingMessagesCount;
|
|
1742
|
+
// The timeout for waiting for pending ops can be overridden via configurations.
|
|
1743
|
+
const pendingOpsTimeout = this.mc.config.getNumber("Fluid.Summarizer.waitForPendingOpsTimeoutMs") ??
|
|
1744
|
+
exports.defaultPendingOpsWaitTimeoutMs;
|
|
1745
|
+
await new Promise((resolve, reject) => {
|
|
1746
|
+
const timeoutId = setTimeout(() => resolve(), pendingOpsTimeout);
|
|
1747
|
+
this.once("saved", () => {
|
|
1748
|
+
clearTimeout(timeoutId);
|
|
1749
|
+
resolve();
|
|
1750
|
+
});
|
|
1751
|
+
this.once("dispose", () => {
|
|
1752
|
+
clearTimeout(timeoutId);
|
|
1753
|
+
reject(new Error("Runtime is disposed while summarizing"));
|
|
1754
|
+
});
|
|
1755
|
+
});
|
|
1756
|
+
// Log that there are pending ops while summarizing. This will help us gather data on how often this
|
|
1757
|
+
// happens, whether we attempted to wait for these ops to be acked and what was the result.
|
|
1758
|
+
summaryNumberLogger.sendTelemetryEvent({
|
|
1759
|
+
eventName: "PendingOpsWhileSummarizing",
|
|
1760
|
+
saved: this.hasPendingMessages() ? false : true,
|
|
1761
|
+
timeout: pendingOpsTimeout,
|
|
1762
|
+
countBefore,
|
|
1763
|
+
countAfter: this.pendingMessagesCount,
|
|
1764
|
+
});
|
|
1765
|
+
// There could still be pending ops. Check if summary should fail or continue.
|
|
1766
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, this.deltaManager.lastSequenceNumber, this.deltaManager.minimumSequenceNumber, finalAttempt, true /* beforeSummaryGeneration */);
|
|
1767
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1768
|
+
return pendingMessagesFailResult;
|
|
1769
|
+
}
|
|
1770
|
+
}
|
|
1620
1771
|
const shouldPauseInboundSignal = this.mc.config.getBoolean("Fluid.ContainerRuntime.SubmitSummary.disableInboundSignalPause") !== true;
|
|
1621
1772
|
let summaryRefSeqNum;
|
|
1622
1773
|
try {
|
|
@@ -1631,7 +1782,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1631
1782
|
this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
|
|
1632
1783
|
// Helper function to check whether we should still continue between each async step.
|
|
1633
1784
|
const checkContinue = () => {
|
|
1634
|
-
var _a;
|
|
1635
1785
|
// Do not check for loss of connectivity directly! Instead leave it up to
|
|
1636
1786
|
// RunWhileConnectedCoordinator to control policy in a single place.
|
|
1637
1787
|
// This will allow easier change of design if we chose to. For example, we may chose to allow
|
|
@@ -1644,7 +1794,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1644
1794
|
// That said, we rely on submitSystemMessage() that today only works in connected state.
|
|
1645
1795
|
// So if we fail here, it either means that RunWhileConnectedCoordinator does not work correctly,
|
|
1646
1796
|
// OR that design changed and we need to remove this check and fix submitSystemMessage.
|
|
1647
|
-
(0,
|
|
1797
|
+
(0, core_utils_1.assert)(this.connected, 0x258 /* "connected" */);
|
|
1648
1798
|
// Ensure that lastSequenceNumber has not changed after pausing.
|
|
1649
1799
|
// We need the summary op's reference sequence number to match our summary sequence number,
|
|
1650
1800
|
// otherwise we'll get the wrong sequence number stamped on the summary's .protocol attributes.
|
|
@@ -1654,7 +1804,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1654
1804
|
error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
|
|
1655
1805
|
};
|
|
1656
1806
|
}
|
|
1657
|
-
(0,
|
|
1807
|
+
(0, core_utils_1.assert)(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber, 0x395 /* it's one and the same thing */);
|
|
1658
1808
|
if (lastAck !== this.summaryCollection.latestAck) {
|
|
1659
1809
|
return {
|
|
1660
1810
|
continue: false,
|
|
@@ -1672,7 +1822,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1672
1822
|
error: continueResult.error,
|
|
1673
1823
|
};
|
|
1674
1824
|
}
|
|
1675
|
-
const trace =
|
|
1825
|
+
const trace = client_utils_1.Trace.start();
|
|
1676
1826
|
let summarizeResult;
|
|
1677
1827
|
// If the GC state needs to be reset, we need to force a full tree summary and update the unreferenced
|
|
1678
1828
|
// state of all the nodes.
|
|
@@ -1693,6 +1843,25 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1693
1843
|
error,
|
|
1694
1844
|
};
|
|
1695
1845
|
}
|
|
1846
|
+
// If validateSummaryBeforeUpload is true, validate that the summary generated is correct before uploading.
|
|
1847
|
+
if (this.validateSummaryBeforeUpload) {
|
|
1848
|
+
// Validate that the summaries generated by summarize nodes is correct.
|
|
1849
|
+
const validateResult = this.summarizerNode.validateSummary();
|
|
1850
|
+
if (!validateResult.success) {
|
|
1851
|
+
const { success, ...loggingProps } = validateResult;
|
|
1852
|
+
const error = new summary_1.RetriableSummaryError(validateResult.reason, validateResult.retryAfterSeconds, { ...loggingProps });
|
|
1853
|
+
return {
|
|
1854
|
+
stage: "base",
|
|
1855
|
+
referenceSequenceNumber: summaryRefSeqNum,
|
|
1856
|
+
minimumSequenceNumber,
|
|
1857
|
+
error,
|
|
1858
|
+
};
|
|
1859
|
+
}
|
|
1860
|
+
const pendingMessagesFailResult = await this.shouldFailSummaryOnPendingOps(summaryNumberLogger, summaryRefSeqNum, minimumSequenceNumber, finalAttempt, false /* beforeSummaryGeneration */);
|
|
1861
|
+
if (pendingMessagesFailResult !== undefined) {
|
|
1862
|
+
return pendingMessagesFailResult;
|
|
1863
|
+
}
|
|
1864
|
+
}
|
|
1696
1865
|
const { summary: summaryTree, stats: partialStats } = summarizeResult;
|
|
1697
1866
|
// Now that we have generated the summary, update the message at last summary to the last message processed.
|
|
1698
1867
|
this.messageAtLastSummary = this.deltaManager.lastMessage;
|
|
@@ -1700,12 +1869,20 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1700
1869
|
// Because handles are unchanged dataStores in the current logic,
|
|
1701
1870
|
// summarized dataStore count is total dataStore count minus handle count
|
|
1702
1871
|
const dataStoreTree = summaryTree.tree[runtime_definitions_1.channelsTreeName];
|
|
1703
|
-
(0,
|
|
1872
|
+
(0, core_utils_1.assert)(dataStoreTree.type === protocol_definitions_1.SummaryType.Tree, 0x1fc /* "summary is not a tree" */);
|
|
1704
1873
|
const handleCount = Object.values(dataStoreTree.tree).filter((value) => value.type === protocol_definitions_1.SummaryType.Handle).length;
|
|
1705
1874
|
const gcSummaryTreeStats = summaryTree.tree[runtime_definitions_1.gcTreeKey]
|
|
1706
1875
|
? (0, runtime_utils_1.calculateStats)(summaryTree.tree[runtime_definitions_1.gcTreeKey])
|
|
1707
1876
|
: undefined;
|
|
1708
|
-
const summaryStats =
|
|
1877
|
+
const summaryStats = {
|
|
1878
|
+
dataStoreCount: this.dataStores.size,
|
|
1879
|
+
summarizedDataStoreCount: this.dataStores.size - handleCount,
|
|
1880
|
+
gcStateUpdatedDataStoreCount: this.garbageCollector.updatedDSCountSinceLastSummary,
|
|
1881
|
+
gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
|
|
1882
|
+
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
1883
|
+
summaryNumber,
|
|
1884
|
+
...partialStats,
|
|
1885
|
+
};
|
|
1709
1886
|
const generateSummaryData = {
|
|
1710
1887
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1711
1888
|
minimumSequenceNumber,
|
|
@@ -1716,7 +1893,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1716
1893
|
};
|
|
1717
1894
|
continueResult = checkContinue();
|
|
1718
1895
|
if (!continueResult.continue) {
|
|
1719
|
-
return
|
|
1896
|
+
return { stage: "generate", ...generateSummaryData, error: continueResult.error };
|
|
1720
1897
|
}
|
|
1721
1898
|
// It may happen that the lastAck it not correct due to missing summaryAck in case of single commit
|
|
1722
1899
|
// summary. So if the previous summarizer closes just after submitting the summary and before
|
|
@@ -1724,7 +1901,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1724
1901
|
// latestSnapshotVersionId from storage and it does not match with the lastAck ackHandle, then use
|
|
1725
1902
|
// the one fetched from storage as parent as that is the latest.
|
|
1726
1903
|
let summaryContext;
|
|
1727
|
-
if (
|
|
1904
|
+
if (lastAck?.summaryAck.contents.handle !== latestSnapshotVersionId &&
|
|
1728
1905
|
latestSnapshotVersionId !== undefined) {
|
|
1729
1906
|
summaryContext = {
|
|
1730
1907
|
proposalHandle: undefined,
|
|
@@ -1735,7 +1912,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1735
1912
|
else if (lastAck === undefined) {
|
|
1736
1913
|
summaryContext = {
|
|
1737
1914
|
proposalHandle: undefined,
|
|
1738
|
-
ackHandle:
|
|
1915
|
+
ackHandle: this.loadedFromVersionId,
|
|
1739
1916
|
referenceSequenceNumber: summaryRefSeqNum,
|
|
1740
1917
|
};
|
|
1741
1918
|
}
|
|
@@ -1751,7 +1928,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1751
1928
|
handle = await this.storage.uploadSummaryWithContext(summarizeResult.summary, summaryContext);
|
|
1752
1929
|
}
|
|
1753
1930
|
catch (error) {
|
|
1754
|
-
return
|
|
1931
|
+
return { stage: "generate", ...generateSummaryData, error };
|
|
1755
1932
|
}
|
|
1756
1933
|
const parent = summaryContext.ackHandle;
|
|
1757
1934
|
const summaryMessage = {
|
|
@@ -1761,27 +1938,42 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1761
1938
|
message,
|
|
1762
1939
|
parents: parent ? [parent] : [],
|
|
1763
1940
|
};
|
|
1764
|
-
const uploadData =
|
|
1941
|
+
const uploadData = {
|
|
1942
|
+
...generateSummaryData,
|
|
1943
|
+
handle,
|
|
1944
|
+
uploadDuration: trace.trace().duration,
|
|
1945
|
+
};
|
|
1765
1946
|
continueResult = checkContinue();
|
|
1766
1947
|
if (!continueResult.continue) {
|
|
1767
|
-
return
|
|
1948
|
+
return { stage: "upload", ...uploadData, error: continueResult.error };
|
|
1768
1949
|
}
|
|
1769
1950
|
let clientSequenceNumber;
|
|
1770
1951
|
try {
|
|
1771
1952
|
clientSequenceNumber = this.submitSummaryMessage(summaryMessage, summaryRefSeqNum);
|
|
1772
1953
|
}
|
|
1773
1954
|
catch (error) {
|
|
1774
|
-
return
|
|
1955
|
+
return { stage: "upload", ...uploadData, error };
|
|
1956
|
+
}
|
|
1957
|
+
const submitData = {
|
|
1958
|
+
stage: "submit",
|
|
1959
|
+
...uploadData,
|
|
1960
|
+
clientSequenceNumber,
|
|
1961
|
+
submitOpDuration: trace.trace().duration,
|
|
1962
|
+
};
|
|
1963
|
+
try {
|
|
1964
|
+
// If validateSummaryBeforeUpload is false, the summary should be validated in this step.
|
|
1965
|
+
this.summarizerNode.completeSummary(handle, !this.validateSummaryBeforeUpload /* validate */);
|
|
1966
|
+
}
|
|
1967
|
+
catch (error) {
|
|
1968
|
+
return { stage: "upload", ...uploadData, error };
|
|
1775
1969
|
}
|
|
1776
|
-
const submitData = Object.assign(Object.assign({ stage: "submit" }, uploadData), { clientSequenceNumber, submitOpDuration: trace.trace().duration });
|
|
1777
|
-
this.summarizerNode.completeSummary(handle);
|
|
1778
1970
|
return submitData;
|
|
1779
1971
|
}
|
|
1780
1972
|
finally {
|
|
1781
1973
|
// Cleanup wip summary in case of failure
|
|
1782
1974
|
this.summarizerNode.clearSummary();
|
|
1783
1975
|
// ! This needs to happen before we resume inbound queues to ensure heuristics are tracked correctly
|
|
1784
|
-
|
|
1976
|
+
this._summarizer?.recordSummaryAttempt?.(summaryRefSeqNum);
|
|
1785
1977
|
// Restart the delta manager
|
|
1786
1978
|
this.deltaManager.inbound.resume();
|
|
1787
1979
|
if (shouldPauseInboundSignal) {
|
|
@@ -1789,16 +1981,63 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1789
1981
|
}
|
|
1790
1982
|
}
|
|
1791
1983
|
}
|
|
1984
|
+
/**
|
|
1985
|
+
* This helper is called during summarization. If there are pending ops, it will return a failed summarize result
|
|
1986
|
+
* (IBaseSummarizeResult) unless this is the final summarize attempt and SkipFailingIncorrectSummary option is set.
|
|
1987
|
+
* @param logger - The logger to be used for sending telemetry.
|
|
1988
|
+
* @param referenceSequenceNumber - The reference sequence number of the summary attempt.
|
|
1989
|
+
* @param minimumSequenceNumber - The minimum sequence number of the summary attempt.
|
|
1990
|
+
* @param finalAttempt - Whether this is the final summary attempt.
|
|
1991
|
+
* @param beforeSummaryGeneration - Whether this is called before summary generation or after.
|
|
1992
|
+
* @returns failed summarize result (IBaseSummarizeResult) if summary should be failed, undefined otherwise.
|
|
1993
|
+
*/
|
|
1994
|
+
async shouldFailSummaryOnPendingOps(logger, referenceSequenceNumber, minimumSequenceNumber, finalAttempt, beforeSummaryGeneration) {
|
|
1995
|
+
if (!this.hasPendingMessages()) {
|
|
1996
|
+
return;
|
|
1997
|
+
}
|
|
1998
|
+
// If "SkipFailingIncorrectSummary" option is true, don't fail the summary in the last attempt.
|
|
1999
|
+
// This is a fallback to make progress in documents where there are consistently pending ops in
|
|
2000
|
+
// the summarizer.
|
|
2001
|
+
if (finalAttempt &&
|
|
2002
|
+
this.mc.config.getBoolean("Fluid.Summarizer.SkipFailingIncorrectSummary")) {
|
|
2003
|
+
const error = telemetry_utils_1.DataProcessingError.create("Pending ops during summarization", "submitSummary", undefined, { pendingMessages: this.pendingMessagesCount });
|
|
2004
|
+
logger.sendErrorEvent({
|
|
2005
|
+
eventName: "SkipFailingIncorrectSummary",
|
|
2006
|
+
referenceSequenceNumber,
|
|
2007
|
+
minimumSequenceNumber,
|
|
2008
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
2009
|
+
}, error);
|
|
2010
|
+
}
|
|
2011
|
+
else {
|
|
2012
|
+
// The retry delay when there are pending ops can be overridden via config so that we can adjust it
|
|
2013
|
+
// based on telemetry while we decide on a stable number.
|
|
2014
|
+
const retryDelayMs = this.mc.config.getNumber("Fluid.Summarizer.PendingOpsRetryDelayMs") ??
|
|
2015
|
+
exports.defaultPendingOpsRetryDelayMs;
|
|
2016
|
+
const error = new summary_1.RetriableSummaryError("PendingOpsWhileSummarizing", retryDelayMs / 1000, {
|
|
2017
|
+
count: this.pendingMessagesCount,
|
|
2018
|
+
beforeGenerate: beforeSummaryGeneration,
|
|
2019
|
+
});
|
|
2020
|
+
return {
|
|
2021
|
+
stage: "base",
|
|
2022
|
+
referenceSequenceNumber,
|
|
2023
|
+
minimumSequenceNumber,
|
|
2024
|
+
error,
|
|
2025
|
+
};
|
|
2026
|
+
}
|
|
2027
|
+
}
|
|
2028
|
+
get pendingMessagesCount() {
|
|
2029
|
+
return this.pendingStateManager.pendingMessagesCount + this.outbox.messageCount;
|
|
2030
|
+
}
|
|
1792
2031
|
hasPendingMessages() {
|
|
1793
|
-
return this.
|
|
2032
|
+
return this.pendingMessagesCount !== 0;
|
|
1794
2033
|
}
|
|
1795
2034
|
updateDocumentDirtyState(dirty) {
|
|
1796
2035
|
if (this.attachState !== container_definitions_1.AttachState.Attached) {
|
|
1797
|
-
(0,
|
|
2036
|
+
(0, core_utils_1.assert)(dirty, 0x3d2 /* Non-attached container is dirty */);
|
|
1798
2037
|
}
|
|
1799
2038
|
else {
|
|
1800
2039
|
// Other way is not true = see this.isContainerMessageDirtyable()
|
|
1801
|
-
(0,
|
|
2040
|
+
(0, core_utils_1.assert)(!dirty || this.hasPendingMessages(), 0x3d3 /* if doc is dirty, there has to be pending ops */);
|
|
1802
2041
|
}
|
|
1803
2042
|
if (this.dirtyContainer === dirty) {
|
|
1804
2043
|
return;
|
|
@@ -1806,7 +2045,6 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1806
2045
|
this.dirtyContainer = dirty;
|
|
1807
2046
|
if (this.emitDirtyDocumentEvent) {
|
|
1808
2047
|
this.emit(dirty ? "dirty" : "saved");
|
|
1809
|
-
this.context.updateDirtyContainerState(dirty);
|
|
1810
2048
|
}
|
|
1811
2049
|
}
|
|
1812
2050
|
submitDataStoreOp(id, contents, localOpMetadata = undefined) {
|
|
@@ -1814,29 +2052,28 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1814
2052
|
address: id,
|
|
1815
2053
|
contents,
|
|
1816
2054
|
};
|
|
1817
|
-
this.submit(ContainerMessageType.FluidDataStoreOp, envelope, localOpMetadata);
|
|
2055
|
+
this.submit({ type: ContainerMessageType.FluidDataStoreOp, contents: envelope }, localOpMetadata);
|
|
1818
2056
|
}
|
|
1819
2057
|
submitDataStoreAliasOp(contents, localOpMetadata) {
|
|
1820
2058
|
const aliasMessage = contents;
|
|
1821
2059
|
if (!(0, dataStore_1.isDataStoreAliasMessage)(aliasMessage)) {
|
|
1822
|
-
throw new
|
|
2060
|
+
throw new telemetry_utils_1.UsageError("malformedDataStoreAliasMessage");
|
|
1823
2061
|
}
|
|
1824
|
-
this.submit(ContainerMessageType.Alias, contents, localOpMetadata);
|
|
2062
|
+
this.submit({ type: ContainerMessageType.Alias, contents }, localOpMetadata);
|
|
1825
2063
|
}
|
|
1826
|
-
async uploadBlob(blob) {
|
|
2064
|
+
async uploadBlob(blob, signal) {
|
|
1827
2065
|
this.verifyNotClosed();
|
|
1828
|
-
return this.blobManager.createBlob(blob);
|
|
2066
|
+
return this.blobManager.createBlob(blob, signal);
|
|
1829
2067
|
}
|
|
1830
2068
|
maybeSubmitIdAllocationOp(type) {
|
|
1831
|
-
var _a, _b;
|
|
1832
2069
|
if (type !== ContainerMessageType.IdAllocation) {
|
|
1833
2070
|
let idAllocationBatchMessage;
|
|
1834
2071
|
let idRange;
|
|
1835
2072
|
if (this.idCompressorEnabled) {
|
|
1836
|
-
(0,
|
|
2073
|
+
(0, core_utils_1.assert)(this.idCompressor !== undefined, 0x67d /* IdCompressor should be defined if enabled */);
|
|
1837
2074
|
idRange = this.idCompressor.takeNextCreationRange();
|
|
1838
2075
|
// Don't include the idRange if there weren't any Ids allocated
|
|
1839
|
-
idRange =
|
|
2076
|
+
idRange = idRange?.ids !== undefined ? idRange : undefined;
|
|
1840
2077
|
}
|
|
1841
2078
|
if (idRange !== undefined) {
|
|
1842
2079
|
const idAllocationMessage = {
|
|
@@ -1847,7 +2084,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1847
2084
|
contents: JSON.stringify(idAllocationMessage),
|
|
1848
2085
|
referenceSequenceNumber: this.deltaManager.lastSequenceNumber,
|
|
1849
2086
|
metadata: undefined,
|
|
1850
|
-
localOpMetadata:
|
|
2087
|
+
localOpMetadata: this.idCompressor?.serialize(true),
|
|
1851
2088
|
type: ContainerMessageType.IdAllocation,
|
|
1852
2089
|
};
|
|
1853
2090
|
}
|
|
@@ -1856,20 +2093,21 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1856
2093
|
}
|
|
1857
2094
|
}
|
|
1858
2095
|
}
|
|
1859
|
-
submit(
|
|
2096
|
+
submit(containerRuntimeMessage, localOpMetadata = undefined, metadata = undefined) {
|
|
1860
2097
|
this.verifyNotClosed();
|
|
1861
2098
|
this.verifyCanSubmitOps();
|
|
1862
2099
|
// There should be no ops in detached container state!
|
|
1863
|
-
(0,
|
|
1864
|
-
const serializedContent = JSON.stringify(
|
|
2100
|
+
(0, core_utils_1.assert)(this.attachState !== container_definitions_1.AttachState.Detached, 0x132 /* "sending ops in detached container" */);
|
|
2101
|
+
const serializedContent = JSON.stringify(containerRuntimeMessage);
|
|
1865
2102
|
// Note that the real (non-proxy) delta manager is used here to get the readonly info. This is because
|
|
1866
2103
|
// container runtime's ability to submit ops depend on the actual readonly state of the delta manager.
|
|
1867
2104
|
if (this.innerDeltaManager.readOnlyInfo.readonly) {
|
|
1868
|
-
this.logger.sendTelemetryEvent({
|
|
2105
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1869
2106
|
eventName: "SubmitOpInReadonly",
|
|
1870
2107
|
connected: this.connected,
|
|
1871
2108
|
});
|
|
1872
2109
|
}
|
|
2110
|
+
const type = containerRuntimeMessage.type;
|
|
1873
2111
|
const message = {
|
|
1874
2112
|
contents: serializedContent,
|
|
1875
2113
|
type,
|
|
@@ -1908,6 +2146,10 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1908
2146
|
this.disableAttachReorder !== true) {
|
|
1909
2147
|
this.outbox.submitAttach(message);
|
|
1910
2148
|
}
|
|
2149
|
+
else if (type === ContainerMessageType.BlobAttach) {
|
|
2150
|
+
// BlobAttach ops must have their metadata visible and cannot be grouped (see opGroupingManager.ts)
|
|
2151
|
+
this.outbox.submitBlobAttach(message);
|
|
2152
|
+
}
|
|
1911
2153
|
else {
|
|
1912
2154
|
this.outbox.submit(message);
|
|
1913
2155
|
}
|
|
@@ -1922,7 +2164,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1922
2164
|
this.closeFn(error);
|
|
1923
2165
|
throw error;
|
|
1924
2166
|
}
|
|
1925
|
-
if (this.isContainerMessageDirtyable(
|
|
2167
|
+
if (this.isContainerMessageDirtyable(containerRuntimeMessage)) {
|
|
1926
2168
|
this.updateDocumentDirtyState(true);
|
|
1927
2169
|
}
|
|
1928
2170
|
}
|
|
@@ -1955,19 +2197,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1955
2197
|
setTimeout(flush, 0);
|
|
1956
2198
|
break;
|
|
1957
2199
|
default:
|
|
1958
|
-
(0,
|
|
2200
|
+
(0, core_utils_1.assert)(this._orderSequentiallyCalls > 0, 0x587 /* Unreachable unless running under orderSequentially */);
|
|
1959
2201
|
break;
|
|
1960
2202
|
}
|
|
1961
2203
|
}
|
|
1962
2204
|
submitSummaryMessage(contents, referenceSequenceNumber) {
|
|
1963
2205
|
this.verifyNotClosed();
|
|
1964
|
-
(0,
|
|
2206
|
+
(0, core_utils_1.assert)(this.connected, 0x133 /* "Container disconnected when trying to submit system message" */);
|
|
1965
2207
|
// System message should not be sent in the middle of the batch.
|
|
1966
|
-
(0,
|
|
2208
|
+
(0, core_utils_1.assert)(this.outbox.isEmpty, 0x3d4 /* System op in the middle of a batch */);
|
|
1967
2209
|
// back-compat: ADO #1385: Make this call unconditional in the future
|
|
1968
|
-
return this.
|
|
1969
|
-
? this.
|
|
1970
|
-
: this.
|
|
2210
|
+
return this.submitSummaryFn !== undefined
|
|
2211
|
+
? this.submitSummaryFn(contents, referenceSequenceNumber)
|
|
2212
|
+
: this.submitFn(protocol_definitions_1.MessageType.Summarize, contents, false);
|
|
1971
2213
|
}
|
|
1972
2214
|
/**
|
|
1973
2215
|
* Throw an error if the runtime is closed. Methods that are expected to potentially
|
|
@@ -1984,7 +2226,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
1984
2226
|
if (this.opReentryCallsToReport > 0) {
|
|
1985
2227
|
this.mc.logger.sendTelemetryEvent({ eventName: "OpReentry" },
|
|
1986
2228
|
// We need to capture the call stack in order to inspect the source of this usage pattern
|
|
1987
|
-
new
|
|
2229
|
+
(0, opLifecycle_1.getLongStack)(() => new telemetry_utils_1.UsageError(errorMessage)));
|
|
1988
2230
|
this.opReentryCallsToReport--;
|
|
1989
2231
|
}
|
|
1990
2232
|
// Creating ops while processing ops can lead
|
|
@@ -2000,38 +2242,47 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2000
2242
|
// The runtime must enforce op coherence by not allowing ops to be submitted
|
|
2001
2243
|
// while ops are being processed.
|
|
2002
2244
|
if (this.enableOpReentryCheck) {
|
|
2003
|
-
throw new
|
|
2245
|
+
throw new telemetry_utils_1.UsageError(errorMessage);
|
|
2004
2246
|
}
|
|
2005
2247
|
}
|
|
2006
2248
|
}
|
|
2007
|
-
|
|
2249
|
+
reSubmitBatch(batch) {
|
|
2250
|
+
this.orderSequentially(() => {
|
|
2251
|
+
for (const message of batch) {
|
|
2252
|
+
this.reSubmit(message);
|
|
2253
|
+
}
|
|
2254
|
+
});
|
|
2255
|
+
this.flush();
|
|
2256
|
+
}
|
|
2257
|
+
reSubmit(message) {
|
|
2008
2258
|
// Need to parse from string for back-compat
|
|
2009
|
-
const
|
|
2010
|
-
this.reSubmitCore(
|
|
2259
|
+
const containerRuntimeMessage = this.parseOpContent(message.content);
|
|
2260
|
+
this.reSubmitCore(containerRuntimeMessage, message.localOpMetadata, message.opMetadata);
|
|
2011
2261
|
}
|
|
2012
2262
|
/**
|
|
2013
2263
|
* Finds the right store and asks it to resubmit the message. This typically happens when we
|
|
2014
2264
|
* reconnect and there are pending messages.
|
|
2015
|
-
* @param
|
|
2265
|
+
* @param message - The original ContainerRuntimeMessage.
|
|
2016
2266
|
* @param localOpMetadata - The local metadata associated with the original message.
|
|
2017
2267
|
*/
|
|
2018
|
-
reSubmitCore(
|
|
2019
|
-
|
|
2268
|
+
reSubmitCore(message, localOpMetadata, opMetadata) {
|
|
2269
|
+
const contents = message.contents;
|
|
2270
|
+
switch (message.type) {
|
|
2020
2271
|
case ContainerMessageType.FluidDataStoreOp:
|
|
2021
2272
|
// For Operations, call resubmitDataStoreOp which will find the right store
|
|
2022
2273
|
// and trigger resubmission on it.
|
|
2023
|
-
this.dataStores.resubmitDataStoreOp(
|
|
2274
|
+
this.dataStores.resubmitDataStoreOp(contents, localOpMetadata);
|
|
2024
2275
|
break;
|
|
2025
2276
|
case ContainerMessageType.Attach:
|
|
2026
2277
|
case ContainerMessageType.Alias:
|
|
2027
|
-
this.submit(
|
|
2278
|
+
this.submit(message, localOpMetadata);
|
|
2028
2279
|
break;
|
|
2029
2280
|
case ContainerMessageType.IdAllocation:
|
|
2030
2281
|
// Remove the stashedState from the op if it's a stashed op
|
|
2031
|
-
if (
|
|
2032
|
-
delete
|
|
2282
|
+
if (contents.stashedState !== undefined) {
|
|
2283
|
+
delete contents.stashedState;
|
|
2033
2284
|
}
|
|
2034
|
-
this.submit(
|
|
2285
|
+
this.submit(message, localOpMetadata);
|
|
2035
2286
|
break;
|
|
2036
2287
|
case ContainerMessageType.ChunkedOp:
|
|
2037
2288
|
throw new Error(`chunkedOp not expected here`);
|
|
@@ -2039,10 +2290,29 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2039
2290
|
this.blobManager.reSubmit(opMetadata);
|
|
2040
2291
|
break;
|
|
2041
2292
|
case ContainerMessageType.Rejoin:
|
|
2042
|
-
this.submit(
|
|
2293
|
+
this.submit(message);
|
|
2043
2294
|
break;
|
|
2044
|
-
default:
|
|
2045
|
-
|
|
2295
|
+
default: {
|
|
2296
|
+
// This case should be very rare - it would imply an op was stashed from a
|
|
2297
|
+
// future version of runtime code and now is being applied on an older version
|
|
2298
|
+
const compatBehavior = message.compatDetails?.behavior;
|
|
2299
|
+
if (compatBehaviorAllowsMessageType(message.type, compatBehavior)) {
|
|
2300
|
+
this.logger.sendTelemetryEvent({
|
|
2301
|
+
eventName: "resubmitUnrecognizedMessageTypeAllowed",
|
|
2302
|
+
messageDetails: { type: message.type, compatBehavior },
|
|
2303
|
+
});
|
|
2304
|
+
}
|
|
2305
|
+
else {
|
|
2306
|
+
const error = telemetry_utils_1.DataProcessingError.create("Resubmitting runtime message of unknown type", "reSubmitCore", undefined /* sequencedMessage */, {
|
|
2307
|
+
messageDetails: JSON.stringify({
|
|
2308
|
+
type: message.type,
|
|
2309
|
+
compatBehavior,
|
|
2310
|
+
}),
|
|
2311
|
+
});
|
|
2312
|
+
this.closeFn(error);
|
|
2313
|
+
throw error;
|
|
2314
|
+
}
|
|
2315
|
+
}
|
|
2046
2316
|
}
|
|
2047
2317
|
}
|
|
2048
2318
|
rollback(content, localOpMetadata) {
|
|
@@ -2055,6 +2325,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2055
2325
|
this.dataStores.rollbackDataStoreOp(contents, localOpMetadata);
|
|
2056
2326
|
break;
|
|
2057
2327
|
default:
|
|
2328
|
+
// Don't check message.compatDetails because this is for rolling back a local op so the type will be known
|
|
2058
2329
|
throw new Error(`Can't rollback ${type}`);
|
|
2059
2330
|
}
|
|
2060
2331
|
}
|
|
@@ -2072,29 +2343,22 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2072
2343
|
/** Implementation of ISummarizerInternalsProvider.refreshLatestSummaryAck */
|
|
2073
2344
|
async refreshLatestSummaryAck(options) {
|
|
2074
2345
|
const { proposalHandle, ackHandle, summaryRefSeq, summaryLogger } = options;
|
|
2346
|
+
// proposalHandle is always passed from RunningSummarizer.
|
|
2347
|
+
(0, core_utils_1.assert)(proposalHandle !== undefined, 0x766 /* proposalHandle should be available */);
|
|
2075
2348
|
const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2349
|
+
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq);
|
|
2350
|
+
/**
|
|
2351
|
+
* When refreshing a summary ack, this check indicates a new ack of a summary that is newer than the
|
|
2352
|
+
* current summary that is tracked, but this summarizer runtime did not produce/track that summary. Thus
|
|
2353
|
+
* it needs to refresh its state. Today refresh is done by fetching the latest snapshot to update the cache
|
|
2354
|
+
* and then close as the current main client is likely to be re-elected as the parent summarizer again.
|
|
2355
|
+
*/
|
|
2356
|
+
if (!result.isSummaryTracked && result.isSummaryNewer) {
|
|
2357
|
+
const fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2081
2358
|
eventName: "RefreshLatestSummaryAckFetch",
|
|
2082
2359
|
ackHandle,
|
|
2083
2360
|
targetSequenceNumber: summaryRefSeq,
|
|
2084
|
-
}, readAndParseBlob);
|
|
2085
|
-
/**
|
|
2086
|
-
* back-compat - Older loaders and drivers (pre 2.0.0-internal.1.4) don't have fetchSource as a param in the
|
|
2087
|
-
* getVersions API. So, they will not fetch the latest snapshot from network in the previous fetch call. For
|
|
2088
|
-
* these scenarios, fetch the snapshot corresponding to the ack handle to have the same behavior before the
|
|
2089
|
-
* change that started fetching latest snapshot always.
|
|
2090
|
-
*/
|
|
2091
|
-
if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
|
|
2092
|
-
fetchResult = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2093
|
-
eventName: "RefreshLatestSummaryAckFetchBackCompat",
|
|
2094
|
-
ackHandle,
|
|
2095
|
-
targetSequenceNumber: summaryRefSeq,
|
|
2096
|
-
}, readAndParseBlob, ackHandle);
|
|
2097
|
-
}
|
|
2361
|
+
}, readAndParseBlob, null);
|
|
2098
2362
|
/**
|
|
2099
2363
|
* If the fetched snapshot is older than the one for which the ack was received, close the container.
|
|
2100
2364
|
* This should never happen because an ack should be sent after the latest summary is updated in the server.
|
|
@@ -2106,25 +2370,19 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2106
2370
|
* state.
|
|
2107
2371
|
*/
|
|
2108
2372
|
if (fetchResult.latestSnapshotRefSeq < summaryRefSeq) {
|
|
2109
|
-
const error =
|
|
2373
|
+
const error = telemetry_utils_1.DataProcessingError.create("Fetched snapshot is older than the received ack", "RefreshLatestSummaryAck", undefined /* sequencedMessage */, {
|
|
2110
2374
|
ackHandle,
|
|
2111
2375
|
summaryRefSeq,
|
|
2112
2376
|
fetchedSnapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2113
2377
|
});
|
|
2114
|
-
this.
|
|
2378
|
+
this.disposeFn(error);
|
|
2115
2379
|
throw error;
|
|
2116
2380
|
}
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
return {
|
|
2121
|
-
snapshotTree: fetchResult.snapshotTree,
|
|
2122
|
-
snapshotRefSeq: fetchResult.latestSnapshotRefSeq,
|
|
2123
|
-
};
|
|
2124
|
-
};
|
|
2125
|
-
const result = await this.summarizerNode.refreshLatestSummary(proposalHandle, summaryRefSeq, fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2381
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryAckFetch");
|
|
2382
|
+
return;
|
|
2383
|
+
}
|
|
2126
2384
|
// Notify the garbage collector so it can update its latest summary state.
|
|
2127
|
-
await this.garbageCollector.refreshLatestSummary(
|
|
2385
|
+
await this.garbageCollector.refreshLatestSummary(result);
|
|
2128
2386
|
}
|
|
2129
2387
|
/**
|
|
2130
2388
|
* Fetches the latest snapshot from storage and uses it to refresh SummarizerNode's
|
|
@@ -2134,31 +2392,38 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2134
2392
|
*/
|
|
2135
2393
|
async refreshLatestSummaryAckFromServer(summaryLogger) {
|
|
2136
2394
|
const readAndParseBlob = async (id) => (0, driver_utils_1.readAndParse)(this.storage, id);
|
|
2137
|
-
const {
|
|
2395
|
+
const { versionId, latestSnapshotRefSeq } = await this.fetchSnapshotFromStorage(summaryLogger, {
|
|
2138
2396
|
eventName: "RefreshLatestSummaryFromServerFetch",
|
|
2139
|
-
}, readAndParseBlob);
|
|
2140
|
-
|
|
2141
|
-
snapshotTree,
|
|
2142
|
-
snapshotRefSeq: latestSnapshotRefSeq,
|
|
2143
|
-
};
|
|
2144
|
-
const result = await this.summarizerNode.refreshLatestSummary(undefined /* proposalHandle */, latestSnapshotRefSeq, async () => fetchLatestSnapshot, readAndParseBlob, summaryLogger);
|
|
2145
|
-
// Notify the garbage collector so it can update its latest summary state.
|
|
2146
|
-
await this.garbageCollector.refreshLatestSummary(undefined /* proposalHandle */, result, readAndParseBlob);
|
|
2397
|
+
}, readAndParseBlob, null);
|
|
2398
|
+
await this.closeStaleSummarizer("RefreshLatestSummaryFromServerFetch");
|
|
2147
2399
|
return { latestSnapshotRefSeq, latestSnapshotVersionId: versionId };
|
|
2148
2400
|
}
|
|
2149
|
-
async
|
|
2150
|
-
|
|
2401
|
+
async closeStaleSummarizer(codePath) {
|
|
2402
|
+
this.mc.logger.sendTelemetryEvent({
|
|
2403
|
+
eventName: "ClosingSummarizerOnSummaryStale",
|
|
2404
|
+
codePath,
|
|
2405
|
+
message: "Stopping fetch from storage",
|
|
2406
|
+
closeSummarizerDelayMs: this.closeSummarizerDelayMs,
|
|
2407
|
+
}, new telemetry_utils_1.GenericError("Restarting summarizer instead of refreshing"));
|
|
2408
|
+
// Delay before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2409
|
+
await (0, core_utils_1.delay)(this.closeSummarizerDelayMs);
|
|
2410
|
+
this._summarizer?.stop("latestSummaryStateStale");
|
|
2411
|
+
this.disposeFn();
|
|
2151
2412
|
}
|
|
2413
|
+
/**
|
|
2414
|
+
* Downloads snapshot from storage with the given versionId or latest if versionId is null.
|
|
2415
|
+
* By default, it also closes the container after downloading the snapshot. However, this may be
|
|
2416
|
+
* overridden via options.
|
|
2417
|
+
*/
|
|
2152
2418
|
async fetchSnapshotFromStorage(logger, event, readAndParseBlob, versionId) {
|
|
2153
|
-
|
|
2154
|
-
const snapshotResults = await telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2419
|
+
return telemetry_utils_1.PerformanceEvent.timedExecAsync(logger, event, async (perfEvent) => {
|
|
2155
2420
|
const stats = {};
|
|
2156
|
-
const trace =
|
|
2421
|
+
const trace = client_utils_1.Trace.start();
|
|
2157
2422
|
const versions = await this.storage.getVersions(versionId, 1, "refreshLatestSummaryAckFromServer", versionId === null ? driver_definitions_1.FetchSource.noCache : undefined);
|
|
2158
|
-
(0,
|
|
2423
|
+
(0, core_utils_1.assert)(!!versions && !!versions[0], 0x137 /* "Failed to get version from storage" */);
|
|
2159
2424
|
stats.getVersionDuration = trace.trace().duration;
|
|
2160
2425
|
const maybeSnapshot = await this.storage.getSnapshotTree(versions[0]);
|
|
2161
|
-
(0,
|
|
2426
|
+
(0, core_utils_1.assert)(!!maybeSnapshot, 0x138 /* "Failed to get snapshot from storage" */);
|
|
2162
2427
|
stats.getSnapshotDuration = trace.trace().duration;
|
|
2163
2428
|
const latestSnapshotRefSeq = await (0, runtime_utils_1.seqFromTree)(maybeSnapshot, readAndParseBlob);
|
|
2164
2429
|
stats.snapshotRefSeq = latestSnapshotRefSeq;
|
|
@@ -2170,33 +2435,65 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2170
2435
|
latestSnapshotRefSeq,
|
|
2171
2436
|
};
|
|
2172
2437
|
});
|
|
2173
|
-
// We choose to close the summarizer after the snapshot cache is updated to avoid
|
|
2174
|
-
// situations which the main client (which is likely to be re-elected as the leader again)
|
|
2175
|
-
// loads the summarizer from cache.
|
|
2176
|
-
if (this.summaryStateUpdateMethod === "restart") {
|
|
2177
|
-
const error = new container_utils_1.GenericError("Restarting summarizer instead of refreshing");
|
|
2178
|
-
this.mc.logger.sendTelemetryEvent(Object.assign(Object.assign({}, event), { eventName: "ClosingSummarizerOnSummaryStale", codePath: event.eventName, message: "Stopping fetch from storage", versionId: versionId != null ? versionId : undefined, closeSummarizerDelayMs: this.closeSummarizerDelayMs }), error);
|
|
2179
|
-
// Delay 10 seconds before restarting summarizer to prevent the summarizer from restarting too frequently.
|
|
2180
|
-
await (0, common_utils_1.delay)(this.closeSummarizerDelayMs);
|
|
2181
|
-
(_a = this._summarizer) === null || _a === void 0 ? void 0 : _a.stop("latestSummaryStateStale");
|
|
2182
|
-
this.closeFn();
|
|
2183
|
-
throw error;
|
|
2184
|
-
}
|
|
2185
|
-
return snapshotResults;
|
|
2186
2438
|
}
|
|
2187
2439
|
notifyAttaching() { } // do nothing (deprecated method)
|
|
2188
|
-
getPendingLocalState() {
|
|
2189
|
-
|
|
2190
|
-
|
|
2440
|
+
async getPendingLocalState(props) {
|
|
2441
|
+
return telemetry_utils_1.PerformanceEvent.timedExecAsync(this.mc.logger, {
|
|
2442
|
+
eventName: "getPendingLocalState",
|
|
2443
|
+
notifyImminentClosure: props?.notifyImminentClosure,
|
|
2444
|
+
}, async (event) => {
|
|
2445
|
+
this.verifyNotClosed();
|
|
2446
|
+
const waitBlobsToAttach = props?.notifyImminentClosure;
|
|
2447
|
+
if (this._orderSequentiallyCalls !== 0) {
|
|
2448
|
+
throw new telemetry_utils_1.UsageError("can't get state during orderSequentially");
|
|
2449
|
+
}
|
|
2450
|
+
const pendingAttachmentBlobs = await this.blobManager.getPendingBlobs(waitBlobsToAttach);
|
|
2451
|
+
const pending = this.pendingStateManager.getLocalState();
|
|
2452
|
+
if (!pendingAttachmentBlobs && !this.hasPendingMessages()) {
|
|
2453
|
+
return; // no pending state to save
|
|
2454
|
+
}
|
|
2455
|
+
// Flush pending batch.
|
|
2456
|
+
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2457
|
+
// to close current batch.
|
|
2458
|
+
this.flush();
|
|
2459
|
+
const pendingState = {
|
|
2460
|
+
pending,
|
|
2461
|
+
pendingAttachmentBlobs,
|
|
2462
|
+
};
|
|
2463
|
+
event.end({
|
|
2464
|
+
attachmentBlobsSize: Object.keys(pendingAttachmentBlobs ?? {}).length,
|
|
2465
|
+
pendingOpsSize: pending?.pendingStates.length,
|
|
2466
|
+
});
|
|
2467
|
+
return pendingState;
|
|
2468
|
+
});
|
|
2469
|
+
}
|
|
2470
|
+
summarizeOnDemand(options) {
|
|
2471
|
+
if (this.isSummarizerClient) {
|
|
2472
|
+
return this.summarizer.summarizeOnDemand(options);
|
|
2473
|
+
}
|
|
2474
|
+
else if (this.summaryManager !== undefined) {
|
|
2475
|
+
return this.summaryManager.summarizeOnDemand(options);
|
|
2476
|
+
}
|
|
2477
|
+
else {
|
|
2478
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2479
|
+
// disableSummaries is turned on. We are throwing instead of returning a failure here,
|
|
2480
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2481
|
+
throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2482
|
+
}
|
|
2483
|
+
}
|
|
2484
|
+
enqueueSummarize(options) {
|
|
2485
|
+
if (this.isSummarizerClient) {
|
|
2486
|
+
return this.summarizer.enqueueSummarize(options);
|
|
2487
|
+
}
|
|
2488
|
+
else if (this.summaryManager !== undefined) {
|
|
2489
|
+
return this.summaryManager.enqueueSummarize(options);
|
|
2490
|
+
}
|
|
2491
|
+
else {
|
|
2492
|
+
// If we're not the summarizer, and we don't have a summaryManager, we expect that
|
|
2493
|
+
// generateSummaries is turned off. We are throwing instead of returning a failure here,
|
|
2494
|
+
// because it is a misuse of the API rather than an expected failure.
|
|
2495
|
+
throw new telemetry_utils_1.UsageError(`Can't summarize, disableSummaries: ${this.summariesDisabled}`);
|
|
2191
2496
|
}
|
|
2192
|
-
// Flush pending batch.
|
|
2193
|
-
// getPendingLocalState() is only exposed through Container.closeAndGetPendingLocalState(), so it's safe
|
|
2194
|
-
// to close current batch.
|
|
2195
|
-
this.flush();
|
|
2196
|
-
return {
|
|
2197
|
-
pending: this.pendingStateManager.getLocalState(),
|
|
2198
|
-
pendingAttachmentBlobs: this.blobManager.getPendingBlobs(),
|
|
2199
|
-
};
|
|
2200
2497
|
}
|
|
2201
2498
|
/**
|
|
2202
2499
|
* * Forms a function that will request a Summarizer.
|
|
@@ -2219,7 +2516,7 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2219
2516
|
const fluidObject = await (0, runtime_utils_1.requestFluidObject)(loaderRouter, request);
|
|
2220
2517
|
const summarizer = fluidObject.ISummarizer;
|
|
2221
2518
|
if (!summarizer) {
|
|
2222
|
-
throw new
|
|
2519
|
+
throw new telemetry_utils_1.UsageError("Fluid object does not implement ISummarizer");
|
|
2223
2520
|
}
|
|
2224
2521
|
return summarizer;
|
|
2225
2522
|
};
|
|
@@ -2228,11 +2525,11 @@ class ContainerRuntime extends common_utils_1.TypedEventEmitter {
|
|
|
2228
2525
|
// eslint-disable-next-line no-restricted-syntax
|
|
2229
2526
|
for (const prop in configuration) {
|
|
2230
2527
|
if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
|
|
2231
|
-
throw new
|
|
2528
|
+
throw new telemetry_utils_1.UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
|
|
2232
2529
|
}
|
|
2233
2530
|
}
|
|
2234
2531
|
if (configuration.minIdleTime > configuration.maxIdleTime) {
|
|
2235
|
-
throw new
|
|
2532
|
+
throw new telemetry_utils_1.UsageError(`"minIdleTime" [${configuration.minIdleTime}] cannot be greater than "maxIdleTime" [${configuration.maxIdleTime}]`);
|
|
2236
2533
|
}
|
|
2237
2534
|
}
|
|
2238
2535
|
get groupedBatchingEnabled() {
|