@fluidframework/container-runtime 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +147 -0
- package/README.md +4 -3
- package/dist/batchTracker.d.ts +3 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +6 -5
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +15 -18
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +212 -171
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +33 -17
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +172 -35
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +722 -425
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +15 -7
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +87 -90
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +10 -10
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.js +2 -2
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +23 -7
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +125 -82
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaManagerProxyBase.d.ts +35 -0
- package/dist/deltaManagerProxyBase.d.ts.map +1 -0
- package/dist/deltaManagerProxyBase.js +77 -0
- package/dist/deltaManagerProxyBase.js.map +1 -0
- package/dist/deltaManagerSummarizerProxy.d.ts +1 -1
- package/dist/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/dist/deltaManagerSummarizerProxy.js +4 -2
- package/dist/deltaManagerSummarizerProxy.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +10 -10
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/error.d.ts +14 -0
- package/dist/error.d.ts.map +1 -0
- package/dist/error.js +21 -0
- package/dist/error.js.map +1 -0
- package/dist/gc/garbageCollection.d.ts +10 -9
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +65 -56
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcConfigs.d.ts.map +1 -1
- package/dist/gc/gcConfigs.js +18 -14
- package/dist/gc/gcConfigs.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +17 -5
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js +14 -15
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +0 -8
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +11 -24
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcSummaryStateTracker.d.ts +4 -7
- package/dist/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/dist/gc/gcSummaryStateTracker.js +19 -58
- package/dist/gc/gcSummaryStateTracker.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +1 -1
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +45 -35
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/gcUnreferencedStateTracker.js +4 -4
- package/dist/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +3 -5
- package/dist/gc/index.js.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/dist/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/dist/id-compressor/appendOnlySortedMap.js +26 -68
- package/dist/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/dist/id-compressor/finalSpace.d.ts +29 -0
- package/dist/id-compressor/finalSpace.d.ts.map +1 -0
- package/dist/id-compressor/finalSpace.js +62 -0
- package/dist/id-compressor/finalSpace.js.map +1 -0
- package/dist/id-compressor/idCompressor.d.ts +25 -250
- package/dist/id-compressor/idCompressor.d.ts.map +1 -1
- package/dist/id-compressor/idCompressor.js +390 -1153
- package/dist/id-compressor/idCompressor.js.map +1 -1
- package/dist/id-compressor/identifiers.d.ts +32 -0
- package/dist/id-compressor/identifiers.d.ts.map +1 -0
- package/dist/id-compressor/identifiers.js +15 -0
- package/dist/id-compressor/identifiers.js.map +1 -0
- package/dist/id-compressor/index.d.ts +5 -6
- package/dist/id-compressor/index.d.ts.map +1 -1
- package/dist/id-compressor/index.js +20 -26
- package/dist/id-compressor/index.js.map +1 -1
- package/dist/id-compressor/persistanceUtilities.d.ts +22 -0
- package/dist/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/dist/id-compressor/persistanceUtilities.js +43 -0
- package/dist/id-compressor/persistanceUtilities.js.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/dist/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js +80 -0
- package/dist/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/dist/id-compressor/sessions.d.ts +115 -0
- package/dist/id-compressor/sessions.d.ts.map +1 -0
- package/dist/id-compressor/sessions.js +305 -0
- package/dist/id-compressor/sessions.js.map +1 -0
- package/dist/id-compressor/utilities.d.ts +49 -0
- package/dist/id-compressor/utilities.d.ts.map +1 -0
- package/dist/id-compressor/utilities.js +166 -0
- package/dist/id-compressor/utilities.js.map +1 -0
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -4
- package/dist/index.js.map +1 -1
- package/dist/metadata.d.ts +18 -0
- package/dist/metadata.d.ts.map +1 -0
- package/dist/metadata.js +7 -0
- package/dist/metadata.js.map +1 -0
- package/dist/opLifecycle/batchManager.d.ts +2 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +15 -7
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +11 -0
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +2 -2
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +12 -7
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -2
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +30 -21
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +19 -13
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +2 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +24 -19
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +39 -6
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +138 -61
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +22 -8
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.js +1 -2
- package/dist/opProperties.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +25 -10
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +101 -64
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +43 -33
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summary/index.d.ts +4 -4
- package/dist/summary/index.d.ts.map +1 -1
- package/dist/summary/index.js +3 -1
- package/dist/summary/index.js.map +1 -1
- package/dist/summary/orderedClientElection.d.ts +3 -3
- package/dist/summary/orderedClientElection.d.ts.map +1 -1
- package/dist/summary/orderedClientElection.js +26 -27
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/runWhileConnectedCoordinator.js +3 -3
- package/dist/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/summary/runningSummarizer.d.ts +31 -10
- package/dist/summary/runningSummarizer.d.ts.map +1 -1
- package/dist/summary/runningSummarizer.js +271 -139
- package/dist/summary/runningSummarizer.js.map +1 -1
- package/dist/summary/summarizer.d.ts +8 -7
- package/dist/summary/summarizer.d.ts.map +1 -1
- package/dist/summary/summarizer.js +79 -78
- package/dist/summary/summarizer.js.map +1 -1
- package/dist/summary/summarizerClientElection.d.ts +2 -2
- package/dist/summary/summarizerClientElection.d.ts.map +1 -1
- package/dist/summary/summarizerClientElection.js +7 -11
- package/dist/summary/summarizerClientElection.js.map +1 -1
- package/dist/summary/summarizerHeuristics.js +10 -14
- package/dist/summary/summarizerHeuristics.js.map +1 -1
- package/dist/summary/summarizerNode/index.d.ts +1 -1
- package/dist/summary/summarizerNode/index.d.ts.map +1 -1
- package/dist/summary/summarizerNode/index.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/dist/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNode.js +144 -149
- package/dist/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/dist/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/dist/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/dist/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js +74 -123
- package/dist/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/dist/summary/summarizerTypes.d.ts +44 -24
- package/dist/summary/summarizerTypes.d.ts.map +1 -1
- package/dist/summary/summarizerTypes.js.map +1 -1
- package/dist/summary/summaryCollection.d.ts +2 -2
- package/dist/summary/summaryCollection.d.ts.map +1 -1
- package/dist/summary/summaryCollection.js +16 -13
- package/dist/summary/summaryCollection.js.map +1 -1
- package/dist/summary/summaryFormat.d.ts +4 -0
- package/dist/summary/summaryFormat.d.ts.map +1 -1
- package/dist/summary/summaryFormat.js +8 -5
- package/dist/summary/summaryFormat.js.map +1 -1
- package/dist/summary/summaryGenerator.d.ts +21 -6
- package/dist/summary/summaryGenerator.d.ts.map +1 -1
- package/dist/summary/summaryGenerator.js +117 -54
- package/dist/summary/summaryGenerator.js.map +1 -1
- package/dist/summary/summaryManager.d.ts +8 -7
- package/dist/summary/summaryManager.d.ts.map +1 -1
- package/dist/summary/summaryManager.js +38 -28
- package/dist/summary/summaryManager.js.map +1 -1
- package/lib/batchTracker.d.ts +3 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +5 -4
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +15 -18
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +187 -146
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +23 -7
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +172 -35
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +678 -380
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +13 -5
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +49 -52
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +3 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.js +1 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +23 -7
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +107 -64
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaManagerProxyBase.d.ts +35 -0
- package/lib/deltaManagerProxyBase.d.ts.map +1 -0
- package/lib/deltaManagerProxyBase.js +73 -0
- package/lib/deltaManagerProxyBase.js.map +1 -0
- package/lib/deltaManagerSummarizerProxy.d.ts +1 -1
- package/lib/deltaManagerSummarizerProxy.d.ts.map +1 -1
- package/lib/deltaManagerSummarizerProxy.js +3 -1
- package/lib/deltaManagerSummarizerProxy.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +7 -7
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/error.d.ts +14 -0
- package/lib/error.d.ts.map +1 -0
- package/lib/error.js +17 -0
- package/lib/error.js.map +1 -0
- package/lib/gc/garbageCollection.d.ts +10 -9
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +61 -52
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcConfigs.d.ts.map +1 -1
- package/lib/gc/gcConfigs.js +16 -12
- package/lib/gc/gcConfigs.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +17 -5
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js +13 -14
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +0 -8
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +5 -17
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcSummaryStateTracker.d.ts +4 -7
- package/lib/gc/gcSummaryStateTracker.d.ts.map +1 -1
- package/lib/gc/gcSummaryStateTracker.js +20 -59
- package/lib/gc/gcSummaryStateTracker.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +1 -1
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +46 -36
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js +1 -1
- package/lib/gc/gcUnreferencedStateTracker.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +2 -2
- package/lib/gc/index.js.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.d.ts +8 -30
- package/lib/id-compressor/appendOnlySortedMap.d.ts.map +1 -1
- package/lib/id-compressor/appendOnlySortedMap.js +25 -66
- package/lib/id-compressor/appendOnlySortedMap.js.map +1 -1
- package/lib/id-compressor/finalSpace.d.ts +29 -0
- package/lib/id-compressor/finalSpace.d.ts.map +1 -0
- package/lib/id-compressor/finalSpace.js +58 -0
- package/lib/id-compressor/finalSpace.js.map +1 -0
- package/lib/id-compressor/idCompressor.d.ts +25 -250
- package/lib/id-compressor/idCompressor.d.ts.map +1 -1
- package/lib/id-compressor/idCompressor.js +385 -1142
- package/lib/id-compressor/idCompressor.js.map +1 -1
- package/lib/id-compressor/identifiers.d.ts +32 -0
- package/lib/id-compressor/identifiers.d.ts.map +1 -0
- package/lib/id-compressor/identifiers.js +11 -0
- package/lib/id-compressor/identifiers.js.map +1 -0
- package/lib/id-compressor/index.d.ts +5 -6
- package/lib/id-compressor/index.d.ts.map +1 -1
- package/lib/id-compressor/index.js +5 -6
- package/lib/id-compressor/index.js.map +1 -1
- package/lib/id-compressor/persistanceUtilities.d.ts +22 -0
- package/lib/id-compressor/persistanceUtilities.d.ts.map +1 -0
- package/lib/id-compressor/persistanceUtilities.js +34 -0
- package/lib/id-compressor/persistanceUtilities.js.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts +46 -0
- package/lib/id-compressor/sessionSpaceNormalizer.d.ts.map +1 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js +76 -0
- package/lib/id-compressor/sessionSpaceNormalizer.js.map +1 -0
- package/lib/id-compressor/sessions.d.ts +115 -0
- package/lib/id-compressor/sessions.d.ts.map +1 -0
- package/lib/id-compressor/sessions.js +290 -0
- package/lib/id-compressor/sessions.js.map +1 -0
- package/lib/id-compressor/utilities.d.ts +49 -0
- package/lib/id-compressor/utilities.d.ts.map +1 -0
- package/lib/id-compressor/utilities.js +148 -0
- package/lib/id-compressor/utilities.js.map +1 -0
- package/lib/index.d.ts +3 -3
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -2
- package/lib/index.js.map +1 -1
- package/lib/metadata.d.ts +18 -0
- package/lib/metadata.d.ts.map +1 -0
- package/lib/metadata.js +6 -0
- package/lib/metadata.js.map +1 -0
- package/lib/opLifecycle/batchManager.d.ts +2 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +15 -7
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +11 -0
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +2 -2
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +10 -5
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -2
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +22 -13
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +17 -11
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +2 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +15 -10
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +39 -6
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +132 -56
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts +6 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +23 -9
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.js +1 -2
- package/lib/opProperties.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +25 -10
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +90 -53
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +25 -15
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summary/index.d.ts +4 -4
- package/lib/summary/index.d.ts.map +1 -1
- package/lib/summary/index.js +2 -2
- package/lib/summary/index.js.map +1 -1
- package/lib/summary/orderedClientElection.d.ts +3 -3
- package/lib/summary/orderedClientElection.d.ts.map +1 -1
- package/lib/summary/orderedClientElection.js +21 -22
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js +1 -1
- package/lib/summary/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/summary/runningSummarizer.d.ts +31 -10
- package/lib/summary/runningSummarizer.d.ts.map +1 -1
- package/lib/summary/runningSummarizer.js +265 -133
- package/lib/summary/runningSummarizer.js.map +1 -1
- package/lib/summary/summarizer.d.ts +8 -7
- package/lib/summary/summarizer.d.ts.map +1 -1
- package/lib/summary/summarizer.js +75 -74
- package/lib/summary/summarizer.js.map +1 -1
- package/lib/summary/summarizerClientElection.d.ts +2 -2
- package/lib/summary/summarizerClientElection.d.ts.map +1 -1
- package/lib/summary/summarizerClientElection.js +6 -10
- package/lib/summary/summarizerClientElection.js.map +1 -1
- package/lib/summary/summarizerHeuristics.js +9 -13
- package/lib/summary/summarizerHeuristics.js.map +1 -1
- package/lib/summary/summarizerNode/index.d.ts +1 -1
- package/lib/summary/summarizerNode/index.d.ts.map +1 -1
- package/lib/summary/summarizerNode/index.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.d.ts +40 -23
- package/lib/summary/summarizerNode/summarizerNode.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNode.js +132 -137
- package/lib/summary/summarizerNode/summarizerNode.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts +25 -29
- package/lib/summary/summarizerNode/summarizerNodeUtils.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeUtils.js +2 -4
- package/lib/summary/summarizerNode/summarizerNodeUtils.js.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts +21 -16
- package/lib/summary/summarizerNode/summarizerNodeWithGc.d.ts.map +1 -1
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js +70 -119
- package/lib/summary/summarizerNode/summarizerNodeWithGc.js.map +1 -1
- package/lib/summary/summarizerTypes.d.ts +44 -24
- package/lib/summary/summarizerTypes.d.ts.map +1 -1
- package/lib/summary/summarizerTypes.js.map +1 -1
- package/lib/summary/summaryCollection.d.ts +2 -2
- package/lib/summary/summaryCollection.d.ts.map +1 -1
- package/lib/summary/summaryCollection.js +9 -6
- package/lib/summary/summaryCollection.js.map +1 -1
- package/lib/summary/summaryFormat.d.ts +4 -0
- package/lib/summary/summaryFormat.d.ts.map +1 -1
- package/lib/summary/summaryFormat.js +7 -4
- package/lib/summary/summaryFormat.js.map +1 -1
- package/lib/summary/summaryGenerator.d.ts +21 -6
- package/lib/summary/summaryGenerator.d.ts.map +1 -1
- package/lib/summary/summaryGenerator.js +109 -47
- package/lib/summary/summaryGenerator.js.map +1 -1
- package/lib/summary/summaryManager.d.ts +8 -7
- package/lib/summary/summaryManager.d.ts.map +1 -1
- package/lib/summary/summaryManager.js +35 -25
- package/lib/summary/summaryManager.js.map +1 -1
- package/package.json +30 -32
- package/src/batchTracker.ts +7 -5
- package/src/blobManager.ts +235 -172
- package/src/connectionTelemetry.ts +19 -5
- package/src/containerRuntime.ts +853 -431
- package/src/dataStore.ts +12 -4
- package/src/dataStoreContext.ts +49 -46
- package/src/dataStoreContexts.ts +4 -4
- package/src/dataStoreRegistry.ts +1 -1
- package/src/dataStores.ts +119 -80
- package/src/deltaManagerProxyBase.ts +111 -0
- package/src/deltaManagerSummarizerProxy.ts +4 -1
- package/src/deltaScheduler.ts +7 -11
- package/src/error.ts +18 -0
- package/src/gc/garbageCollection.md +53 -5
- package/src/gc/garbageCollection.ts +58 -51
- package/src/gc/gcConfigs.ts +4 -2
- package/src/gc/gcDefinitions.ts +17 -21
- package/src/gc/gcEarlyAdoption.md +145 -0
- package/src/gc/gcHelpers.ts +1 -12
- package/src/gc/gcSummaryStateTracker.ts +19 -65
- package/src/gc/gcTelemetry.ts +15 -13
- package/src/gc/gcUnreferencedStateTracker.ts +1 -1
- package/src/gc/index.ts +2 -4
- package/src/id-compressor/appendOnlySortedMap.ts +26 -87
- package/src/id-compressor/finalSpace.ts +67 -0
- package/src/id-compressor/idCompressor.ts +458 -1682
- package/src/id-compressor/identifiers.ts +42 -0
- package/src/id-compressor/index.ts +11 -20
- package/src/id-compressor/persistanceUtilities.ts +58 -0
- package/src/id-compressor/sessionSpaceNormalizer.ts +83 -0
- package/src/id-compressor/sessions.ts +405 -0
- package/src/id-compressor/utilities.ts +187 -0
- package/src/index.ts +9 -2
- package/src/metadata.ts +19 -0
- package/src/opLifecycle/README.md +20 -0
- package/src/opLifecycle/batchManager.ts +9 -1
- package/src/opLifecycle/definitions.ts +11 -0
- package/src/opLifecycle/index.ts +1 -1
- package/src/opLifecycle/opCompressor.ts +6 -5
- package/src/opLifecycle/opDecompressor.ts +47 -17
- package/src/opLifecycle/opGroupingManager.ts +18 -8
- package/src/opLifecycle/opSplitter.ts +10 -7
- package/src/opLifecycle/outbox.ts +177 -72
- package/src/opLifecycle/remoteMessageProcessor.ts +32 -9
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +123 -78
- package/src/scheduleManager.ts +22 -11
- package/src/summary/index.ts +7 -4
- package/src/summary/orderedClientElection.ts +10 -6
- package/src/summary/runWhileConnectedCoordinator.ts +1 -1
- package/src/summary/runningSummarizer.ts +291 -163
- package/src/summary/summarizer.ts +27 -16
- package/src/summary/summarizerClientElection.ts +2 -2
- package/src/summary/summarizerHeuristics.ts +1 -1
- package/src/summary/summarizerNode/index.ts +2 -2
- package/src/summary/summarizerNode/summarizerNode.ts +142 -184
- package/src/summary/summarizerNode/summarizerNodeUtils.ts +27 -35
- package/src/summary/summarizerNode/summarizerNodeWithGc.ts +72 -148
- package/src/summary/summarizerTypes.ts +49 -24
- package/src/summary/summaryCollection.ts +9 -4
- package/src/summary/summaryFormat.ts +9 -2
- package/src/summary/summaryGenerator.ts +72 -49
- package/src/summary/summaryManager.ts +44 -16
- package/dist/id-compressor/idRange.d.ts +0 -11
- package/dist/id-compressor/idRange.d.ts.map +0 -1
- package/dist/id-compressor/idRange.js +0 -29
- package/dist/id-compressor/idRange.js.map +0 -1
- package/dist/id-compressor/numericUuid.d.ts +0 -59
- package/dist/id-compressor/numericUuid.d.ts.map +0 -1
- package/dist/id-compressor/numericUuid.js +0 -325
- package/dist/id-compressor/numericUuid.js.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/dist/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/dist/id-compressor/sessionIdNormalizer.js +0 -488
- package/dist/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/dist/id-compressor/utils.d.ts +0 -57
- package/dist/id-compressor/utils.d.ts.map +0 -1
- package/dist/id-compressor/utils.js +0 -90
- package/dist/id-compressor/utils.js.map +0 -1
- package/dist/id-compressor/uuidUtilities.d.ts +0 -30
- package/dist/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/dist/id-compressor/uuidUtilities.js +0 -106
- package/dist/id-compressor/uuidUtilities.js.map +0 -1
- package/lib/id-compressor/idRange.d.ts +0 -11
- package/lib/id-compressor/idRange.d.ts.map +0 -1
- package/lib/id-compressor/idRange.js +0 -25
- package/lib/id-compressor/idRange.js.map +0 -1
- package/lib/id-compressor/numericUuid.d.ts +0 -59
- package/lib/id-compressor/numericUuid.d.ts.map +0 -1
- package/lib/id-compressor/numericUuid.js +0 -315
- package/lib/id-compressor/numericUuid.js.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.d.ts +0 -138
- package/lib/id-compressor/sessionIdNormalizer.d.ts.map +0 -1
- package/lib/id-compressor/sessionIdNormalizer.js +0 -484
- package/lib/id-compressor/sessionIdNormalizer.js.map +0 -1
- package/lib/id-compressor/utils.d.ts +0 -57
- package/lib/id-compressor/utils.d.ts.map +0 -1
- package/lib/id-compressor/utils.js +0 -79
- package/lib/id-compressor/utils.js.map +0 -1
- package/lib/id-compressor/uuidUtilities.d.ts +0 -30
- package/lib/id-compressor/uuidUtilities.d.ts.map +0 -1
- package/lib/id-compressor/uuidUtilities.js +0 -98
- package/lib/id-compressor/uuidUtilities.js.map +0 -1
- package/src/id-compressor/idRange.ts +0 -35
- package/src/id-compressor/numericUuid.ts +0 -383
- package/src/id-compressor/sessionIdNormalizer.ts +0 -609
- package/src/id-compressor/utils.ts +0 -114
- package/src/id-compressor/uuidUtilities.ts +0 -123
|
@@ -6,8 +6,8 @@
|
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.OpCompressor = void 0;
|
|
8
8
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
9
|
-
const
|
|
10
|
-
const
|
|
9
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
10
|
+
const client_utils_1 = require("@fluid-internal/client-utils");
|
|
11
11
|
const lz4js_1 = require("lz4js");
|
|
12
12
|
const containerRuntime_1 = require("../containerRuntime");
|
|
13
13
|
const batchManager_1 = require("./batchManager");
|
|
@@ -18,17 +18,22 @@ const batchManager_1 = require("./batchManager");
|
|
|
18
18
|
*/
|
|
19
19
|
class OpCompressor {
|
|
20
20
|
constructor(logger) {
|
|
21
|
-
this.logger = telemetry_utils_1.
|
|
21
|
+
this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpCompressor" });
|
|
22
22
|
}
|
|
23
23
|
compressBatch(batch) {
|
|
24
|
-
(0,
|
|
24
|
+
(0, core_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x5a4 /* Batch should not be empty */);
|
|
25
25
|
const compressionStart = Date.now();
|
|
26
26
|
const contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));
|
|
27
27
|
const compressedContents = (0, lz4js_1.compress)(contentsAsBuffer);
|
|
28
|
-
const compressedContent =
|
|
28
|
+
const compressedContent = client_utils_1.IsoBuffer.from(compressedContents).toString("base64");
|
|
29
29
|
const duration = Date.now() - compressionStart;
|
|
30
30
|
const messages = [];
|
|
31
|
-
messages.push(
|
|
31
|
+
messages.push({
|
|
32
|
+
...batch.content[0],
|
|
33
|
+
contents: JSON.stringify({ packedContents: compressedContent }),
|
|
34
|
+
metadata: batch.content[0].metadata,
|
|
35
|
+
compression: containerRuntime_1.CompressionAlgorithms.lz4,
|
|
36
|
+
});
|
|
32
37
|
// Add empty placeholder messages to reserve the sequence numbers
|
|
33
38
|
for (const message of batch.content.slice(1)) {
|
|
34
39
|
messages.push({
|
|
@@ -63,7 +68,7 @@ class OpCompressor {
|
|
|
63
68
|
if (e.message === "Invalid string length") {
|
|
64
69
|
// This is how JSON.stringify signals that
|
|
65
70
|
// the content size exceeds its capacity
|
|
66
|
-
const error = new
|
|
71
|
+
const error = new telemetry_utils_1.UsageError("Payload too large");
|
|
67
72
|
this.logger.sendErrorEvent({
|
|
68
73
|
eventName: "BatchTooLarge",
|
|
69
74
|
size: batch.contentSizeInBytes,
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,
|
|
1
|
+
{"version":3,"file":"opCompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opCompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,qEAAgF;AAChF,2DAAoD;AACpD,+DAAyD;AACzD,iCAAiC;AAEjC,0DAA4D;AAC5D,iDAAoD;AAGpD;;;;GAIG;AACH,MAAa,YAAY;IAGxB,YAAY,MAA4B;QACvC,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,cAAc,EAAE,CAAC,CAAC;IACxE,CAAC;IAEM,aAAa,CAAC,KAAa;QACjC,IAAA,mBAAM,EACL,KAAK,CAAC,kBAAkB,GAAG,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,EACxD,KAAK,CAAC,+BAA+B,CACrC,CAAC;QAEF,MAAM,gBAAgB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACpC,MAAM,gBAAgB,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QAC9E,MAAM,kBAAkB,GAAG,IAAA,gBAAQ,EAAC,gBAAgB,CAAC,CAAC;QACtD,MAAM,iBAAiB,GAAG,wBAAS,CAAC,IAAI,CAAC,kBAAkB,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,gBAAgB,CAAC;QAE/C,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,QAAQ,CAAC,IAAI,CAAC;YACb,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC;YACnB,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,iBAAiB,EAAE,CAAC;YAC/D,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,QAAQ;YACnC,WAAW,EAAE,wCAAqB,CAAC,GAAG;SACtC,CAAC,CAAC;QAEH,iEAAiE;QACjE,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACb,IAAI,EAAE,OAAO,CAAC,IAAI;gBAClB,eAAe,EAAE,OAAO,CAAC,eAAe;gBACxC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,uBAAuB,EAAE,OAAO,CAAC,uBAAuB;aACxD,CAAC,CAAC;SACH;QAED,MAAM,eAAe,GAAW;YAC/B,kBAAkB,EAAE,iBAAiB,CAAC,MAAM;YAC5C,OAAO,EAAE,QAAQ;YACjB,uBAAuB,EAAE,KAAK,CAAC,uBAAuB;SACtD,CAAC;QAEF,IAAI,KAAK,CAAC,kBAAkB,GAAG,MAAM,EAAE;YACtC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;gBAChC,SAAS,EAAE,iBAAiB;gBAC5B,QAAQ;gBACR,qBAAqB,EAAE,KAAK,CAAC,kBAAkB;gBAC/C,oBAAoB,EAAE,eAAe,CAAC,kBAAkB;gBACxD,OAAO,EAAE,eAAe,CAAC,OAAO,CAAC,MAAM;gBACvC,UAAU,EAAE,IAAA,iCAAkB,EAAC,eAAe,CAAC;aAC/C,CAAC,CAAC;SACH;QAED,OAAO,eAAe,CAAC;IACxB,CAAC;IAEO,cAAc,CAAC,KAAa;QACnC,IAAI;YACH,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;SACzE;QAAC,OAAO,CAAM,EAAE;YAChB,IAAI,CAAC,CAAC,OAAO,KAAK,uBAAuB,EAAE;gBAC1C,0CAA0C;gBAC1C,wCAAwC;gBACxC,MAAM,KAAK,GAAG,IAAI,4BAAU,CAAC,mBAAmB,CAAC,CAAC;gBAClD,IAAI,CAAC,MAAM,CAAC,cAAc,CACzB;oBACC,SAAS,EAAE,eAAe;oBAC1B,IAAI,EAAE,KAAK,CAAC,kBAAkB;oBAC9B,MAAM,EAAE,KAAK,CAAC,OAAO,CAAC,MAAM;iBAC5B,EACD,KAAK,CACL,CAAC;gBACF,MAAM,KAAK,CAAC;aACZ;YAED,MAAM,CAAC,CAAC;SACR;IACF,CAAC;CACD;AA/ED,oCA+EC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { createChildLogger, UsageError } from \"@fluidframework/telemetry-utils\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IsoBuffer } from \"@fluid-internal/client-utils\";\nimport { compress } from \"lz4js\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { estimateSocketSize } from \"./batchManager\";\nimport { IBatch, BatchMessage } from \"./definitions\";\n\n/**\n * Compresses batches of ops. It generates a single compressed op that contains\n * the contents of each op in the batch. It then submits empty ops for each original\n * op to reserve sequence numbers.\n */\nexport class OpCompressor {\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpCompressor\" });\n\t}\n\n\tpublic compressBatch(batch: IBatch): IBatch {\n\t\tassert(\n\t\t\tbatch.contentSizeInBytes > 0 && batch.content.length > 0,\n\t\t\t0x5a4 /* Batch should not be empty */,\n\t\t);\n\n\t\tconst compressionStart = Date.now();\n\t\tconst contentsAsBuffer = new TextEncoder().encode(this.serializeBatch(batch));\n\t\tconst compressedContents = compress(contentsAsBuffer);\n\t\tconst compressedContent = IsoBuffer.from(compressedContents).toString(\"base64\");\n\t\tconst duration = Date.now() - compressionStart;\n\n\t\tconst messages: BatchMessage[] = [];\n\t\tmessages.push({\n\t\t\t...batch.content[0],\n\t\t\tcontents: JSON.stringify({ packedContents: compressedContent }),\n\t\t\tmetadata: batch.content[0].metadata,\n\t\t\tcompression: CompressionAlgorithms.lz4,\n\t\t});\n\n\t\t// Add empty placeholder messages to reserve the sequence numbers\n\t\tfor (const message of batch.content.slice(1)) {\n\t\t\tmessages.push({\n\t\t\t\ttype: message.type,\n\t\t\t\tlocalOpMetadata: message.localOpMetadata,\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\treferenceSequenceNumber: message.referenceSequenceNumber,\n\t\t\t});\n\t\t}\n\n\t\tconst compressedBatch: IBatch = {\n\t\t\tcontentSizeInBytes: compressedContent.length,\n\t\t\tcontent: messages,\n\t\t\treferenceSequenceNumber: batch.referenceSequenceNumber,\n\t\t};\n\n\t\tif (batch.contentSizeInBytes > 200000) {\n\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\teventName: \"CompressedBatch\",\n\t\t\t\tduration,\n\t\t\t\tsizeBeforeCompression: batch.contentSizeInBytes,\n\t\t\t\tsizeAfterCompression: compressedBatch.contentSizeInBytes,\n\t\t\t\topCount: compressedBatch.content.length,\n\t\t\t\tsocketSize: estimateSocketSize(compressedBatch),\n\t\t\t});\n\t\t}\n\n\t\treturn compressedBatch;\n\t}\n\n\tprivate serializeBatch(batch: IBatch): string {\n\t\ttry {\n\t\t\treturn `[${batch.content.map((message) => message.contents).join(\",\")}]`;\n\t\t} catch (e: any) {\n\t\t\tif (e.message === \"Invalid string length\") {\n\t\t\t\t// This is how JSON.stringify signals that\n\t\t\t\t// the content size exceeds its capacity\n\t\t\t\tconst error = new UsageError(\"Payload too large\");\n\t\t\t\tthis.logger.sendErrorEvent(\n\t\t\t\t\t{\n\t\t\t\t\t\teventName: \"BatchTooLarge\",\n\t\t\t\t\t\tsize: batch.contentSizeInBytes,\n\t\t\t\t\t\tlength: batch.content.length,\n\t\t\t\t\t},\n\t\t\t\t\terror,\n\t\t\t\t);\n\t\t\t\tthrow error;\n\t\t\t}\n\n\t\t\tthrow e;\n\t\t}\n\t}\n}\n"]}
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
|
|
6
|
-
import {
|
|
6
|
+
import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
|
|
7
7
|
import { IMessageProcessingResult } from "./definitions";
|
|
8
8
|
/**
|
|
9
9
|
* State machine that "unrolls" contents of compressed batches of ops after decompressing them.
|
|
@@ -18,7 +18,7 @@ export declare class OpDecompressor {
|
|
|
18
18
|
private rootMessageContents;
|
|
19
19
|
private processedCount;
|
|
20
20
|
private readonly logger;
|
|
21
|
-
constructor(logger:
|
|
21
|
+
constructor(logger: ITelemetryBaseLogger);
|
|
22
22
|
processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
|
|
23
23
|
private isCompressed;
|
|
24
24
|
}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;
|
|
1
|
+
{"version":3,"file":"opDecompressor.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAIjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAGvE,OAAO,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AASzD;;;;;;;GAOG;AACH,qBAAa,cAAc;IAC1B,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,cAAc,CAAK;IAC3B,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAEZ,MAAM,EAAE,oBAAoB;IAIjC,cAAc,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAqGnF,OAAO,CAAC,YAAY;CA2CpB"}
|
|
@@ -6,7 +6,8 @@
|
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.OpDecompressor = void 0;
|
|
8
8
|
const lz4js_1 = require("lz4js");
|
|
9
|
-
const
|
|
9
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
10
|
+
const client_utils_1 = require("@fluid-internal/client-utils");
|
|
10
11
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
11
12
|
const containerRuntime_1 = require("../containerRuntime");
|
|
12
13
|
/**
|
|
@@ -21,22 +22,22 @@ class OpDecompressor {
|
|
|
21
22
|
constructor(logger) {
|
|
22
23
|
this.activeBatch = false;
|
|
23
24
|
this.processedCount = 0;
|
|
24
|
-
this.logger = telemetry_utils_1.
|
|
25
|
+
this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpDecompressor" });
|
|
25
26
|
}
|
|
26
27
|
processMessage(message) {
|
|
27
|
-
|
|
28
|
-
|
|
29
|
-
|
|
28
|
+
(0, core_utils_1.assert)(message.compression === undefined || message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x511 /* Only lz4 compression is supported */);
|
|
29
|
+
if (message.metadata?.batch === true &&
|
|
30
|
+
this.isCompressed(message)) {
|
|
30
31
|
// Beginning of a compressed batch
|
|
31
|
-
(0,
|
|
32
|
+
(0, core_utils_1.assert)(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);
|
|
32
33
|
if (message.compression) {
|
|
33
34
|
// lz4 is the only supported compression algorithm for now
|
|
34
|
-
(0,
|
|
35
|
+
(0, core_utils_1.assert)(message.compression === containerRuntime_1.CompressionAlgorithms.lz4, 0x4b9 /* lz4 is currently the only supported compression algorithm */);
|
|
35
36
|
}
|
|
36
37
|
this.activeBatch = true;
|
|
37
|
-
const contents =
|
|
38
|
+
const contents = client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
|
|
38
39
|
const decompressedMessage = (0, lz4js_1.decompress)(contents);
|
|
39
|
-
const intoString = (0,
|
|
40
|
+
const intoString = (0, client_utils_1.Uint8ArrayToString)(decompressedMessage);
|
|
40
41
|
const asObj = JSON.parse(intoString);
|
|
41
42
|
this.rootMessageContents = asObj;
|
|
42
43
|
return {
|
|
@@ -45,16 +46,17 @@ class OpDecompressor {
|
|
|
45
46
|
};
|
|
46
47
|
}
|
|
47
48
|
if (this.rootMessageContents !== undefined &&
|
|
48
|
-
|
|
49
|
+
message.metadata?.batch === undefined &&
|
|
49
50
|
this.activeBatch) {
|
|
50
|
-
(0,
|
|
51
|
+
(0, core_utils_1.assert)(message.contents === undefined, 0x512 /* Expecting empty message */);
|
|
51
52
|
// Continuation of compressed batch
|
|
52
53
|
return {
|
|
53
54
|
message: newMessage(message, this.rootMessageContents[this.processedCount++]),
|
|
54
55
|
state: "Accepted",
|
|
55
56
|
};
|
|
56
57
|
}
|
|
57
|
-
if (this.rootMessageContents !== undefined &&
|
|
58
|
+
if (this.rootMessageContents !== undefined &&
|
|
59
|
+
message.metadata?.batch === false) {
|
|
58
60
|
// End of compressed batch
|
|
59
61
|
const returnMessage = newMessage(message, this.rootMessageContents[this.processedCount++]);
|
|
60
62
|
this.activeBatch = false;
|
|
@@ -65,10 +67,11 @@ class OpDecompressor {
|
|
|
65
67
|
state: "Processed",
|
|
66
68
|
};
|
|
67
69
|
}
|
|
68
|
-
if (
|
|
70
|
+
if (message.metadata?.batch === undefined &&
|
|
71
|
+
this.isCompressed(message)) {
|
|
69
72
|
// Single compressed message
|
|
70
|
-
(0,
|
|
71
|
-
const contents =
|
|
73
|
+
(0, core_utils_1.assert)(this.activeBatch === false, 0x4ba /* shouldn't receive compressed message in middle of a batch */);
|
|
74
|
+
const contents = client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64");
|
|
72
75
|
const decompressedMessage = (0, lz4js_1.decompress)(contents);
|
|
73
76
|
const intoString = new TextDecoder().decode(decompressedMessage);
|
|
74
77
|
const asObj = JSON.parse(intoString);
|
|
@@ -83,7 +86,6 @@ class OpDecompressor {
|
|
|
83
86
|
};
|
|
84
87
|
}
|
|
85
88
|
isCompressed(message) {
|
|
86
|
-
var _a, _b, _c;
|
|
87
89
|
if (message.compression === containerRuntime_1.CompressionAlgorithms.lz4) {
|
|
88
90
|
return true;
|
|
89
91
|
}
|
|
@@ -102,15 +104,15 @@ class OpDecompressor {
|
|
|
102
104
|
if (message.contents !== null &&
|
|
103
105
|
typeof message.contents === "object" &&
|
|
104
106
|
Object.keys(message.contents).length === 1 &&
|
|
105
|
-
|
|
106
|
-
|
|
107
|
+
typeof message.contents.packedContents ===
|
|
108
|
+
"string" &&
|
|
107
109
|
message.contents.packedContents.length > 0 &&
|
|
108
|
-
|
|
110
|
+
client_utils_1.IsoBuffer.from(message.contents.packedContents, "base64").toString("base64") ===
|
|
109
111
|
message.contents.packedContents) {
|
|
110
112
|
this.logger.sendTelemetryEvent({
|
|
111
113
|
eventName: "LegacyCompression",
|
|
112
114
|
type: message.type,
|
|
113
|
-
batch:
|
|
115
|
+
batch: message.metadata?.batch,
|
|
114
116
|
});
|
|
115
117
|
return true;
|
|
116
118
|
}
|
|
@@ -123,5 +125,12 @@ class OpDecompressor {
|
|
|
123
125
|
}
|
|
124
126
|
exports.OpDecompressor = OpDecompressor;
|
|
125
127
|
// We should not be mutating the input message nor its metadata
|
|
126
|
-
const newMessage = (originalMessage, contents) => (
|
|
128
|
+
const newMessage = (originalMessage, contents) => ({
|
|
129
|
+
...originalMessage,
|
|
130
|
+
contents,
|
|
131
|
+
compression: undefined,
|
|
132
|
+
// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.
|
|
133
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
134
|
+
metadata: { ...originalMessage.metadata },
|
|
135
|
+
});
|
|
127
136
|
//# sourceMappingURL=opDecompressor.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,+DAAqF;AACrF,qEAAmF;AACnF,0DAA4D;AAG5D;;;;;;;GAOG;AACH,MAAa,cAAc;IAM1B,YAAY,MAA2B;QAL/B,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;QAI1B,IAAI,CAAC,MAAM,GAAG,6BAAW,CAAC,MAAM,CAAC,MAAM,EAAE,gBAAgB,CAAC,CAAC;IAC5D,CAAC;IAEM,cAAc,CAAC,OAAkC;;QACvD,IAAA,qBAAM,EACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QAEF,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,IAAI,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE;YACnE,kCAAkC;YAClC,IAAA,qBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACvF,IAAI,OAAO,CAAC,WAAW,EAAE;gBACxB,0DAA0D;gBAC1D,IAAA,qBAAM,EACL,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACjD,KAAK,CAAC,+DAA+D,CACrE,CAAC;aACF;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACtC,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS;YACrC,IAAI,CAAC,WAAW,EACf;YACD,IAAA,qBAAM,EAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAE5E,mCAAmC;YACnC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IAAI,IAAI,CAAC,mBAAmB,KAAK,SAAS,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,KAAK,EAAE;YAChF,0BAA0B;YAC1B,MAAM,aAAa,GAAG,UAAU,CAC/B,OAAO,EACP,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAC/C,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,IAAI,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK,MAAK,SAAS,IAAI,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE;YACxE,4BAA4B;YAC5B,IAAA,qBAAM,EACL,IAAI,CAAC,WAAW,KAAK,KAAK,EAC1B,KAAK,CAAC,+DAA+D,CACrE,CAAC;YAEF,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC;YAC3E,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtC,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,OAAO;YACN,OAAO;YACP,KAAK,EAAE,SAAS;SAChB,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,OAAkC;;QACtD,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EAAE;YACtD,OAAO,IAAI,CAAC;SACZ;QAED;;;;;;;;;;WAUG;QACH,IAAI;YACH,IACC,OAAO,CAAC,QAAQ,KAAK,IAAI;gBACzB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC1C,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,cAAc,MAAK,SAAS;gBAC9C,OAAO,CAAA,MAAA,OAAO,CAAC,QAAQ,0CAAE,cAAc,CAAA,KAAK,QAAQ;gBACpD,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBAC1C,wBAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,cAAc,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAC3E,OAAO,CAAC,QAAQ,CAAC,cAAc,EAC/B;gBACD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC9B,SAAS,EAAE,mBAAmB;oBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,KAAK,EAAE,MAAA,OAAO,CAAC,QAAQ,0CAAE,KAAK;iBAC9B,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;aACZ;SACD;QAAC,OAAO,GAAG,EAAE;YACb,OAAO,KAAK,CAAC;SACb;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD;AAxID,wCAwIC;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAa,EACe,EAAE,CAAC,iCAC5B,eAAe,KAClB,QAAQ,EACR,WAAW,EAAE,SAAS,EACtB,QAAQ,oBAAO,eAAe,CAAC,QAAQ,KACtC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert, IsoBuffer, Uint8ArrayToString } from \"@fluidframework/common-utils\";\nimport { ChildLogger, ITelemetryLoggerExt } from \"@fluidframework/telemetry-utils\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IMessageProcessingResult } from \"./definitions\";\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryLoggerExt) {\n\t\tthis.logger = ChildLogger.create(logger, \"OpDecompressor\");\n\t}\n\n\tpublic processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\n\t\tif (message.metadata?.batch === true && this.isCompressed(message)) {\n\t\t\t// Beginning of a compressed batch\n\t\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\t\tif (message.compression) {\n\t\t\t\t// lz4 is the only supported compression algorithm for now\n\t\t\t\tassert(\n\t\t\t\t\tmessage.compression === CompressionAlgorithms.lz4,\n\t\t\t\t\t0x4b9 /* lz4 is currently the only supported compression algorithm */,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tthis.activeBatch = true;\n\n\t\t\tconst contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\t\t\tthis.rootMessageContents = asObj;\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\tmessage.metadata?.batch === undefined &&\n\t\t\tthis.activeBatch\n\t\t) {\n\t\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t\t// Continuation of compressed batch\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (this.rootMessageContents !== undefined && message.metadata?.batch === false) {\n\t\t\t// End of compressed batch\n\t\t\tconst returnMessage = newMessage(\n\t\t\t\tmessage,\n\t\t\t\tthis.rootMessageContents[this.processedCount++],\n\t\t\t);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn {\n\t\t\t\tmessage: returnMessage,\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\tif (message.metadata?.batch === undefined && this.isCompressed(message)) {\n\t\t\t// Single compressed message\n\t\t\tassert(\n\t\t\t\tthis.activeBatch === false,\n\t\t\t\t0x4ba /* shouldn't receive compressed message in middle of a batch */,\n\t\t\t);\n\n\t\t\tconst contents = IsoBuffer.from(message.contents.packedContents, \"base64\");\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = new TextDecoder().decode(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, asObj[0]),\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tmessage,\n\t\t\tstate: \"Skipped\",\n\t\t};\n\t}\n\n\tprivate isCompressed(message: ISequencedDocumentMessage) {\n\t\tif (message.compression === CompressionAlgorithms.lz4) {\n\t\t\treturn true;\n\t\t}\n\n\t\t/**\n\t\t * Back-compat self healing mechanism for ADO:3538, as loaders from\n\t\t * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not\n\t\t * support adding the proper compression metadata to compressed messages submitted\n\t\t * by the runtime. Should be removed after the loader reaches sufficient saturation\n\t\t * for a version greater or equal than client_v2.0.0-internal.2.2.0.\n\t\t *\n\t\t * The condition holds true for compressed messages, regardless of metadata. We are ultimately\n\t\t * looking for a message with a single property `packedContents` inside `contents`, of type 'string'\n\t\t * with a base64 encoded value.\n\t\t */\n\t\ttry {\n\t\t\tif (\n\t\t\t\tmessage.contents !== null &&\n\t\t\t\ttypeof message.contents === \"object\" &&\n\t\t\t\tObject.keys(message.contents).length === 1 &&\n\t\t\t\tmessage.contents?.packedContents !== undefined &&\n\t\t\t\ttypeof message.contents?.packedContents === \"string\" &&\n\t\t\t\tmessage.contents.packedContents.length > 0 &&\n\t\t\t\tIsoBuffer.from(message.contents.packedContents, \"base64\").toString(\"base64\") ===\n\t\t\t\t\tmessage.contents.packedContents\n\t\t\t) {\n\t\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"LegacyCompression\",\n\t\t\t\t\ttype: message.type,\n\t\t\t\t\tbatch: message.metadata?.batch,\n\t\t\t\t});\n\t\t\t\treturn true;\n\t\t\t}\n\t\t} catch (err) {\n\t\t\treturn false;\n\t\t}\n\n\t\treturn false;\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: any,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\tmetadata: { ...originalMessage.metadata },\n});\n"]}
|
|
1
|
+
{"version":3,"file":"opDecompressor.js","sourceRoot":"","sources":["../../src/opLifecycle/opDecompressor.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,iCAAmC;AAEnC,2DAAoD;AACpD,+DAA6E;AAC7E,qEAAoE;AAEpE,0DAA4D;AAW5D;;;;;;;GAOG;AACH,MAAa,cAAc;IAM1B,YAAY,MAA4B;QALhC,gBAAW,GAAG,KAAK,CAAC;QAEpB,mBAAc,GAAG,CAAC,CAAC;QAI1B,IAAI,CAAC,MAAM,GAAG,IAAA,mCAAiB,EAAC,EAAE,MAAM,EAAE,SAAS,EAAE,gBAAgB,EAAE,CAAC,CAAC;IAC1E,CAAC;IAEM,cAAc,CAAC,OAAkC;QACvD,IAAA,mBAAM,EACL,OAAO,CAAC,WAAW,KAAK,SAAS,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACtF,KAAK,CAAC,uCAAuC,CAC7C,CAAC;QAEF,IACE,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,IAAI;YAChE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EACzB;YACD,kCAAkC;YAClC,IAAA,mBAAM,EAAC,IAAI,CAAC,WAAW,KAAK,KAAK,EAAE,KAAK,CAAC,4CAA4C,CAAC,CAAC;YACvF,IAAI,OAAO,CAAC,WAAW,EAAE;gBACxB,0DAA0D;gBAC1D,IAAA,mBAAM,EACL,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EACjD,KAAK,CAAC,+DAA+D,CACrE,CAAC;aACF;YAED,IAAI,CAAC,WAAW,GAAG,IAAI,CAAC;YAExB,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;YACF,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAA,iCAAkB,EAAC,mBAAmB,CAAC,CAAC;YAC3D,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YACrC,IAAI,CAAC,mBAAmB,GAAG,KAAK,CAAC;YAEjC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACrC,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,SAAS;YACrE,IAAI,CAAC,WAAW,EACf;YACD,IAAA,mBAAM,EAAC,OAAO,CAAC,QAAQ,KAAK,SAAS,EAAE,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAE5E,mCAAmC;YACnC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAAC;gBAC7E,KAAK,EAAE,UAAU;aACjB,CAAC;SACF;QAED,IACC,IAAI,CAAC,mBAAmB,KAAK,SAAS;YACrC,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,KAAK,EAChE;YACD,0BAA0B;YAC1B,MAAM,aAAa,GAAG,UAAU,CAC/B,OAAO,EACP,IAAI,CAAC,mBAAmB,CAAC,IAAI,CAAC,cAAc,EAAE,CAAC,CAC/C,CAAC;YAEF,IAAI,CAAC,WAAW,GAAG,KAAK,CAAC;YACzB,IAAI,CAAC,mBAAmB,GAAG,SAAS,CAAC;YACrC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC;YAExB,OAAO;gBACN,OAAO,EAAE,aAAa;gBACtB,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,IACE,OAAO,CAAC,QAAuC,EAAE,KAAK,KAAK,SAAS;YACrE,IAAI,CAAC,YAAY,CAAC,OAAO,CAAC,EACzB;YACD,4BAA4B;YAC5B,IAAA,mBAAM,EACL,IAAI,CAAC,WAAW,KAAK,KAAK,EAC1B,KAAK,CAAC,+DAA+D,CACrE,CAAC;YAEF,MAAM,QAAQ,GAAG,wBAAS,CAAC,IAAI,CAC7B,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC;YACF,MAAM,mBAAmB,GAAG,IAAA,kBAAU,EAAC,QAAQ,CAAC,CAAC;YACjD,MAAM,UAAU,GAAG,IAAI,WAAW,EAAE,CAAC,MAAM,CAAC,mBAAmB,CAAC,CAAC;YACjE,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;YAErC,OAAO;gBACN,OAAO,EAAE,UAAU,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;gBACtC,KAAK,EAAE,WAAW;aAClB,CAAC;SACF;QAED,OAAO;YACN,OAAO;YACP,KAAK,EAAE,SAAS;SAChB,CAAC;IACH,CAAC;IAEO,YAAY,CAAC,OAAkC;QACtD,IAAI,OAAO,CAAC,WAAW,KAAK,wCAAqB,CAAC,GAAG,EAAE;YACtD,OAAO,IAAI,CAAC;SACZ;QAED;;;;;;;;;;WAUG;QACH,IAAI;YACH,IACC,OAAO,CAAC,QAAQ,KAAK,IAAI;gBACzB,OAAO,OAAO,CAAC,QAAQ,KAAK,QAAQ;gBACpC,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,MAAM,KAAK,CAAC;gBAC1C,OAAQ,OAAO,CAAC,QAAyC,CAAC,cAAc;oBACvE,QAAQ;gBACR,OAAO,CAAC,QAAoC,CAAC,cAAc,CAAC,MAAM,GAAG,CAAC;gBACvE,wBAAS,CAAC,IAAI,CACZ,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D,QAAQ,CACR,CAAC,QAAQ,CAAC,QAAQ,CAAC;oBAClB,OAAO,CAAC,QAAoC,CAAC,cAAc,EAC5D;gBACD,IAAI,CAAC,MAAM,CAAC,kBAAkB,CAAC;oBAC9B,SAAS,EAAE,mBAAmB;oBAC9B,IAAI,EAAE,OAAO,CAAC,IAAI;oBAClB,KAAK,EAAG,OAAO,CAAC,QAAuC,EAAE,KAAK;iBAC9D,CAAC,CAAC;gBACH,OAAO,IAAI,CAAC;aACZ;SACD;QAAC,OAAO,GAAG,EAAE;YACb,OAAO,KAAK,CAAC;SACb;QAED,OAAO,KAAK,CAAC;IACd,CAAC;CACD;AA1JD,wCA0JC;AAED,+DAA+D;AAC/D,MAAM,UAAU,GAAG,CAClB,eAA0C,EAC1C,QAAa,EACe,EAAE,CAAC,CAAC;IAChC,GAAG,eAAe;IAClB,QAAQ;IACR,WAAW,EAAE,SAAS;IACtB,qIAAqI;IACrI,4EAA4E;IAC5E,QAAQ,EAAE,EAAE,GAAI,eAAe,CAAC,QAAgB,EAAE;CAClD,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { decompress } from \"lz4js\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { assert } from \"@fluidframework/core-utils\";\nimport { IsoBuffer, Uint8ArrayToString } from \"@fluid-internal/client-utils\";\nimport { createChildLogger } from \"@fluidframework/telemetry-utils\";\nimport { ITelemetryBaseLogger } from \"@fluidframework/core-interfaces\";\nimport { CompressionAlgorithms } from \"../containerRuntime\";\nimport { IBatchMetadata } from \"../metadata\";\nimport { IMessageProcessingResult } from \"./definitions\";\n\n/**\n * Compression makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IPackedContentsContents {\n\tpackedContents: string;\n}\n\n/**\n * State machine that \"unrolls\" contents of compressed batches of ops after decompressing them.\n * This class relies on some implicit contracts defined below:\n * 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true\n * 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set\n * 3. The final message of a batch will have batch metadata set to false\n * 4. An individually compressed op will have undefined batch metadata and compression set to true\n */\nexport class OpDecompressor {\n\tprivate activeBatch = false;\n\tprivate rootMessageContents: any | undefined;\n\tprivate processedCount = 0;\n\tprivate readonly logger;\n\n\tconstructor(logger: ITelemetryBaseLogger) {\n\t\tthis.logger = createChildLogger({ logger, namespace: \"OpDecompressor\" });\n\t}\n\n\tpublic processMessage(message: ISequencedDocumentMessage): IMessageProcessingResult {\n\t\tassert(\n\t\t\tmessage.compression === undefined || message.compression === CompressionAlgorithms.lz4,\n\t\t\t0x511 /* Only lz4 compression is supported */,\n\t\t);\n\n\t\tif (\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === true &&\n\t\t\tthis.isCompressed(message)\n\t\t) {\n\t\t\t// Beginning of a compressed batch\n\t\t\tassert(this.activeBatch === false, 0x4b8 /* shouldn't have multiple active batches */);\n\t\t\tif (message.compression) {\n\t\t\t\t// lz4 is the only supported compression algorithm for now\n\t\t\t\tassert(\n\t\t\t\t\tmessage.compression === CompressionAlgorithms.lz4,\n\t\t\t\t\t0x4b9 /* lz4 is currently the only supported compression algorithm */,\n\t\t\t\t);\n\t\t\t}\n\n\t\t\tthis.activeBatch = true;\n\n\t\t\tconst contents = IsoBuffer.from(\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\"base64\",\n\t\t\t);\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = Uint8ArrayToString(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\t\t\tthis.rootMessageContents = asObj;\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&\n\t\t\tthis.activeBatch\n\t\t) {\n\t\t\tassert(message.contents === undefined, 0x512 /* Expecting empty message */);\n\n\t\t\t// Continuation of compressed batch\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, this.rootMessageContents[this.processedCount++]),\n\t\t\t\tstate: \"Accepted\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\tthis.rootMessageContents !== undefined &&\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === false\n\t\t) {\n\t\t\t// End of compressed batch\n\t\t\tconst returnMessage = newMessage(\n\t\t\t\tmessage,\n\t\t\t\tthis.rootMessageContents[this.processedCount++],\n\t\t\t);\n\n\t\t\tthis.activeBatch = false;\n\t\t\tthis.rootMessageContents = undefined;\n\t\t\tthis.processedCount = 0;\n\n\t\t\treturn {\n\t\t\t\tmessage: returnMessage,\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\tif (\n\t\t\t(message.metadata as IBatchMetadata | undefined)?.batch === undefined &&\n\t\t\tthis.isCompressed(message)\n\t\t) {\n\t\t\t// Single compressed message\n\t\t\tassert(\n\t\t\t\tthis.activeBatch === false,\n\t\t\t\t0x4ba /* shouldn't receive compressed message in middle of a batch */,\n\t\t\t);\n\n\t\t\tconst contents = IsoBuffer.from(\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\"base64\",\n\t\t\t);\n\t\t\tconst decompressedMessage = decompress(contents);\n\t\t\tconst intoString = new TextDecoder().decode(decompressedMessage);\n\t\t\tconst asObj = JSON.parse(intoString);\n\n\t\t\treturn {\n\t\t\t\tmessage: newMessage(message, asObj[0]),\n\t\t\t\tstate: \"Processed\",\n\t\t\t};\n\t\t}\n\n\t\treturn {\n\t\t\tmessage,\n\t\t\tstate: \"Skipped\",\n\t\t};\n\t}\n\n\tprivate isCompressed(message: ISequencedDocumentMessage) {\n\t\tif (message.compression === CompressionAlgorithms.lz4) {\n\t\t\treturn true;\n\t\t}\n\n\t\t/**\n\t\t * Back-compat self healing mechanism for ADO:3538, as loaders from\n\t\t * version client_v2.0.0-internal.1.2.0 to client_v2.0.0-internal.2.2.0 do not\n\t\t * support adding the proper compression metadata to compressed messages submitted\n\t\t * by the runtime. Should be removed after the loader reaches sufficient saturation\n\t\t * for a version greater or equal than client_v2.0.0-internal.2.2.0.\n\t\t *\n\t\t * The condition holds true for compressed messages, regardless of metadata. We are ultimately\n\t\t * looking for a message with a single property `packedContents` inside `contents`, of type 'string'\n\t\t * with a base64 encoded value.\n\t\t */\n\t\ttry {\n\t\t\tif (\n\t\t\t\tmessage.contents !== null &&\n\t\t\t\ttypeof message.contents === \"object\" &&\n\t\t\t\tObject.keys(message.contents).length === 1 &&\n\t\t\t\ttypeof (message.contents as { packedContents?: unknown }).packedContents ===\n\t\t\t\t\t\"string\" &&\n\t\t\t\t(message.contents as IPackedContentsContents).packedContents.length > 0 &&\n\t\t\t\tIsoBuffer.from(\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents,\n\t\t\t\t\t\"base64\",\n\t\t\t\t).toString(\"base64\") ===\n\t\t\t\t\t(message.contents as IPackedContentsContents).packedContents\n\t\t\t) {\n\t\t\t\tthis.logger.sendTelemetryEvent({\n\t\t\t\t\teventName: \"LegacyCompression\",\n\t\t\t\t\ttype: message.type,\n\t\t\t\t\tbatch: (message.metadata as IBatchMetadata | undefined)?.batch,\n\t\t\t\t});\n\t\t\t\treturn true;\n\t\t\t}\n\t\t} catch (err) {\n\t\t\treturn false;\n\t\t}\n\n\t\treturn false;\n\t}\n}\n\n// We should not be mutating the input message nor its metadata\nconst newMessage = (\n\toriginalMessage: ISequencedDocumentMessage,\n\tcontents: any,\n): ISequencedDocumentMessage => ({\n\t...originalMessage,\n\tcontents,\n\tcompression: undefined,\n\t// TODO: It should already be the case that we're not modifying any metadata, not clear if/why this shallow clone should be required.\n\t// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion\n\tmetadata: { ...(originalMessage.metadata as any) },\n});\n"]}
|
|
@@ -6,7 +6,7 @@ import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions"
|
|
|
6
6
|
import { IBatch } from "./definitions";
|
|
7
7
|
export declare class OpGroupingManager {
|
|
8
8
|
private readonly groupedBatchingEnabled;
|
|
9
|
-
static groupedBatchOp
|
|
9
|
+
static readonly groupedBatchOp = "groupedBatch";
|
|
10
10
|
constructor(groupedBatchingEnabled: boolean);
|
|
11
11
|
groupBatch(batch: IBatch): IBatch;
|
|
12
12
|
ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[];
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opGroupingManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"opGroupingManager.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AAEjF,OAAO,EAAE,MAAM,EAAE,MAAM,eAAe,CAAC;AAsBvC,qBAAa,iBAAiB;IAGjB,OAAO,CAAC,QAAQ,CAAC,sBAAsB;IAFnD,MAAM,CAAC,QAAQ,CAAC,cAAc,kBAAkB;gBAEnB,sBAAsB,EAAE,OAAO;IAErD,UAAU,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;IAwCjC,SAAS,CAAC,EAAE,EAAE,yBAAyB,GAAG,yBAAyB,EAAE;CAe5E"}
|
|
@@ -5,8 +5,10 @@
|
|
|
5
5
|
*/
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.OpGroupingManager = void 0;
|
|
8
|
-
const
|
|
9
|
-
|
|
8
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
9
|
+
function isGroupContents(opContents) {
|
|
10
|
+
return opContents?.type === OpGroupingManager.groupedBatchOp;
|
|
11
|
+
}
|
|
10
12
|
class OpGroupingManager {
|
|
11
13
|
constructor(groupedBatchingEnabled) {
|
|
12
14
|
this.groupedBatchingEnabled = groupedBatchingEnabled;
|
|
@@ -16,14 +18,10 @@ class OpGroupingManager {
|
|
|
16
18
|
return batch;
|
|
17
19
|
}
|
|
18
20
|
for (const message of batch.content) {
|
|
19
|
-
// Blob attaches cannot be grouped (grouped batching would hide metadata)
|
|
20
|
-
if (message.type === __1.ContainerMessageType.BlobAttach) {
|
|
21
|
-
return batch;
|
|
22
|
-
}
|
|
23
21
|
if (message.metadata) {
|
|
24
22
|
const keys = Object.keys(message.metadata);
|
|
25
|
-
(0,
|
|
26
|
-
(0,
|
|
23
|
+
(0, core_utils_1.assert)(keys.length < 2, 0x5dd /* cannot group ops with metadata */);
|
|
24
|
+
(0, core_utils_1.assert)(keys.length === 0 || keys[0] === "batch", 0x5de /* unexpected op metadata */);
|
|
27
25
|
}
|
|
28
26
|
}
|
|
29
27
|
const serializedContent = JSON.stringify({
|
|
@@ -34,7 +32,9 @@ class OpGroupingManager {
|
|
|
34
32
|
compression: message.compression,
|
|
35
33
|
})),
|
|
36
34
|
});
|
|
37
|
-
const groupedBatch =
|
|
35
|
+
const groupedBatch = {
|
|
36
|
+
...batch,
|
|
37
|
+
content: [
|
|
38
38
|
{
|
|
39
39
|
localOpMetadata: undefined,
|
|
40
40
|
metadata: undefined,
|
|
@@ -42,17 +42,23 @@ class OpGroupingManager {
|
|
|
42
42
|
contents: serializedContent,
|
|
43
43
|
type: OpGroupingManager.groupedBatchOp,
|
|
44
44
|
},
|
|
45
|
-
]
|
|
45
|
+
],
|
|
46
|
+
};
|
|
46
47
|
return groupedBatch;
|
|
47
48
|
}
|
|
48
49
|
ungroupOp(op) {
|
|
49
|
-
|
|
50
|
-
if (((_a = op.contents) === null || _a === void 0 ? void 0 : _a.type) !== OpGroupingManager.groupedBatchOp) {
|
|
50
|
+
if (!isGroupContents(op.contents)) {
|
|
51
51
|
return [op];
|
|
52
52
|
}
|
|
53
53
|
const messages = op.contents.contents;
|
|
54
54
|
let fakeCsn = 1;
|
|
55
|
-
return messages.map((subMessage) => (
|
|
55
|
+
return messages.map((subMessage) => ({
|
|
56
|
+
...op,
|
|
57
|
+
clientSequenceNumber: fakeCsn++,
|
|
58
|
+
contents: subMessage.contents,
|
|
59
|
+
metadata: subMessage.metadata,
|
|
60
|
+
compression: subMessage.compression,
|
|
61
|
+
}));
|
|
56
62
|
}
|
|
57
63
|
}
|
|
58
64
|
exports.OpGroupingManager = OpGroupingManager;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opGroupingManager.js","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH
|
|
1
|
+
{"version":3,"file":"opGroupingManager.js","sourceRoot":"","sources":["../../src/opLifecycle/opGroupingManager.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AAEH,2DAAoD;AAmBpD,SAAS,eAAe,CACvB,UAAyE;IAEzE,OAAO,UAAU,EAAE,IAAI,KAAK,iBAAiB,CAAC,cAAc,CAAC;AAC9D,CAAC;AAED,MAAa,iBAAiB;IAG7B,YAA6B,sBAA+B;QAA/B,2BAAsB,GAAtB,sBAAsB,CAAS;IAAG,CAAC;IAEzD,UAAU,CAAC,KAAa;QAC9B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,sBAAsB,EAAE;YAC7D,OAAO,KAAK,CAAC;SACb;QAED,KAAK,MAAM,OAAO,IAAI,KAAK,CAAC,OAAO,EAAE;YACpC,IAAI,OAAO,CAAC,QAAQ,EAAE;gBACrB,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC;gBAC3C,IAAA,mBAAM,EAAC,IAAI,CAAC,MAAM,GAAG,CAAC,EAAE,KAAK,CAAC,oCAAoC,CAAC,CAAC;gBACpE,IAAA,mBAAM,EACL,IAAI,CAAC,MAAM,KAAK,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,KAAK,OAAO,EACxC,KAAK,CAAC,4BAA4B,CAClC,CAAC;aACF;SACD;QAED,MAAM,iBAAiB,GAAG,IAAI,CAAC,SAAS,CAAC;YACxC,IAAI,EAAE,iBAAiB,CAAC,cAAc;YACtC,QAAQ,EAAE,KAAK,CAAC,OAAO,CAAC,GAAG,CAAkB,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC;gBAC1D,QAAQ,EAAE,OAAO,CAAC,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,QAAQ,CAAC;gBACnF,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,WAAW,EAAE,OAAO,CAAC,WAAW;aAChC,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,MAAM,YAAY,GAAW;YAC5B,GAAG,KAAK;YACR,OAAO,EAAE;gBACR;oBACC,eAAe,EAAE,SAAS;oBAC1B,QAAQ,EAAE,SAAS;oBACnB,uBAAuB,EAAE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,uBAAuB;oBACjE,QAAQ,EAAE,iBAAiB;oBAC3B,IAAI,EAAE,iBAAiB,CAAC,cAAsC;iBAC9D;aACD;SACD,CAAC;QACF,OAAO,YAAY,CAAC;IACrB,CAAC;IAEM,SAAS,CAAC,EAA6B;QAC7C,IAAI,CAAC,eAAe,CAAC,EAAE,CAAC,QAAQ,CAAC,EAAE;YAClC,OAAO,CAAC,EAAE,CAAC,CAAC;SACZ;QAED,MAAM,QAAQ,GAAG,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC;QACtC,IAAI,OAAO,GAAG,CAAC,CAAC;QAChB,OAAO,QAAQ,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;YACpC,GAAG,EAAE;YACL,oBAAoB,EAAE,OAAO,EAAE;YAC/B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,QAAQ,EAAE,UAAU,CAAC,QAAQ;YAC7B,WAAW,EAAE,UAAU,CAAC,WAAW;SACnC,CAAC,CAAC,CAAC;IACL,CAAC;;AA3DF,8CA4DC;AA3DgB,gCAAc,GAAG,cAAc,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { assert } from \"@fluidframework/core-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ContainerMessageType } from \"..\";\nimport { IBatch } from \"./definitions\";\n\n/**\n * Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.\n */\ninterface IGroupedBatchMessageContents {\n\ttype: typeof OpGroupingManager.groupedBatchOp;\n\tcontents: IGroupedMessage[];\n}\n\ninterface IGroupedMessage {\n\tcontents?: unknown;\n\tmetadata?: Record<string, unknown>;\n\tcompression?: string;\n}\n\nfunction isGroupContents(\n\topContents: IGroupedBatchMessageContents | { type?: unknown } | undefined,\n): opContents is IGroupedBatchMessageContents {\n\treturn opContents?.type === OpGroupingManager.groupedBatchOp;\n}\n\nexport class OpGroupingManager {\n\tstatic readonly groupedBatchOp = \"groupedBatch\";\n\n\tconstructor(private readonly groupedBatchingEnabled: boolean) {}\n\n\tpublic groupBatch(batch: IBatch): IBatch {\n\t\tif (batch.content.length < 2 || !this.groupedBatchingEnabled) {\n\t\t\treturn batch;\n\t\t}\n\n\t\tfor (const message of batch.content) {\n\t\t\tif (message.metadata) {\n\t\t\t\tconst keys = Object.keys(message.metadata);\n\t\t\t\tassert(keys.length < 2, 0x5dd /* cannot group ops with metadata */);\n\t\t\t\tassert(\n\t\t\t\t\tkeys.length === 0 || keys[0] === \"batch\",\n\t\t\t\t\t0x5de /* unexpected op metadata */,\n\t\t\t\t);\n\t\t\t}\n\t\t}\n\n\t\tconst serializedContent = JSON.stringify({\n\t\t\ttype: OpGroupingManager.groupedBatchOp,\n\t\t\tcontents: batch.content.map<IGroupedMessage>((message) => ({\n\t\t\t\tcontents: message.contents === undefined ? undefined : JSON.parse(message.contents),\n\t\t\t\tmetadata: message.metadata,\n\t\t\t\tcompression: message.compression,\n\t\t\t})),\n\t\t});\n\n\t\tconst groupedBatch: IBatch = {\n\t\t\t...batch,\n\t\t\tcontent: [\n\t\t\t\t{\n\t\t\t\t\tlocalOpMetadata: undefined,\n\t\t\t\t\tmetadata: undefined,\n\t\t\t\t\treferenceSequenceNumber: batch.content[0].referenceSequenceNumber,\n\t\t\t\t\tcontents: serializedContent,\n\t\t\t\t\ttype: OpGroupingManager.groupedBatchOp as ContainerMessageType,\n\t\t\t\t},\n\t\t\t],\n\t\t};\n\t\treturn groupedBatch;\n\t}\n\n\tpublic ungroupOp(op: ISequencedDocumentMessage): ISequencedDocumentMessage[] {\n\t\tif (!isGroupContents(op.contents)) {\n\t\t\treturn [op];\n\t\t}\n\n\t\tconst messages = op.contents.contents;\n\t\tlet fakeCsn = 1;\n\t\treturn messages.map((subMessage) => ({\n\t\t\t...op,\n\t\t\tclientSequenceNumber: fakeCsn++,\n\t\t\tcontents: subMessage.contents,\n\t\t\tmetadata: subMessage.metadata,\n\t\t\tcompression: subMessage.compression,\n\t\t}));\n\t}\n}\n"]}
|
|
@@ -2,9 +2,9 @@
|
|
|
2
2
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
|
-
import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
|
|
6
5
|
import { IBatchMessage } from "@fluidframework/container-definitions";
|
|
7
6
|
import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
|
|
7
|
+
import { ITelemetryBaseLogger } from "@fluidframework/core-interfaces";
|
|
8
8
|
import { BatchMessage, IBatch, IChunkedOp, IMessageProcessingResult } from "./definitions";
|
|
9
9
|
/**
|
|
10
10
|
* Responsible for creating and reconstructing chunked messages.
|
|
@@ -15,7 +15,7 @@ export declare class OpSplitter {
|
|
|
15
15
|
private readonly maxBatchSizeInBytes;
|
|
16
16
|
private readonly chunkMap;
|
|
17
17
|
private readonly logger;
|
|
18
|
-
constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger:
|
|
18
|
+
constructor(chunks: [string, string[]][], submitBatchFn: ((batch: IBatchMessage[], referenceSequenceNumber?: number) => number) | undefined, chunkSizeInBytes: number, maxBatchSizeInBytes: number, logger: ITelemetryBaseLogger);
|
|
19
19
|
get isBatchChunkingEnabled(): boolean;
|
|
20
20
|
get chunks(): ReadonlyMap<string, string[]>;
|
|
21
21
|
processRemoteMessage(message: ISequencedDocumentMessage): IMessageProcessingResult;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;
|
|
1
|
+
{"version":3,"file":"opSplitter.d.ts","sourceRoot":"","sources":["../../src/opLifecycle/opSplitter.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAQH,OAAO,EAAE,aAAa,EAAE,MAAM,uCAAuC,CAAC;AACtE,OAAO,EAAE,yBAAyB,EAAE,MAAM,sCAAsC,CAAC;AACjF,OAAO,EAAE,oBAAoB,EAAE,MAAM,iCAAiC,CAAC;AAGvE,OAAO,EAAE,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,wBAAwB,EAAE,MAAM,eAAe,CAAC;AAE3F;;GAEG;AACH,qBAAa,UAAU;IAOrB,OAAO,CAAC,QAAQ,CAAC,aAAa;aAGd,gBAAgB,EAAE,MAAM;IACxC,OAAO,CAAC,QAAQ,CAAC,mBAAmB;IATrC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAwB;IACjD,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAGvB,MAAM,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,EAAE,EACX,aAAa,EAC3B,CAAC,CAAC,KAAK,EAAE,aAAa,EAAE,EAAE,uBAAuB,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC,GACtE,SAAS,EACI,gBAAgB,EAAE,MAAM,EACvB,mBAAmB,EAAE,MAAM,EAC5C,MAAM,EAAE,oBAAoB;IAM7B,IAAW,sBAAsB,IAAI,OAAO,CAE3C;IAED,IAAW,MAAM,IAAI,WAAW,CAAC,MAAM,EAAE,MAAM,EAAE,CAAC,CAEjD;IAEM,oBAAoB,CAAC,OAAO,EAAE,yBAAyB,GAAG,wBAAwB;IAsClF,kBAAkB,CAAC,QAAQ,EAAE,MAAM;IAM1C,OAAO,CAAC,QAAQ;IA0BhB;;;;;;;;;;;;;;;;;;;;;OAqBG;IACI,sBAAsB,CAAC,KAAK,EAAE,MAAM,GAAG,MAAM;CAkEpD;AAoBD;;;;;;;;;;;GAWG;AACH,eAAO,MAAM,OAAO,OACf,YAAY,oBACE,MAAM,YACf,OAAO,KACd,UAAU,EAqCZ,CAAC"}
|
|
@@ -6,8 +6,7 @@
|
|
|
6
6
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
7
7
|
exports.splitOp = exports.OpSplitter = void 0;
|
|
8
8
|
const telemetry_utils_1 = require("@fluidframework/telemetry-utils");
|
|
9
|
-
const
|
|
10
|
-
const container_utils_1 = require("@fluidframework/container-utils");
|
|
9
|
+
const core_utils_1 = require("@fluidframework/core-utils");
|
|
11
10
|
const containerRuntime_1 = require("../containerRuntime");
|
|
12
11
|
const batchManager_1 = require("./batchManager");
|
|
13
12
|
/**
|
|
@@ -19,7 +18,7 @@ class OpSplitter {
|
|
|
19
18
|
this.chunkSizeInBytes = chunkSizeInBytes;
|
|
20
19
|
this.maxBatchSizeInBytes = maxBatchSizeInBytes;
|
|
21
20
|
this.chunkMap = new Map(chunks);
|
|
22
|
-
this.logger = telemetry_utils_1.
|
|
21
|
+
this.logger = (0, telemetry_utils_1.createChildLogger)({ logger, namespace: "OpSplitter" });
|
|
23
22
|
}
|
|
24
23
|
get isBatchChunkingEnabled() {
|
|
25
24
|
return this.chunkSizeInBytes < Number.POSITIVE_INFINITY && this.submitBatchFn !== undefined;
|
|
@@ -34,6 +33,8 @@ class OpSplitter {
|
|
|
34
33
|
state: "Skipped",
|
|
35
34
|
};
|
|
36
35
|
}
|
|
36
|
+
// TODO: Verify whether this should be able to handle server-generated ops (with null clientId)
|
|
37
|
+
// eslint-disable-next-line @typescript-eslint/no-unnecessary-type-assertion
|
|
37
38
|
const clientId = message.clientId;
|
|
38
39
|
const chunkedContent = message.contents;
|
|
39
40
|
this.addChunk(clientId, chunkedContent, message);
|
|
@@ -48,7 +49,7 @@ class OpSplitter {
|
|
|
48
49
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
49
50
|
const serializedContent = this.chunkMap.get(clientId).join("");
|
|
50
51
|
this.clearPartialChunks(clientId);
|
|
51
|
-
const newMessage =
|
|
52
|
+
const newMessage = { ...message };
|
|
52
53
|
newMessage.contents = serializedContent === "" ? undefined : JSON.parse(serializedContent);
|
|
53
54
|
newMessage.type = chunkedContent.originalType;
|
|
54
55
|
newMessage.metadata = chunkedContent.originalMetadata;
|
|
@@ -73,7 +74,12 @@ class OpSplitter {
|
|
|
73
74
|
// We are expecting the chunks to be processed sequentially, in the same order as they are sent.
|
|
74
75
|
// Therefore, the chunkId of the incoming op needs to match the length of the array (1-based indexing)
|
|
75
76
|
// holding the existing chunks for that particular clientId.
|
|
76
|
-
throw new
|
|
77
|
+
throw new telemetry_utils_1.DataCorruptionError("Chunk Id mismatch", {
|
|
78
|
+
...(0, telemetry_utils_1.extractSafePropertiesFromMessage)(originalMessage),
|
|
79
|
+
chunkMapLength: map.length,
|
|
80
|
+
chunkId: chunkedContent.chunkId,
|
|
81
|
+
totalChunks: chunkedContent.totalChunks,
|
|
82
|
+
});
|
|
77
83
|
}
|
|
78
84
|
map.push(chunkedContent.contents);
|
|
79
85
|
}
|
|
@@ -100,14 +106,13 @@ class OpSplitter {
|
|
|
100
106
|
* @returns A new adjusted batch which can be sent over the wire
|
|
101
107
|
*/
|
|
102
108
|
splitFirstBatchMessage(batch) {
|
|
103
|
-
|
|
104
|
-
(0,
|
|
105
|
-
(0,
|
|
106
|
-
(0,
|
|
107
|
-
(0,
|
|
108
|
-
(0, common_utils_1.assert)(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
|
|
109
|
+
(0, core_utils_1.assert)(this.isBatchChunkingEnabled, 0x513 /* Chunking needs to be enabled */);
|
|
110
|
+
(0, core_utils_1.assert)(batch.contentSizeInBytes > 0 && batch.content.length > 0, 0x514 /* Batch needs to be non-empty */);
|
|
111
|
+
(0, core_utils_1.assert)(batch.referenceSequenceNumber !== undefined, 0x58a /* Batch must have a reference sequence number if non-empty */);
|
|
112
|
+
(0, core_utils_1.assert)(this.chunkSizeInBytes !== 0, 0x515 /* Chunk size needs to be non-zero */);
|
|
113
|
+
(0, core_utils_1.assert)(this.chunkSizeInBytes < this.maxBatchSizeInBytes, 0x516 /* Chunk size needs to be smaller than the max batch size */);
|
|
109
114
|
const firstMessage = batch.content[0]; // we expect this to be the large compressed op, which needs to be split
|
|
110
|
-
(0,
|
|
115
|
+
(0, core_utils_1.assert)((firstMessage.contents?.length ?? 0) >= this.chunkSizeInBytes, 0x518 /* First message in the batch needs to be chunkable */);
|
|
111
116
|
const restOfMessages = batch.content.slice(1); // we expect these to be empty ops, created to reserve sequence numbers
|
|
112
117
|
const socketSize = (0, batchManager_1.estimateSocketSize)(batch);
|
|
113
118
|
const chunks = (0, exports.splitOp)(firstMessage, this.chunkSizeInBytes,
|
|
@@ -115,14 +120,14 @@ class OpSplitter {
|
|
|
115
120
|
// we will inject an empty op to minimize the risk of the payload failing due to
|
|
116
121
|
// the overhead from the trailing empty ops in the batch.
|
|
117
122
|
socketSize >= this.maxBatchSizeInBytes);
|
|
118
|
-
(0,
|
|
123
|
+
(0, core_utils_1.assert)(this.submitBatchFn !== undefined, 0x519 /* We don't support old loaders */);
|
|
119
124
|
// Send the first N-1 chunks immediately
|
|
120
125
|
for (const chunk of chunks.slice(0, -1)) {
|
|
121
126
|
this.submitBatchFn([chunkToBatchMessage(chunk, batch.referenceSequenceNumber)], batch.referenceSequenceNumber);
|
|
122
127
|
}
|
|
123
128
|
// The last chunk will be part of the new batch and needs to
|
|
124
129
|
// preserve the batch metadata of the original batch
|
|
125
|
-
const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], batch.referenceSequenceNumber, { batch:
|
|
130
|
+
const lastChunk = chunkToBatchMessage(chunks[chunks.length - 1], batch.referenceSequenceNumber, { batch: firstMessage.metadata?.batch });
|
|
126
131
|
this.logger.sendPerformanceEvent({
|
|
127
132
|
// Used to be "Chunked compressed batch"
|
|
128
133
|
eventName: "CompressedChunkedBatch",
|
|
@@ -134,7 +139,7 @@ class OpSplitter {
|
|
|
134
139
|
});
|
|
135
140
|
return {
|
|
136
141
|
content: [lastChunk, ...restOfMessages],
|
|
137
|
-
contentSizeInBytes:
|
|
142
|
+
contentSizeInBytes: lastChunk.contents?.length ?? 0,
|
|
138
143
|
referenceSequenceNumber: batch.referenceSequenceNumber,
|
|
139
144
|
};
|
|
140
145
|
}
|
|
@@ -167,7 +172,7 @@ const chunkToBatchMessage = (chunk, referenceSequenceNumber, metadata = undefine
|
|
|
167
172
|
*/
|
|
168
173
|
const splitOp = (op, chunkSizeInBytes, extraOp = false) => {
|
|
169
174
|
const chunks = [];
|
|
170
|
-
(0,
|
|
175
|
+
(0, core_utils_1.assert)(op.contents !== undefined && op.contents !== null, 0x51a /* We should have something to chunk */);
|
|
171
176
|
const contentLength = op.contents.length;
|
|
172
177
|
const chunkCount = Math.floor((contentLength - 1) / chunkSizeInBytes) + 1 + (extraOp ? 1 : 0);
|
|
173
178
|
let offset = 0;
|
|
@@ -187,10 +192,10 @@ const splitOp = (op, chunkSizeInBytes, extraOp = false) => {
|
|
|
187
192
|
}
|
|
188
193
|
chunks.push(chunk);
|
|
189
194
|
offset += chunkSizeInBytes;
|
|
190
|
-
(0,
|
|
195
|
+
(0, core_utils_1.assert)(chunkId >= chunkCount - 1 || offset <= contentLength, 0x58b /* Content offset within bounds */);
|
|
191
196
|
}
|
|
192
|
-
(0,
|
|
193
|
-
(0,
|
|
197
|
+
(0, core_utils_1.assert)(offset >= contentLength, 0x58c /* Content offset equal or larger than content length */);
|
|
198
|
+
(0, core_utils_1.assert)(chunks.length === chunkCount, 0x5a5 /* Expected number of chunks */);
|
|
194
199
|
return chunks;
|
|
195
200
|
};
|
|
196
201
|
exports.splitOp = splitOp;
|