@fluidframework/container-runtime 2.0.0-dev.2.3.0.115467 → 2.0.0-dev.3.1.0.125672
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +21 -10
- package/.mocharc.js +2 -2
- package/api-extractor.json +2 -2
- package/dist/batchTracker.d.ts +1 -2
- package/dist/batchTracker.d.ts.map +1 -1
- package/dist/batchTracker.js +2 -1
- package/dist/batchTracker.js.map +1 -1
- package/dist/blobManager.d.ts +53 -34
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +236 -124
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.d.ts.map +1 -1
- package/dist/connectionTelemetry.js +11 -9
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerHandleContext.d.ts.map +1 -1
- package/dist/containerHandleContext.js +3 -1
- package/dist/containerHandleContext.js.map +1 -1
- package/dist/containerRuntime.d.ts +95 -46
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +288 -135
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +11 -9
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +2 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +38 -21
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +7 -3
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStoreRegistry.d.ts.map +1 -1
- package/dist/dataStoreRegistry.js +3 -1
- package/dist/dataStoreRegistry.js.map +1 -1
- package/dist/dataStores.d.ts +12 -9
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +68 -46
- package/dist/dataStores.js.map +1 -1
- package/dist/deltaScheduler.d.ts.map +1 -1
- package/dist/deltaScheduler.js +8 -3
- package/dist/deltaScheduler.js.map +1 -1
- package/dist/garbageCollection.d.ts +50 -26
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +348 -196
- package/dist/garbageCollection.js.map +1 -1
- package/dist/garbageCollectionConstants.d.ts +7 -3
- package/dist/garbageCollectionConstants.d.ts.map +1 -1
- package/dist/garbageCollectionConstants.js +10 -8
- package/dist/garbageCollectionConstants.js.map +1 -1
- package/dist/garbageCollectionHelpers.d.ts +15 -0
- package/dist/garbageCollectionHelpers.d.ts.map +1 -0
- package/dist/garbageCollectionHelpers.js +27 -0
- package/dist/garbageCollectionHelpers.js.map +1 -0
- package/dist/gcSweepReadyUsageDetection.d.ts +5 -5
- package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/dist/gcSweepReadyUsageDetection.js +14 -10
- package/dist/gcSweepReadyUsageDetection.js.map +1 -1
- package/dist/index.d.ts +3 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -5
- package/dist/index.js.map +1 -1
- package/dist/opLifecycle/batchManager.d.ts +13 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +48 -7
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/definitions.d.ts +25 -1
- package/dist/opLifecycle/definitions.d.ts.map +1 -1
- package/dist/opLifecycle/definitions.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +2 -2
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opCompressor.d.ts +1 -1
- package/dist/opLifecycle/opCompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opCompressor.js +24 -10
- package/dist/opLifecycle/opCompressor.js.map +1 -1
- package/dist/opLifecycle/opDecompressor.d.ts +2 -1
- package/dist/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/dist/opLifecycle/opDecompressor.js +33 -17
- package/dist/opLifecycle/opDecompressor.js.map +1 -1
- package/dist/opLifecycle/opSplitter.d.ts +34 -2
- package/dist/opLifecycle/opSplitter.d.ts.map +1 -1
- package/dist/opLifecycle/opSplitter.js +117 -5
- package/dist/opLifecycle/opSplitter.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +5 -0
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +38 -27
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/dist/opLifecycle/remoteMessageProcessor.js +17 -2
- package/dist/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/dist/opProperties.d.ts.map +1 -1
- package/dist/opProperties.js +1 -3
- package/dist/opProperties.js.map +1 -1
- package/dist/orderedClientElection.d.ts.map +1 -1
- package/dist/orderedClientElection.js +10 -4
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +4 -13
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +134 -161
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/dist/runWhileConnectedCoordinator.js.map +1 -1
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +34 -22
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/scheduleManager.d.ts +0 -1
- package/dist/scheduleManager.d.ts.map +1 -1
- package/dist/scheduleManager.js +11 -21
- package/dist/scheduleManager.js.map +1 -1
- package/dist/serializedSnapshotStorage.d.ts.map +1 -1
- package/dist/serializedSnapshotStorage.js +3 -1
- package/dist/serializedSnapshotStorage.js.map +1 -1
- package/dist/summarizer.d.ts +2 -3
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +39 -18
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerClientElection.d.ts +1 -2
- package/dist/summarizerClientElection.d.ts.map +1 -1
- package/dist/summarizerClientElection.js +3 -30
- package/dist/summarizerClientElection.js.map +1 -1
- package/dist/summarizerHandle.d.ts.map +1 -1
- package/dist/summarizerHandle.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +6 -9
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +22 -25
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.d.ts.map +1 -1
- package/dist/summaryCollection.js +18 -8
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js +18 -11
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +32 -14
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +21 -9
- package/dist/summaryManager.js.map +1 -1
- package/dist/throttler.d.ts +2 -2
- package/dist/throttler.d.ts.map +1 -1
- package/dist/throttler.js +4 -4
- package/dist/throttler.js.map +1 -1
- package/garbageCollection.md +15 -2
- package/lib/batchTracker.d.ts +1 -2
- package/lib/batchTracker.d.ts.map +1 -1
- package/lib/batchTracker.js +2 -1
- package/lib/batchTracker.js.map +1 -1
- package/lib/blobManager.d.ts +53 -34
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +239 -127
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.d.ts.map +1 -1
- package/lib/connectionTelemetry.js +11 -9
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerHandleContext.d.ts.map +1 -1
- package/lib/containerHandleContext.js +3 -1
- package/lib/containerHandleContext.js.map +1 -1
- package/lib/containerRuntime.d.ts +95 -46
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +291 -138
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +11 -9
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +2 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +40 -23
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +7 -3
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStoreRegistry.d.ts.map +1 -1
- package/lib/dataStoreRegistry.js +3 -1
- package/lib/dataStoreRegistry.js.map +1 -1
- package/lib/dataStores.d.ts +12 -9
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +74 -52
- package/lib/dataStores.js.map +1 -1
- package/lib/deltaScheduler.d.ts.map +1 -1
- package/lib/deltaScheduler.js +9 -4
- package/lib/deltaScheduler.js.map +1 -1
- package/lib/garbageCollection.d.ts +50 -26
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +347 -195
- package/lib/garbageCollection.js.map +1 -1
- package/lib/garbageCollectionConstants.d.ts +7 -3
- package/lib/garbageCollectionConstants.d.ts.map +1 -1
- package/lib/garbageCollectionConstants.js +9 -7
- package/lib/garbageCollectionConstants.js.map +1 -1
- package/lib/garbageCollectionHelpers.d.ts +15 -0
- package/lib/garbageCollectionHelpers.d.ts.map +1 -0
- package/lib/garbageCollectionHelpers.js +23 -0
- package/lib/garbageCollectionHelpers.js.map +1 -0
- package/lib/gcSweepReadyUsageDetection.d.ts +5 -5
- package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/lib/gcSweepReadyUsageDetection.js +14 -10
- package/lib/gcSweepReadyUsageDetection.js.map +1 -1
- package/lib/index.d.ts +3 -4
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -3
- package/lib/index.js.map +1 -1
- package/lib/opLifecycle/batchManager.d.ts +13 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +48 -7
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/definitions.d.ts +25 -1
- package/lib/opLifecycle/definitions.d.ts.map +1 -1
- package/lib/opLifecycle/definitions.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +2 -2
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opCompressor.d.ts +1 -1
- package/lib/opLifecycle/opCompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opCompressor.js +24 -10
- package/lib/opLifecycle/opCompressor.js.map +1 -1
- package/lib/opLifecycle/opDecompressor.d.ts +2 -1
- package/lib/opLifecycle/opDecompressor.d.ts.map +1 -1
- package/lib/opLifecycle/opDecompressor.js +33 -17
- package/lib/opLifecycle/opDecompressor.js.map +1 -1
- package/lib/opLifecycle/opSplitter.d.ts +34 -2
- package/lib/opLifecycle/opSplitter.d.ts.map +1 -1
- package/lib/opLifecycle/opSplitter.js +116 -5
- package/lib/opLifecycle/opSplitter.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +5 -0
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +38 -27
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.d.ts.map +1 -1
- package/lib/opLifecycle/remoteMessageProcessor.js +17 -2
- package/lib/opLifecycle/remoteMessageProcessor.js.map +1 -1
- package/lib/opProperties.d.ts.map +1 -1
- package/lib/opProperties.js +1 -3
- package/lib/opProperties.js.map +1 -1
- package/lib/orderedClientElection.d.ts.map +1 -1
- package/lib/orderedClientElection.js +10 -4
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +4 -13
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +134 -161
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runWhileConnectedCoordinator.d.ts.map +1 -1
- package/lib/runWhileConnectedCoordinator.js.map +1 -1
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +35 -23
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/scheduleManager.d.ts +0 -1
- package/lib/scheduleManager.d.ts.map +1 -1
- package/lib/scheduleManager.js +11 -21
- package/lib/scheduleManager.js.map +1 -1
- package/lib/serializedSnapshotStorage.d.ts.map +1 -1
- package/lib/serializedSnapshotStorage.js +3 -1
- package/lib/serializedSnapshotStorage.js.map +1 -1
- package/lib/summarizer.d.ts +2 -3
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +39 -18
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerClientElection.d.ts +1 -2
- package/lib/summarizerClientElection.d.ts.map +1 -1
- package/lib/summarizerClientElection.js +3 -30
- package/lib/summarizerClientElection.js.map +1 -1
- package/lib/summarizerHandle.d.ts.map +1 -1
- package/lib/summarizerHandle.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +6 -9
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +22 -25
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.d.ts.map +1 -1
- package/lib/summaryCollection.js +18 -8
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js +20 -13
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +32 -14
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +21 -9
- package/lib/summaryManager.js.map +1 -1
- package/lib/throttler.d.ts +2 -2
- package/lib/throttler.d.ts.map +1 -1
- package/lib/throttler.js +4 -4
- package/lib/throttler.js.map +1 -1
- package/package.json +27 -24
- package/prettier.config.cjs +1 -1
- package/src/batchTracker.ts +55 -50
- package/src/blobManager.ts +799 -593
- package/src/connectionTelemetry.ts +280 -249
- package/src/containerHandleContext.ts +27 -29
- package/src/containerRuntime.ts +3123 -2793
- package/src/dataStore.ts +172 -159
- package/src/dataStoreContext.ts +1048 -991
- package/src/dataStoreContexts.ts +178 -161
- package/src/dataStoreRegistry.ts +25 -20
- package/src/dataStores.ts +784 -711
- package/src/deltaScheduler.ts +158 -150
- package/src/garbageCollection.ts +1795 -1546
- package/src/garbageCollectionConstants.ts +10 -7
- package/src/garbageCollectionHelpers.ts +37 -0
- package/src/gcSweepReadyUsageDetection.ts +89 -83
- package/src/index.ts +67 -69
- package/src/opLifecycle/batchManager.ts +148 -86
- package/src/opLifecycle/definitions.ts +45 -19
- package/src/opLifecycle/index.ts +6 -5
- package/src/opLifecycle/opCompressor.ts +57 -39
- package/src/opLifecycle/opDecompressor.ts +104 -64
- package/src/opLifecycle/opSplitter.ts +226 -66
- package/src/opLifecycle/outbox.ts +206 -182
- package/src/opLifecycle/remoteMessageProcessor.ts +63 -47
- package/src/opProperties.ts +11 -9
- package/src/orderedClientElection.ts +489 -457
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +379 -381
- package/src/runWhileConnectedCoordinator.ts +78 -71
- package/src/runningSummarizer.ts +619 -582
- package/src/scheduleManager.ts +299 -280
- package/src/serializedSnapshotStorage.ts +116 -111
- package/src/summarizer.ts +417 -381
- package/src/summarizerClientElection.ts +107 -129
- package/src/summarizerHandle.ts +11 -9
- package/src/summarizerHeuristics.ts +183 -186
- package/src/summarizerTypes.ts +344 -333
- package/src/summaryCollection.ts +378 -349
- package/src/summaryFormat.ts +146 -127
- package/src/summaryGenerator.ts +464 -406
- package/src/summaryManager.ts +377 -348
- package/src/throttler.ts +131 -122
- package/tsconfig.esnext.json +6 -6
- package/tsconfig.json +9 -13
package/dist/throttler.js
CHANGED
|
@@ -55,7 +55,7 @@ class Throttler {
|
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
// Remove all attempts that have already fallen out of the window.
|
|
58
|
-
this.startTimes = this.startTimes.filter((t) =>
|
|
58
|
+
this.startTimes = this.startTimes.filter((t) => now - t < this.delayWindowMs);
|
|
59
59
|
// Compute delay, but do not exceed the specified max delay.
|
|
60
60
|
const delayMs = Math.min(this.delayFn(this.startTimes.length), this.maxDelayMs);
|
|
61
61
|
// Record this attempt start time.
|
|
@@ -84,7 +84,7 @@ exports.Throttler = Throttler;
|
|
|
84
84
|
*/
|
|
85
85
|
const formExponentialFn = ({ multiplier = 2, coefficient = 1, offset = 0, initialDelay = undefined, } = {}) => (numAttempts) => Math.max(0, numAttempts <= 0 && initialDelay !== undefined
|
|
86
86
|
? initialDelay
|
|
87
|
-
: coefficient *
|
|
87
|
+
: coefficient * Math.pow(multiplier, numAttempts) + offset);
|
|
88
88
|
exports.formExponentialFn = formExponentialFn;
|
|
89
89
|
/** f(n) = C x (B^(n+A)) + F = (C x B^A) x B^n + F */
|
|
90
90
|
const formExponentialFnWithAttemptOffset = (attemptOffset, { multiplier = 2, coefficient = 1, offset = 0, initialDelay = undefined, } = {}) => (0, exports.formExponentialFn)({
|
|
@@ -101,10 +101,10 @@ exports.formExponentialFnWithAttemptOffset = formExponentialFnWithAttemptOffset;
|
|
|
101
101
|
* If not provided, coefficient will default to 1, and offset to 0, yielding:
|
|
102
102
|
* 0 ms, 1 ms, 2 ms, 3 ms, ..., n ms delays; a linear back-off.
|
|
103
103
|
*/
|
|
104
|
-
const formLinearFn = ({ coefficient = 1, offset = 0
|
|
104
|
+
const formLinearFn = ({ coefficient = 1, offset = 0 } = {}) => (numAttempts) => Math.max(0, coefficient * numAttempts + offset);
|
|
105
105
|
exports.formLinearFn = formLinearFn;
|
|
106
106
|
/** f(n) = C x (n+A) + F = C x n + (C x A + F) */
|
|
107
|
-
const formLinearFnWithAttemptOffset = (attemptOffset, { coefficient = 1, offset = 0
|
|
107
|
+
const formLinearFnWithAttemptOffset = (attemptOffset, { coefficient = 1, offset = 0 } = {}) => (0, exports.formLinearFn)({
|
|
108
108
|
coefficient,
|
|
109
109
|
offset: coefficient * attemptOffset + offset,
|
|
110
110
|
});
|
package/dist/throttler.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"throttler.js","sourceRoot":"","sources":["../src/throttler.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AA2BH;;;GAGG;AACH,MAAa,SAAS;
|
|
1
|
+
{"version":3,"file":"throttler.js","sourceRoot":"","sources":["../src/throttler.ts"],"names":[],"mappings":";AAAA;;;GAGG;;;AA2BH;;;GAGG;AACH,MAAa,SAAS;IAuBrB;IACC,qDAAqD;IACrC,aAAqB;IACrC,6CAA6C;IAC7B,UAAkB;IAClC;;;;OAIG;IACa,OAAwC;QARxC,kBAAa,GAAb,aAAa,CAAQ;QAErB,eAAU,GAAV,UAAU,CAAQ;QAMlB,YAAO,GAAP,OAAO,CAAiC;QAhCjD,eAAU,GAAa,EAAE,CAAC;IAiC/B,CAAC;IA/BJ,IAAW,WAAW;QACrB,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC;IAC/B,CAAC;IAED;;;OAGG;IACI,WAAW;QACjB,OAAO,CAAC,GAAG,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7B,CAAC;IAED;;;OAGG;IACH,IAAW,iBAAiB;QAC3B,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC7F,CAAC;IAeM,QAAQ;QACd,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAEvB,MAAM,iBAAiB,GAAG,IAAI,CAAC,iBAAiB,CAAC;QACjD,IAAI,iBAAiB,KAAK,SAAS,EAAE;YACpC,4DAA4D;YAC5D,6DAA6D;YAC7D,MAAM,OAAO,GAAG,iBAAiB,GAAG,GAAG,CAAC;YACxC,IAAI,OAAO,GAAG,CAAC,EAAE;gBAChB,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;aAC1D;SACD;QAED,kEAAkE;QAClE,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,GAAG,CAAC,GAAG,IAAI,CAAC,aAAa,CAAC,CAAC;QAE9E,4DAA4D;QAC5D,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;QAEhF,kCAAkC;QAClC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QAE1B,gFAAgF;QAChF,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,OAAO,CAAC,CAAC;QAE1D,IAAI,OAAO,KAAK,IAAI,CAAC,UAAU,EAAE;YAChC,0DAA0D;YAC1D,iEAAiE;YACjE,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;SACxB;QAED,OAAO,OAAO,CAAC;IAChB,CAAC;CACD;AArED,8BAqEC;AAED;;;;;;;;;;GAUG;AACI,MAAM,iBAAiB,GAC7B,CAAC,EACA,UAAU,GAAG,CAAC,EACd,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,CAAC,EACV,YAAY,GAAG,SAA+B,MAC3C,EAAE,EAAyB,EAAE,CACjC,CAAC,WAAW,EAAE,EAAE,CACf,IAAI,CAAC,GAAG,CACP,CAAC,EACD,WAAW,IAAI,CAAC,IAAI,YAAY,KAAK,SAAS;IAC7C,CAAC,CAAC,YAAY;IACd,CAAC,CAAC,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,WAAW,CAAC,GAAG,MAAM,CAC3D,CAAC;AAbS,QAAA,iBAAiB,qBAa1B;AAEJ,qDAAqD;AAC9C,MAAM,kCAAkC,GAAG,CACjD,aAAqB,EACrB,EACC,UAAU,GAAG,CAAC,EACd,WAAW,GAAG,CAAC,EACf,MAAM,GAAG,CAAC,EACV,YAAY,GAAG,SAA+B,MAC3C,EAAE,EACL,EAAE,CACH,IAAA,yBAAiB,EAAC;IACjB,UAAU;IACV,WAAW,EAAE,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,EAAE,aAAa,CAAC;IAC9D,MAAM;IACN,YAAY;CACZ,CAAC,CAAC;AAdS,QAAA,kCAAkC,sCAc3C;AAEJ;;;;;;GAMG;AACI,MAAM,YAAY,GACxB,CAAC,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,EAAE,EAAyB,EAAE,CAChE,CAAC,WAAW,EAAE,EAAE,CACf,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,WAAW,GAAG,WAAW,GAAG,MAAM,CAAC,CAAC;AAHrC,QAAA,YAAY,gBAGyB;AAElD,iDAAiD;AAC1C,MAAM,6BAA6B,GAAG,CAC5C,aAAqB,EACrB,EAAE,WAAW,GAAG,CAAC,EAAE,MAAM,GAAG,CAAC,EAAE,GAAG,EAAE,EACnC,EAAE,CACH,IAAA,oBAAY,EAAC;IACZ,WAAW;IACX,MAAM,EAAE,WAAW,GAAG,aAAa,GAAG,MAAM;CAC5C,CAAC,CAAC;AAPS,QAAA,6BAA6B,iCAOtC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nexport interface IThrottler {\n\t/**\n\t * Computes what the throttle delay should be, and records an attempt\n\t * which will be used for calculating future attempt delays.\n\t */\n\tgetDelay(): number;\n\n\t/**\n\t * Number of attempts that occurred within the sliding window as of\n\t * the most recent delay computation.\n\t */\n\treadonly numAttempts: number;\n\n\t/** Width of sliding delay window in milliseconds. */\n\treadonly delayWindowMs: number;\n\t/** Maximum delay allowed in milliseconds. */\n\treadonly maxDelayMs: number;\n\t/**\n\t * Delay function used to calculate what the delay should be.\n\t * The input is the number of attempts that occurred within the sliding window.\n\t * The result is the calculated delay in milliseconds.\n\t */\n\treadonly delayFn: (numAttempts: number) => number;\n}\n\n/**\n * Used to give increasing delay times for throttling a single functionality.\n * Delay is based on previous attempts within specified time window, subtracting delay time.\n */\nexport class Throttler implements IThrottler {\n\tprivate startTimes: number[] = [];\n\n\tpublic get numAttempts() {\n\t\treturn this.startTimes.length;\n\t}\n\n\t/**\n\t * Gets all attempt start times after compensating for the delay times\n\t * by adding the delay times to the actual times.\n\t */\n\tpublic getAttempts(): readonly number[] {\n\t\treturn [...this.startTimes];\n\t}\n\n\t/**\n\t * Latest attempt time after compensating for the delay time itself\n\t * by adding the delay time to the actual time.\n\t */\n\tpublic get latestAttemptTime() {\n\t\treturn this.startTimes.length > 0 ? this.startTimes[this.startTimes.length - 1] : undefined;\n\t}\n\n\tconstructor(\n\t\t/** Width of sliding delay window in milliseconds. */\n\t\tpublic readonly delayWindowMs: number,\n\t\t/** Maximum delay allowed in milliseconds. */\n\t\tpublic readonly maxDelayMs: number,\n\t\t/**\n\t\t * Delay function used to calculate what the delay should be.\n\t\t * The input is the number of attempts that occurred within the sliding window.\n\t\t * The result is the calculated delay in milliseconds.\n\t\t */\n\t\tpublic readonly delayFn: (numAttempts: number) => number,\n\t) {}\n\n\tpublic getDelay() {\n\t\tconst now = Date.now();\n\n\t\tconst latestAttemptTime = this.latestAttemptTime;\n\t\tif (latestAttemptTime !== undefined) {\n\t\t\t// If getDelay was called sooner than the most recent delay,\n\t\t\t// subtract the remaining time, since we previously added it.\n\t\t\tconst earlyMs = latestAttemptTime - now;\n\t\t\tif (earlyMs > 0) {\n\t\t\t\tthis.startTimes = this.startTimes.map((t) => t - earlyMs);\n\t\t\t}\n\t\t}\n\n\t\t// Remove all attempts that have already fallen out of the window.\n\t\tthis.startTimes = this.startTimes.filter((t) => now - t < this.delayWindowMs);\n\n\t\t// Compute delay, but do not exceed the specified max delay.\n\t\tconst delayMs = Math.min(this.delayFn(this.startTimes.length), this.maxDelayMs);\n\n\t\t// Record this attempt start time.\n\t\tthis.startTimes.push(now);\n\n\t\t// Account for the delay time, by effectively removing it from the delay window.\n\t\tthis.startTimes = this.startTimes.map((t) => t + delayMs);\n\n\t\tif (delayMs === this.maxDelayMs) {\n\t\t\t// We hit max delay, so adding more won't affect anything.\n\t\t\t// Shift off oldest time to stop this array from growing forever.\n\t\t\tthis.startTimes.shift();\n\t\t}\n\n\t\treturn delayMs;\n\t}\n}\n\n/**\n * Helper function to generate simple exponential throttle functions.\n * f(n) = [coefficient] x ([multiplier]^n) + [flatOffset]\n * where n = number of attempts, and f(n) = delay time in milliseconds.\n * If not provided, coefficient will default to 1, multiplier to 2,\n * minimum delay to 0, and the offset to 0, yielding:\n * 0 ms, 2 ms, 4 ms, 8 ms, ..., 2^n ms\n * where M = multiplier; an exponential back-off.\n * Use initialDelay to decide what should happen when numAttempts is 0,\n * leave it undefined to not special case.\n */\nexport const formExponentialFn =\n\t({\n\t\tmultiplier = 2,\n\t\tcoefficient = 1,\n\t\toffset = 0,\n\t\tinitialDelay = undefined as number | undefined,\n\t} = {}): IThrottler[\"delayFn\"] =>\n\t(numAttempts) =>\n\t\tMath.max(\n\t\t\t0,\n\t\t\tnumAttempts <= 0 && initialDelay !== undefined\n\t\t\t\t? initialDelay\n\t\t\t\t: coefficient * Math.pow(multiplier, numAttempts) + offset,\n\t\t);\n\n/** f(n) = C x (B^(n+A)) + F = (C x B^A) x B^n + F */\nexport const formExponentialFnWithAttemptOffset = (\n\tattemptOffset: number,\n\t{\n\t\tmultiplier = 2,\n\t\tcoefficient = 1,\n\t\toffset = 0,\n\t\tinitialDelay = undefined as number | undefined,\n\t} = {},\n) =>\n\tformExponentialFn({\n\t\tmultiplier,\n\t\tcoefficient: coefficient * Math.pow(multiplier, attemptOffset),\n\t\toffset,\n\t\tinitialDelay,\n\t});\n\n/**\n * Helper function to generate simple linear throttle functions.\n * f(n) = [coefficient] x n + [flatOffset]\n * where n = number of attempts, and f(n) = delay time in milliseconds.\n * If not provided, coefficient will default to 1, and offset to 0, yielding:\n * 0 ms, 1 ms, 2 ms, 3 ms, ..., n ms delays; a linear back-off.\n */\nexport const formLinearFn =\n\t({ coefficient = 1, offset = 0 } = {}): IThrottler[\"delayFn\"] =>\n\t(numAttempts) =>\n\t\tMath.max(0, coefficient * numAttempts + offset);\n\n/** f(n) = C x (n+A) + F = C x n + (C x A + F) */\nexport const formLinearFnWithAttemptOffset = (\n\tattemptOffset: number,\n\t{ coefficient = 1, offset = 0 } = {},\n) =>\n\tformLinearFn({\n\t\tcoefficient,\n\t\toffset: coefficient * attemptOffset + offset,\n\t});\n"]}
|
package/garbageCollection.md
CHANGED
|
@@ -1,35 +1,47 @@
|
|
|
1
1
|
# Garbage Collection
|
|
2
|
+
|
|
2
3
|
Garbage collection (GC) is the process by which Fluid Framework safely delete objects that are not used. The only responsibility of the users of Fluid Framework is to add and remove references to Fluid objects correctly.
|
|
3
4
|
|
|
4
5
|
## Why have Garbage Collection?
|
|
6
|
+
|
|
5
7
|
GC reduces the size of the Fluid file at rest, the in-memory content and the summary that is uploaded to / downloaded from the server. It saves COGS on the server and it makes containers load faster as there is less data to download and process.
|
|
6
8
|
|
|
7
9
|
## What do I need to do?
|
|
10
|
+
|
|
8
11
|
All Fluid objects that are in use must be properly referenced so that they are not deleted by GC. Similarly, references to all unused Fluid objects should be removed so that they can be deleted by GC. It is the responsibility of the users of Fluid Framework to correctly add and remove references to Fluid objects.
|
|
9
12
|
|
|
10
13
|
## How do I reference / unreference Fluid objects?
|
|
14
|
+
|
|
11
15
|
Currently, the only Fluid objects that are eligible for GC are data stores and attachment blobs. The following sections describe how you can mark them as referenced or unreferenced. These sections speak of a "referenced DDS" which refers to a DDS that is created by a referenced data store.
|
|
16
|
+
|
|
12
17
|
### Data stores
|
|
18
|
+
|
|
13
19
|
There are 2 ways to reference a data store:
|
|
14
|
-
|
|
20
|
+
|
|
21
|
+
- Store the data stores's handle (see [IFluidHandle](../../../common/lib/core-interfaces/src/handles.ts)) in a referenced DDS that supports handle in its data. For example, a data store's handle can be stored in a referenced `SharedMap` DDS.
|
|
15
22
|
|
|
16
23
|
Note that storing a handle of any of a data store's DDS will also mark the data store as referenced.
|
|
17
|
-
|
|
24
|
+
|
|
25
|
+
- Alias the data store. Aliased data stores are rooted in the container, i.e., they are always referenced and cannot be unreferenced later. Aliased data stores can never be deleted so only do so if you want them to live forever.
|
|
18
26
|
|
|
19
27
|
Once there are no more referenced DDSes in the container containing a handle to a particular data store, that data store is unreferenced and is eligible for GC.
|
|
20
28
|
|
|
21
29
|
> Note: There should be at least one aliased data store with at least one DDS in a container. This is the starting point for GC to look for other referenced objects in the container.
|
|
22
30
|
|
|
23
31
|
### Attachment blobs
|
|
32
|
+
|
|
24
33
|
The only way to reference an attachment blob is to store its IFluidHandle in a referenced DDS similar to data stores.
|
|
25
34
|
|
|
26
35
|
Once there are no more referenced DDSes in the container containing a handle to a particular attachment blob, that attachment blob is unreferenced and is eligible for GC.
|
|
27
36
|
|
|
28
37
|
## GC algorithm
|
|
38
|
+
|
|
29
39
|
The GC algorithm runs in two phases:
|
|
30
40
|
|
|
31
41
|
### Mark phase
|
|
42
|
+
|
|
32
43
|
In this phase, the GC algorithm identifies all Fluid objects that are unreferenced and marks them as such:
|
|
44
|
+
|
|
33
45
|
1. It starts at the root (aliased) data stores and marks them and all their DDSes as referenced.
|
|
34
46
|
2. It recursively finds the handles stored in referenced DDSes and marks the objects corresponding to the handles as referenced until is has scanned all objects.
|
|
35
47
|
3. All the objects in the system that are not marked as referenced are marked as unreferenced. The unreferenced state of the object and timestamp of when it is unreferenced is added to the summary. The timestamp is used to determine how long the object has been unreferenced for and is used for the sweep phase.
|
|
@@ -41,6 +53,7 @@ If you wish to disable this, set the `gcAllowed` option to `false` in `IGCRuntim
|
|
|
41
53
|
See `IGCRuntimeOptions` in [containerRuntime.ts](./src/containerRuntime.ts) for more options to control GC behavior.
|
|
42
54
|
|
|
43
55
|
### Sweep phase
|
|
56
|
+
|
|
44
57
|
In this phase, the GC algorithm identifies all Fluid objects that have been unreferenced for a specific amount of time (typically 30-40 days) and deletes them. Objects are only swept once the GC system is sure that they could never be referenced again by any active clients, i.e., clients that have the object in memory and could reference it.
|
|
45
58
|
|
|
46
59
|
GC sweep phase has not been enabled yet. More details will be added here when sweep is enabled.
|
package/lib/batchTracker.d.ts
CHANGED
|
@@ -2,8 +2,7 @@
|
|
|
2
2
|
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
|
-
|
|
6
|
-
import EventEmitter from "events";
|
|
5
|
+
import { EventEmitter } from "events";
|
|
7
6
|
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
8
7
|
export declare class BatchTracker {
|
|
9
8
|
private readonly batchEventEmitter;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batchTracker.d.ts","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG
|
|
1
|
+
{"version":3,"file":"batchTracker.d.ts","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAEH,OAAO,EAAE,YAAY,EAAE,MAAM,QAAQ,CAAC;AACtC,OAAO,EAAE,gBAAgB,EAAE,MAAM,oCAAoC,CAAC;AAKtE,qBAAa,YAAY;IAOvB,OAAO,CAAC,QAAQ,CAAC,iBAAiB;IANnC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAmB;IAC1C,OAAO,CAAC,wBAAwB,CAAqB;IACrD,OAAO,CAAC,iBAAiB,CAAa;IACtC,OAAO,CAAC,6BAA6B,CAAqB;gBAGxC,iBAAiB,EAAE,YAAY,EAChD,MAAM,EAAE,gBAAgB,EACxB,oBAAoB,EAAE,MAAM,EAC5B,sBAAsB,EAAE,MAAM,EAC9B,gBAAgB,GAAE,MAAM,MAAgC;CA8CzD;AAED;;;;;;;GAOG;AACH,eAAO,MAAM,gBAAgB,sBACT,YAAY,UACvB,gBAAgB,yBACF,MAAM,2BACJ,MAAM,iBAC+D,CAAC"}
|
package/lib/batchTracker.js
CHANGED
|
@@ -15,7 +15,8 @@ export class BatchTracker {
|
|
|
15
15
|
this.trackedBatchCount++;
|
|
16
16
|
});
|
|
17
17
|
this.batchEventEmitter.on("batchEnd", (error, message) => {
|
|
18
|
-
assert(this.startBatchSequenceNumber !== undefined &&
|
|
18
|
+
assert(this.startBatchSequenceNumber !== undefined &&
|
|
19
|
+
this.batchProcessingStartTimeStamp !== undefined, 0x2ba /* "batchBegin must fire before batchEnd" */);
|
|
19
20
|
const length = message.sequenceNumber - this.startBatchSequenceNumber + 1;
|
|
20
21
|
if (length >= batchLengthThreshold) {
|
|
21
22
|
this.logger.sendPerformanceEvent({
|
package/lib/batchTracker.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"batchTracker.js","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,MAAM,OAAO,YAAY;
|
|
1
|
+
{"version":3,"file":"batchTracker.js","sourceRoot":"","sources":["../src/batchTracker.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAIH,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,MAAM,8BAA8B,CAAC;AAEnE,OAAO,EAAE,WAAW,EAAE,MAAM,iCAAiC,CAAC;AAE9D,MAAM,OAAO,YAAY;IAMxB,YACkB,iBAA+B,EAChD,MAAwB,EACxB,oBAA4B,EAC5B,sBAA8B,EAC9B,mBAAiC,GAAG,EAAE,CAAC,WAAW,CAAC,GAAG,EAAE;QAJvC,sBAAiB,GAAjB,iBAAiB,CAAc;QAJzC,sBAAiB,GAAW,CAAC,CAAC;QAUrC,IAAI,CAAC,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAErD,IAAI,CAAC,iBAAiB,CAAC,EAAE,CAAC,YAAY,EAAE,CAAC,OAAkC,EAAE,EAAE;YAC9E,IAAI,CAAC,wBAAwB,GAAG,OAAO,CAAC,cAAc,CAAC;YACvD,IAAI,CAAC,6BAA6B,GAAG,gBAAgB,EAAE,CAAC;YACxD,IAAI,CAAC,iBAAiB,EAAE,CAAC;QAC1B,CAAC,CAAC,CAAC;QAEH,IAAI,CAAC,iBAAiB,CAAC,EAAE,CACxB,UAAU,EACV,CAAC,KAAsB,EAAE,OAAkC,EAAE,EAAE;YAC9D,MAAM,CACL,IAAI,CAAC,wBAAwB,KAAK,SAAS;gBAC1C,IAAI,CAAC,6BAA6B,KAAK,SAAS,EACjD,KAAK,CAAC,4CAA4C,CAClD,CAAC;YAEF,MAAM,MAAM,GAAG,OAAO,CAAC,cAAc,GAAG,IAAI,CAAC,wBAAwB,GAAG,CAAC,CAAC;YAC1E,IAAI,MAAM,IAAI,oBAAoB,EAAE;gBACnC,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,cAAc;oBACzB,MAAM;oBACN,SAAS,EAAE,oBAAoB;oBAC/B,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;oBACjE,UAAU,EAAE,KAAK,KAAK,SAAS;iBAC/B,CAAC,CAAC;aACH;YAED,IAAI,IAAI,CAAC,iBAAiB,GAAG,sBAAsB,KAAK,CAAC,EAAE;gBAC1D,IAAI,CAAC,MAAM,CAAC,oBAAoB,CAAC;oBAChC,SAAS,EAAE,QAAQ;oBACnB,MAAM;oBACN,YAAY,EAAE,sBAAsB;oBACpC,sBAAsB,EAAE,OAAO,CAAC,cAAc;oBAC9C,QAAQ,EAAE,gBAAgB,EAAE,GAAG,IAAI,CAAC,6BAA6B;iBACjE,CAAC,CAAC;aACH;YAED,IAAI,CAAC,wBAAwB,GAAG,SAAS,CAAC;YAC1C,IAAI,CAAC,6BAA6B,GAAG,SAAS,CAAC;QAChD,CAAC,CACD,CAAC;IACH,CAAC;CACD;AAED;;;;;;;GAOG;AACH,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC/B,iBAA+B,EAC/B,MAAwB,EACxB,uBAA+B,IAAI,EACnC,yBAAiC,IAAI,EACpC,EAAE,CAAC,IAAI,YAAY,CAAC,iBAAiB,EAAE,MAAM,EAAE,oBAAoB,EAAE,sBAAsB,CAAC,CAAC","sourcesContent":["/*!\n * Copyright (c) Microsoft Corporation and contributors. All rights reserved.\n * Licensed under the MIT License.\n */\n\nimport { EventEmitter } from \"events\";\nimport { ITelemetryLogger } from \"@fluidframework/common-definitions\";\nimport { assert, performance } from \"@fluidframework/common-utils\";\nimport { ISequencedDocumentMessage } from \"@fluidframework/protocol-definitions\";\nimport { ChildLogger } from \"@fluidframework/telemetry-utils\";\n\nexport class BatchTracker {\n\tprivate readonly logger: ITelemetryLogger;\n\tprivate startBatchSequenceNumber: number | undefined;\n\tprivate trackedBatchCount: number = 0;\n\tprivate batchProcessingStartTimeStamp: number | undefined;\n\n\tconstructor(\n\t\tprivate readonly batchEventEmitter: EventEmitter,\n\t\tlogger: ITelemetryLogger,\n\t\tbatchLengthThreshold: number,\n\t\tbatchCountSamplingRate: number,\n\t\tdateTimeProvider: () => number = () => performance.now(),\n\t) {\n\t\tthis.logger = ChildLogger.create(logger, \"Batching\");\n\n\t\tthis.batchEventEmitter.on(\"batchBegin\", (message: ISequencedDocumentMessage) => {\n\t\t\tthis.startBatchSequenceNumber = message.sequenceNumber;\n\t\t\tthis.batchProcessingStartTimeStamp = dateTimeProvider();\n\t\t\tthis.trackedBatchCount++;\n\t\t});\n\n\t\tthis.batchEventEmitter.on(\n\t\t\t\"batchEnd\",\n\t\t\t(error: any | undefined, message: ISequencedDocumentMessage) => {\n\t\t\t\tassert(\n\t\t\t\t\tthis.startBatchSequenceNumber !== undefined &&\n\t\t\t\t\t\tthis.batchProcessingStartTimeStamp !== undefined,\n\t\t\t\t\t0x2ba /* \"batchBegin must fire before batchEnd\" */,\n\t\t\t\t);\n\n\t\t\t\tconst length = message.sequenceNumber - this.startBatchSequenceNumber + 1;\n\t\t\t\tif (length >= batchLengthThreshold) {\n\t\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\t\teventName: \"LengthTooBig\",\n\t\t\t\t\t\tlength,\n\t\t\t\t\t\tthreshold: batchLengthThreshold,\n\t\t\t\t\t\tbatchEndSequenceNumber: message.sequenceNumber,\n\t\t\t\t\t\tduration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n\t\t\t\t\t\tbatchError: error !== undefined,\n\t\t\t\t\t});\n\t\t\t\t}\n\n\t\t\t\tif (this.trackedBatchCount % batchCountSamplingRate === 0) {\n\t\t\t\t\tthis.logger.sendPerformanceEvent({\n\t\t\t\t\t\teventName: \"Length\",\n\t\t\t\t\t\tlength,\n\t\t\t\t\t\tsamplingRate: batchCountSamplingRate,\n\t\t\t\t\t\tbatchEndSequenceNumber: message.sequenceNumber,\n\t\t\t\t\t\tduration: dateTimeProvider() - this.batchProcessingStartTimeStamp,\n\t\t\t\t\t});\n\t\t\t\t}\n\n\t\t\t\tthis.startBatchSequenceNumber = undefined;\n\t\t\t\tthis.batchProcessingStartTimeStamp = undefined;\n\t\t\t},\n\t\t);\n\t}\n}\n\n/**\n * Track batch sizes in terms of op counts and processing times\n *\n * @param batchEventEmitter - event emitter which tracks the lifecycle of batch operations\n * @param logger - See {@link @fluidframework/common-definitions#ITelemetryLogger}\n * @param batchLengthThreshold - threshold for the length of a batch when to send an error event\n * @param batchCountSamplingRate - rate for batches for which to send an event with its characteristics\n */\nexport const BindBatchTracker = (\n\tbatchEventEmitter: EventEmitter,\n\tlogger: ITelemetryLogger,\n\tbatchLengthThreshold: number = 1000,\n\tbatchCountSamplingRate: number = 1000,\n) => new BatchTracker(batchEventEmitter, logger, batchLengthThreshold, batchCountSamplingRate);\n"]}
|
package/lib/blobManager.d.ts
CHANGED
|
@@ -40,30 +40,37 @@ export interface IPendingBlobs {
|
|
|
40
40
|
blob: string;
|
|
41
41
|
};
|
|
42
42
|
}
|
|
43
|
-
export
|
|
43
|
+
export interface IBlobManagerEvents {
|
|
44
|
+
(event: "noPendingBlobs", listener: () => void): any;
|
|
45
|
+
}
|
|
46
|
+
export declare class BlobManager extends TypedEventEmitter<IBlobManagerEvents> {
|
|
44
47
|
private readonly routeContext;
|
|
45
48
|
private readonly getStorage;
|
|
46
49
|
/**
|
|
47
|
-
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
|
|
48
|
-
*
|
|
49
|
-
*
|
|
50
|
-
*
|
|
50
|
+
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
|
|
51
|
+
* deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
|
|
52
|
+
* the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
|
|
53
|
+
* include the storage ID in the summary when it sees the op.
|
|
51
54
|
*
|
|
52
|
-
* The op
|
|
53
|
-
*
|
|
54
|
-
*
|
|
55
|
+
* The op will also include a local ID to inform all clients of the relation to the storage ID, without
|
|
56
|
+
* knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
|
|
57
|
+
* before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
|
|
55
58
|
*/
|
|
56
59
|
private readonly sendBlobAttachOp;
|
|
57
|
-
private readonly
|
|
60
|
+
private readonly blobRequested;
|
|
61
|
+
private readonly addedBlobReference;
|
|
62
|
+
private readonly isBlobDeleted;
|
|
58
63
|
private readonly runtime;
|
|
64
|
+
private readonly getCurrentReferenceTimestampMs;
|
|
59
65
|
static readonly basePath = "_blobs";
|
|
60
66
|
private static readonly redirectTableBlobName;
|
|
61
67
|
private readonly mc;
|
|
62
68
|
/**
|
|
63
|
-
* Map of local
|
|
64
|
-
*
|
|
65
|
-
*
|
|
66
|
-
*
|
|
69
|
+
* Map of local IDs to storage IDs. Contains identity entries (id → id) for storage IDs. All requested IDs should
|
|
70
|
+
* be a key in this map. Blobs created while the container is detached are stored in IDetachedBlobStorage which
|
|
71
|
+
* gives local IDs; the storage IDs are filled in at attach time.
|
|
72
|
+
* Note: It contains mappings from all clients, i.e., from remote clients as well. local ID comes from the client
|
|
73
|
+
* that uploaded the blob but its mapping to storage ID is needed in all clients in order to retrieve the blob.
|
|
67
74
|
*/
|
|
68
75
|
private readonly redirectTable;
|
|
69
76
|
/**
|
|
@@ -73,32 +80,34 @@ export declare class BlobManager {
|
|
|
73
80
|
*/
|
|
74
81
|
private readonly pendingBlobs;
|
|
75
82
|
/**
|
|
76
|
-
* Track ops in flight for online flow.
|
|
77
|
-
*
|
|
83
|
+
* Track ops in flight for online flow. This is used for optimizations where if we receive an ack for a storage ID,
|
|
84
|
+
* we can resolve all pending blobs with the same storage ID even though they may have different local IDs. That's
|
|
85
|
+
* because we know that the server will not delete the blob corresponding to that storage ID.
|
|
78
86
|
*/
|
|
79
87
|
private readonly opsInFlight;
|
|
80
88
|
private readonly retryThrottler;
|
|
81
89
|
/** If true, throw an error when a tombstone attachment blob is retrieved. */
|
|
82
|
-
private readonly
|
|
90
|
+
private readonly throwOnTombstoneLoad;
|
|
83
91
|
/**
|
|
84
|
-
* This stores
|
|
92
|
+
* This stores IDs of tombstoned blobs.
|
|
85
93
|
* Tombstone is a temporary feature that imitates a blob getting swept by garbage collection.
|
|
86
94
|
*/
|
|
87
95
|
private readonly tombstonedBlobs;
|
|
88
96
|
constructor(routeContext: IFluidHandleContext, snapshot: IBlobManagerLoadInfo, getStorage: () => IDocumentStorageService,
|
|
89
97
|
/**
|
|
90
|
-
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which
|
|
91
|
-
*
|
|
92
|
-
*
|
|
93
|
-
*
|
|
98
|
+
* Submit a BlobAttach op. When a blob is uploaded, there is a short grace period before which the blob is
|
|
99
|
+
* deleted. The BlobAttach op notifies the server that blob is in use. The server will then not delete the
|
|
100
|
+
* the blob as long as it is listed as referenced in future summaries. The summarizing client will know to
|
|
101
|
+
* include the storage ID in the summary when it sees the op.
|
|
94
102
|
*
|
|
95
|
-
* The op
|
|
96
|
-
*
|
|
97
|
-
*
|
|
103
|
+
* The op will also include a local ID to inform all clients of the relation to the storage ID, without
|
|
104
|
+
* knowledge of which they cannot request the blob from storage. It's important that this op is sequenced
|
|
105
|
+
* before any ops that reference the local ID, otherwise, an invalid handle could be added to the document.
|
|
98
106
|
*/
|
|
99
|
-
sendBlobAttachOp: (
|
|
107
|
+
sendBlobAttachOp: (localId: string, storageId?: string) => void, blobRequested: (blobPath: string) => void, addedBlobReference: (fromNodePath: string, toNodePath: string) => void, isBlobDeleted: (blobPath: string) => boolean, runtime: IBlobManagerRuntime, stashedBlobs: IPendingBlobs | undefined, getCurrentReferenceTimestampMs: () => number | undefined);
|
|
100
108
|
private get pendingOfflineUploads();
|
|
101
109
|
get hasPendingOfflineUploads(): boolean;
|
|
110
|
+
get hasPendingBlobs(): boolean;
|
|
102
111
|
/**
|
|
103
112
|
* Upload blobs added while offline. This must be completed before connecting and resubmitting ops.
|
|
104
113
|
*/
|
|
@@ -124,6 +133,12 @@ export declare class BlobManager {
|
|
|
124
133
|
private createBlobDetached;
|
|
125
134
|
createBlob(blob: ArrayBufferLike): Promise<IFluidHandle<ArrayBufferLike>>;
|
|
126
135
|
private uploadBlob;
|
|
136
|
+
/**
|
|
137
|
+
* Set up a mapping in the redirect table from fromId to toId. Also, notify the runtime that a reference is added
|
|
138
|
+
* which is required for GC.
|
|
139
|
+
*/
|
|
140
|
+
private setRedirection;
|
|
141
|
+
private deleteAndEmitsIfEmpty;
|
|
127
142
|
private onUploadResolve;
|
|
128
143
|
private onUploadReject;
|
|
129
144
|
private transitionToOffline;
|
|
@@ -133,6 +148,7 @@ export declare class BlobManager {
|
|
|
133
148
|
* @param metadata - op metadata containing storage and/or local IDs
|
|
134
149
|
*/
|
|
135
150
|
reSubmit(metadata: Record<string, unknown> | undefined): void;
|
|
151
|
+
private logTimeInfo;
|
|
136
152
|
processBlobAttachOp(message: ISequencedDocumentMessage, local: boolean): void;
|
|
137
153
|
/**
|
|
138
154
|
* Reads blobs needed to load BlobManager from storage.
|
|
@@ -153,18 +169,21 @@ export declare class BlobManager {
|
|
|
153
169
|
*/
|
|
154
170
|
getGCData(fullGC?: boolean): IGarbageCollectionData;
|
|
155
171
|
/**
|
|
156
|
-
* This is called to update blobs whose routes are
|
|
157
|
-
* @param
|
|
172
|
+
* This is called to update blobs whose routes are unused. The unused blobs are deleted.
|
|
173
|
+
* @param unusedRoutes - The routes of the blob nodes that are unused.
|
|
158
174
|
*/
|
|
159
|
-
|
|
175
|
+
updateUnusedRoutes(unusedRoutes: string[]): void;
|
|
160
176
|
/**
|
|
161
|
-
* This is called to update blobs whose routes are
|
|
162
|
-
*
|
|
163
|
-
* @param
|
|
164
|
-
|
|
165
|
-
|
|
177
|
+
* This is called to update blobs whose routes are tombstones. Tombstoned blobs enable testing scenarios with
|
|
178
|
+
* accessing deleted content without actually deleting content from summaries.
|
|
179
|
+
* @param tombstonedRoutes - The routes of blob nodes that are tombstones.
|
|
180
|
+
*/
|
|
181
|
+
updateTombstonedRoutes(tombstonedRoutes: string[]): void;
|
|
182
|
+
/**
|
|
183
|
+
* Verifies that the blob with given id is valid, i.e., it has not been garbage collected. If the blob is GC'd,
|
|
184
|
+
* log an error and throw if necessary.
|
|
166
185
|
*/
|
|
167
|
-
|
|
186
|
+
private verifyBlobValidity;
|
|
168
187
|
summarize(telemetryContext?: ITelemetryContext): ISummaryTreeWithStats;
|
|
169
188
|
setRedirectTable(table: Map<string, string>): void;
|
|
170
189
|
getPendingBlobs(): IPendingBlobs;
|
package/lib/blobManager.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,
|
|
1
|
+
{"version":3,"file":"blobManager.d.ts","sourceRoot":"","sources":["../src/blobManager.ts"],"names":[],"mappings":"AAAA;;;GAGG;AAGH,OAAO,EAAE,YAAY,EAAE,mBAAmB,EAAE,MAAM,iCAAiC,CAAC;AACpF,OAAO,EAAE,uBAAuB,EAAE,MAAM,oCAAoC,CAAC;AAC7E,OAAO,EAEN,yBAAyB,EACzB,aAAa,EACb,MAAM,sCAAsC,CAAC;AAO9C,OAAO,EAKN,iBAAiB,EACjB,MAAM,8BAA8B,CAAC;AACtC,OAAO,EACN,iBAAiB,EACjB,uBAAuB,EACvB,MAAM,+CAA+C,CAAC;AAQvD,OAAO,EACN,sBAAsB,EACtB,qBAAqB,EACrB,iBAAiB,EACjB,MAAM,qCAAqC,CAAC;AAO7C;;;;;;GAMG;AACH,qBAAa,UAAW,YAAW,YAAY,CAAC,eAAe,CAAC;aAc9C,IAAI,EAAE,MAAM;aACZ,YAAY,EAAE,mBAAmB;IAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAf/B,OAAO,CAAC,QAAQ,CAAkB;IAElC,IAAW,YAAY,IAAI,YAAY,CAEtC;IAED,IAAW,UAAU,IAAI,OAAO,CAE/B;IAED,SAAgB,YAAY,EAAE,MAAM,CAAC;gBAGpB,IAAI,EAAE,MAAM,EACZ,YAAY,EAAE,mBAAmB,EAC1C,GAAG,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC;IAKxB,WAAW;IAIX,IAAI,CAAC,MAAM,EAAE,YAAY;CAGhC;AAmBD;;GAEG;AACH,MAAM,WAAW,oBAAoB;IACpC,GAAG,CAAC,EAAE,MAAM,EAAE,CAAC;IACf,aAAa,CAAC,EAAE,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC;CACnC;AAID,oBAAY,mBAAmB,GAAG,IAAI,CACrC,iBAAiB,EACjB,aAAa,GAAG,WAAW,GAAG,QAAQ,GAAG,eAAe,CACxD,GACA,iBAAiB,CAAC,uBAAuB,CAAC,CAAC;AAwB5C,MAAM,WAAW,aAAa;IAC7B,CAAC,EAAE,EAAE,MAAM,GAAG;QAAE,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC;CAC/B;AAED,MAAM,WAAW,kBAAkB;IAClC,CAAC,KAAK,EAAE,gBAAgB,EAAE,QAAQ,EAAE,MAAM,IAAI,OAAE;CAChD;AAED,qBAAa,WAAY,SAAQ,iBAAiB,CAAC,kBAAkB,CAAC;IA8CpE,OAAO,CAAC,QAAQ,CAAC,YAAY;IAE7B,OAAO,CAAC,QAAQ,CAAC,UAAU;IAC3B;;;;;;;;;OASG;IACH,OAAO,CAAC,QAAQ,CAAC,gBAAgB;IAGjC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAI9B,OAAO,CAAC,QAAQ,CAAC,kBAAkB;IAGnC,OAAO,CAAC,QAAQ,CAAC,aAAa;IAC9B,OAAO,CAAC,QAAQ,CAAC,OAAO;IAExB,OAAO,CAAC,QAAQ,CAAC,8BAA8B;IAvEhD,gBAAuB,QAAQ,YAAY;IAC3C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,qBAAqB,CAAoB;IACjE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAoB;IAEvC;;;;;;OAMG;IACH,OAAO,CAAC,QAAQ,CAAC,aAAa,CAAkC;IAEhE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAuC;IAEpE;;;;OAIG;IACH,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAEhE,OAAO,CAAC,QAAQ,CAAC,cAAc,CAO7B;IAEF,6EAA6E;IAC7E,OAAO,CAAC,QAAQ,CAAC,oBAAoB,CAAU;IAC/C;;;OAGG;IACH,OAAO,CAAC,QAAQ,CAAC,eAAe,CAA0B;gBAGxC,YAAY,EAAE,mBAAmB,EAClD,QAAQ,EAAE,oBAAoB,EACb,UAAU,EAAE,MAAM,uBAAuB;IAC1D;;;;;;;;;OASG;IACc,gBAAgB,EAAE,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,CAAC,EAAE,MAAM,KAAK,IAAI,EAG/D,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,IAAI,EAIzC,kBAAkB,EAAE,CAAC,YAAY,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,KAAK,IAAI,EAGtE,aAAa,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,EAC5C,OAAO,EAAE,mBAAmB,EAC7C,YAAY,2BAAoB,EACf,8BAA8B,EAAE,MAAM,MAAM,GAAG,SAAS;IAwB1E,OAAO,KAAK,qBAAqB,GAIhC;IAED,IAAW,wBAAwB,IAAI,OAAO,CAE7C;IAED,IAAW,eAAe,IAAI,OAAO,CAKpC;IAED;;OAEG;IACU,WAAW;IAcxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAUtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;IAIzB;;;OAGG;IACH,OAAO,KAAK,UAAU,GAerB;IAEY,OAAO,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC;IAmC9D,OAAO,CAAC,aAAa;YAUP,kBAAkB;IAUnB,UAAU,CAAC,IAAI,EAAE,eAAe,GAAG,OAAO,CAAC,YAAY,CAAC,eAAe,CAAC,CAAC;YA4BxE,UAAU;IAYxB;;;OAGG;IACH,OAAO,CAAC,cAAc;IAStB,OAAO,CAAC,qBAAqB;IAS7B,OAAO,CAAC,eAAe;YAoDT,cAAc;IAkB5B,OAAO,CAAC,mBAAmB;IAkC3B;;;;OAIG;IACI,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,SAAS;IAoB7D,OAAO,CAAC,WAAW;IA0BZ,mBAAmB,CAAC,OAAO,EAAE,yBAAyB,EAAE,KAAK,EAAE,OAAO;IA2C7E;;;;;OAKG;WACiB,IAAI,CACvB,SAAS,EAAE,aAAa,GAAG,SAAS,EACpC,YAAY,EAAE,CAAC,EAAE,EAAE,MAAM,KAAK,OAAO,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,EAAE,CAAC,GACvD,OAAO,CAAC,oBAAoB,CAAC;IAehC;;OAEG;IACH,OAAO,CAAC,IAAI;IAgBZ;;;;;OAKG;IACI,SAAS,CAAC,MAAM,GAAE,OAAe,GAAG,sBAAsB;IASjE;;;OAGG;IACI,kBAAkB,CAAC,YAAY,EAAE,MAAM,EAAE,GAAG,IAAI;IAcvD;;;;OAIG;IACI,sBAAsB,CAAC,gBAAgB,EAAE,MAAM,EAAE;IA0BxD;;;OAGG;IACH,OAAO,CAAC,kBAAkB;IAiDnB,SAAS,CAAC,gBAAgB,CAAC,EAAE,iBAAiB,GAAG,qBAAqB;IA2BtE,gBAAgB,CAAC,KAAK,EAAE,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC;IAoB3C,eAAe,IAAI,aAAa;CAOvC"}
|