@fluidframework/container-runtime 2.0.0-dev.1.4.5.105745 → 2.0.0-dev.2.2.0.111723
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +1 -1
- package/dist/batchManager.d.ts +11 -6
- package/dist/batchManager.d.ts.map +1 -1
- package/dist/batchManager.js +23 -13
- package/dist/batchManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +74 -20
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +190 -137
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +6 -0
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +14 -21
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +71 -57
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.js +1 -1
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/dataStores.d.ts +11 -10
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +50 -20
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +36 -19
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +207 -121
- package/dist/garbageCollection.js.map +1 -1
- package/dist/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/dist/gcSweepReadyUsageDetection.js +3 -12
- package/dist/gcSweepReadyUsageDetection.js.map +1 -1
- package/dist/index.d.ts +4 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -5
- package/dist/index.js.map +1 -1
- package/dist/opCompressor.d.ts +18 -0
- package/dist/opCompressor.d.ts.map +1 -0
- package/dist/opCompressor.js +50 -0
- package/dist/opCompressor.js.map +1 -0
- package/dist/opDecompressor.d.ts +20 -0
- package/dist/opDecompressor.d.ts.map +1 -0
- package/dist/opDecompressor.js +72 -0
- package/dist/opDecompressor.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +6 -26
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +42 -62
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runningSummarizer.d.ts +3 -2
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +10 -3
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/scheduleManager.js.map +1 -1
- package/dist/summarizer.js +7 -2
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerClientElection.js +1 -1
- package/dist/summarizerClientElection.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +0 -3
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +19 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryFormat.d.ts +4 -2
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +3 -2
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +10 -6
- package/dist/summaryManager.js.map +1 -1
- package/garbageCollection.md +27 -22
- package/lib/batchManager.d.ts +11 -6
- package/lib/batchManager.d.ts.map +1 -1
- package/lib/batchManager.js +23 -13
- package/lib/batchManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +74 -20
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +189 -136
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +6 -0
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +14 -21
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +75 -61
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.js +1 -1
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/dataStores.d.ts +11 -10
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +53 -23
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +36 -19
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +208 -122
- package/lib/garbageCollection.js.map +1 -1
- package/lib/gcSweepReadyUsageDetection.d.ts.map +1 -1
- package/lib/gcSweepReadyUsageDetection.js +3 -12
- package/lib/gcSweepReadyUsageDetection.js.map +1 -1
- package/lib/index.d.ts +4 -6
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +2 -4
- package/lib/index.js.map +1 -1
- package/lib/opCompressor.d.ts +18 -0
- package/lib/opCompressor.d.ts.map +1 -0
- package/lib/opCompressor.js +46 -0
- package/lib/opCompressor.js.map +1 -0
- package/lib/opDecompressor.d.ts +20 -0
- package/lib/opDecompressor.d.ts.map +1 -0
- package/lib/opDecompressor.js +68 -0
- package/lib/opDecompressor.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +6 -26
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +42 -62
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runningSummarizer.d.ts +3 -2
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +10 -3
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/scheduleManager.js.map +1 -1
- package/lib/summarizer.js +7 -2
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerClientElection.js +1 -1
- package/lib/summarizerClientElection.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +0 -3
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +19 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryFormat.d.ts +4 -2
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +3 -2
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +10 -6
- package/lib/summaryManager.js.map +1 -1
- package/package.json +37 -63
- package/prettier.config.cjs +8 -0
- package/src/batchManager.ts +32 -15
- package/src/containerRuntime.ts +260 -156
- package/src/dataStore.ts +13 -1
- package/src/dataStoreContext.ts +100 -76
- package/src/dataStoreContexts.ts +1 -1
- package/src/dataStores.ts +61 -23
- package/src/garbageCollection.ts +257 -126
- package/src/gcSweepReadyUsageDetection.ts +2 -10
- package/src/index.ts +4 -4
- package/src/opCompressor.ts +59 -0
- package/src/opDecompressor.ts +82 -0
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +57 -96
- package/src/runningSummarizer.ts +11 -3
- package/src/scheduleManager.ts +1 -0
- package/src/summarizer.ts +6 -6
- package/src/summarizerClientElection.ts +1 -1
- package/src/summarizerHeuristics.ts +0 -3
- package/src/summarizerTypes.ts +20 -7
- package/src/summaryFormat.ts +4 -2
- package/src/summaryGenerator.ts +3 -2
- package/src/summaryManager.ts +18 -7
|
@@ -74,16 +74,8 @@ export class SweepReadyUsageDetectionHandler {
|
|
|
74
74
|
localStorageOverride?: Pick<Storage, "getItem" | "setItem">,
|
|
75
75
|
) {
|
|
76
76
|
const noopStorage = { getItem: () => null, setItem: () => {} };
|
|
77
|
-
|
|
78
|
-
|
|
79
|
-
} else {
|
|
80
|
-
try {
|
|
81
|
-
// localStorage is not defined in Node environment so this throws
|
|
82
|
-
this.localStorage = localStorage ?? noopStorage;
|
|
83
|
-
} catch (error) {
|
|
84
|
-
this.localStorage = noopStorage;
|
|
85
|
-
}
|
|
86
|
-
}
|
|
77
|
+
// localStorage is not defined in Node environment, so fall back to noopStorage if needed.
|
|
78
|
+
this.localStorage = localStorageOverride ?? globalThis.localStorage ?? noopStorage;
|
|
87
79
|
|
|
88
80
|
if (this.localStorage === noopStorage) {
|
|
89
81
|
// This means the Skip Closure Period logic will not work.
|
package/src/index.ts
CHANGED
|
@@ -23,23 +23,22 @@ export {
|
|
|
23
23
|
RuntimeHeaders,
|
|
24
24
|
ISummaryConfiguration,
|
|
25
25
|
DefaultSummaryConfiguration,
|
|
26
|
+
ICompressionRuntimeOptions,
|
|
27
|
+
CompressionAlgorithms,
|
|
26
28
|
} from "./containerRuntime";
|
|
27
|
-
export { DeltaScheduler } from "./deltaScheduler";
|
|
28
29
|
export { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
29
30
|
export {
|
|
30
31
|
gcBlobPrefix,
|
|
32
|
+
gcTombstoneBlobKey,
|
|
31
33
|
gcTreeKey,
|
|
32
|
-
IGarbageCollectionRuntime,
|
|
33
34
|
IGCStats,
|
|
34
35
|
} from "./garbageCollection";
|
|
35
36
|
export {
|
|
36
37
|
IPendingFlush,
|
|
37
|
-
IPendingFlushMode,
|
|
38
38
|
IPendingLocalState,
|
|
39
39
|
IPendingMessage,
|
|
40
40
|
IPendingState,
|
|
41
41
|
} from "./pendingStateManager";
|
|
42
|
-
export { ScheduleManager } from "./scheduleManager";
|
|
43
42
|
export { Summarizer } from "./summarizer";
|
|
44
43
|
export {
|
|
45
44
|
EnqueueSummarizeResult,
|
|
@@ -54,6 +53,7 @@ export {
|
|
|
54
53
|
INackSummaryResult,
|
|
55
54
|
IOnDemandSummarizeOptions,
|
|
56
55
|
IProvideSummarizer,
|
|
56
|
+
IRefreshSummaryAckOptions,
|
|
57
57
|
ISubmitSummaryOpResult,
|
|
58
58
|
ISubmitSummaryOptions,
|
|
59
59
|
ISummarizeOptions,
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
|
+
import { IsoBuffer } from "@fluidframework/common-utils";
|
|
8
|
+
import { ChildLogger } from "@fluidframework/telemetry-utils";
|
|
9
|
+
import { compress } from "lz4js";
|
|
10
|
+
import { BatchMessage } from "./batchManager";
|
|
11
|
+
import { CompressionAlgorithms, ContainerRuntimeMessage } from "./containerRuntime";
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Compresses batches of ops. It generates a single compressed op that contains
|
|
15
|
+
* the contents of each op in the batch. It then submits empty ops for each original
|
|
16
|
+
* op to reserve sequence numbers.
|
|
17
|
+
*/
|
|
18
|
+
export class OpCompressor {
|
|
19
|
+
private readonly logger;
|
|
20
|
+
private compressedBatchCount = 0;
|
|
21
|
+
|
|
22
|
+
constructor(logger: ITelemetryLogger) {
|
|
23
|
+
this.logger = ChildLogger.create(logger, "OpCompressor");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
public compressBatch(batch: BatchMessage[], originalLength: number): BatchMessage[] {
|
|
27
|
+
const batchToSend: BatchMessage[] = [];
|
|
28
|
+
this.compressedBatchCount++;
|
|
29
|
+
const batchedContents: ContainerRuntimeMessage[] = [];
|
|
30
|
+
for (const message of batch) {
|
|
31
|
+
batchedContents.push(message.deserializedContent);
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
const compressionStart = Date.now();
|
|
35
|
+
const contentsAsBuffer = new TextEncoder().encode(JSON.stringify(batchedContents));
|
|
36
|
+
const compressedContents = compress(contentsAsBuffer);
|
|
37
|
+
const compressedContent = IsoBuffer.from(compressedContents).toString("base64");
|
|
38
|
+
const duration = Date.now() - compressionStart;
|
|
39
|
+
|
|
40
|
+
if (originalLength > 200000 || this.compressedBatchCount % 100) {
|
|
41
|
+
this.logger.sendPerformanceEvent({
|
|
42
|
+
eventName: "CompressedBatch",
|
|
43
|
+
duration,
|
|
44
|
+
sizeBeforeCompression: originalLength,
|
|
45
|
+
sizeAfterCompression: compressedContent.length,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
batchToSend.push({ ...batch[0], contents: JSON.stringify({ packedContents: compressedContent }),
|
|
50
|
+
metadata: { ...batch[0].metadata, compressed: true },
|
|
51
|
+
compression: CompressionAlgorithms.lz4 });
|
|
52
|
+
|
|
53
|
+
for (const message of batch.slice(1)) {
|
|
54
|
+
batchToSend.push({ ...message, contents: undefined });
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return batchToSend;
|
|
58
|
+
}
|
|
59
|
+
}
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { decompress } from "lz4js";
|
|
7
|
+
import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
|
|
8
|
+
import { assert, IsoBuffer, Uint8ArrayToString } from "@fluidframework/common-utils";
|
|
9
|
+
import { CompressionAlgorithms } from ".";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* State machine that "unrolls" contents of compressed batches of ops after decompressing them.
|
|
13
|
+
* This class relies on some implicit contracts defined below:
|
|
14
|
+
* 1. A compressed batch will have its first message with batch metadata set to true and compressed set to true
|
|
15
|
+
* 2. Messages in the middle of a compressed batch will have neither batch metadata nor the compression property set
|
|
16
|
+
* 3. The final message of a batch will have batch metadata set to false
|
|
17
|
+
* 4. An individually compressed op will have undefined batch metadata and compression set to true
|
|
18
|
+
*/
|
|
19
|
+
export class OpDecompressor {
|
|
20
|
+
private activeBatch = false;
|
|
21
|
+
private rootMessageContents: any | undefined;
|
|
22
|
+
private processedCount = 0;
|
|
23
|
+
|
|
24
|
+
public processMessage(message: ISequencedDocumentMessage): ISequencedDocumentMessage {
|
|
25
|
+
// We're checking for compression = true or top level compression property so
|
|
26
|
+
// that we can enable compression without waiting on all ordering services
|
|
27
|
+
// to pick up protocol change. Eventually only the top level property should
|
|
28
|
+
// be used.
|
|
29
|
+
if (message.metadata?.batch === true
|
|
30
|
+
&& (message.metadata?.compressed || message.compression !== undefined)) {
|
|
31
|
+
// Beginning of a compressed batch
|
|
32
|
+
assert(this.activeBatch === false, "shouldn't have multiple active batches");
|
|
33
|
+
if (message.compression) {
|
|
34
|
+
// lz4 is the only supported compression algorithm for now
|
|
35
|
+
assert(message.compression === CompressionAlgorithms.lz4,
|
|
36
|
+
"lz4 is currently the only supported compression algorithm");
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
this.activeBatch = true;
|
|
40
|
+
|
|
41
|
+
const contents = IsoBuffer.from(message.contents.packedContents, "base64");
|
|
42
|
+
const decompressedMessage = decompress(contents);
|
|
43
|
+
const intoString = Uint8ArrayToString(decompressedMessage);
|
|
44
|
+
const asObj = JSON.parse(intoString);
|
|
45
|
+
this.rootMessageContents = asObj;
|
|
46
|
+
|
|
47
|
+
return { ...message, contents: this.rootMessageContents[this.processedCount++] };
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
if (this.rootMessageContents !== undefined && message.metadata?.batch === undefined && this.activeBatch) {
|
|
51
|
+
// Continuation of compressed batch
|
|
52
|
+
return { ...message, contents: this.rootMessageContents[this.processedCount++] };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
if (this.rootMessageContents !== undefined && message.metadata?.batch === false) {
|
|
56
|
+
// End of compressed batch
|
|
57
|
+
const returnMessage = { ...message,
|
|
58
|
+
contents: this.rootMessageContents[this.processedCount++] };
|
|
59
|
+
|
|
60
|
+
this.activeBatch = false;
|
|
61
|
+
this.rootMessageContents = undefined;
|
|
62
|
+
this.processedCount = 0;
|
|
63
|
+
|
|
64
|
+
return returnMessage;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
if (message.metadata?.batch === undefined &&
|
|
68
|
+
(message.metadata?.compressed || message.compression === CompressionAlgorithms.lz4)) {
|
|
69
|
+
// Single compressed message
|
|
70
|
+
assert(this.activeBatch === false, "shouldn't receive compressed message in middle of a batch");
|
|
71
|
+
|
|
72
|
+
const contents = IsoBuffer.from(message.contents.packedContents, "base64");
|
|
73
|
+
const decompressedMessage = decompress(contents);
|
|
74
|
+
const intoString = new TextDecoder().decode(decompressedMessage);
|
|
75
|
+
const asObj = JSON.parse(intoString);
|
|
76
|
+
|
|
77
|
+
return { ...message, contents: asObj[0] };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
return message;
|
|
81
|
+
}
|
|
82
|
+
}
|
package/src/packageVersion.ts
CHANGED
|
@@ -10,7 +10,6 @@ import { DataProcessingError } from "@fluidframework/container-utils";
|
|
|
10
10
|
import {
|
|
11
11
|
ISequencedDocumentMessage,
|
|
12
12
|
} from "@fluidframework/protocol-definitions";
|
|
13
|
-
import { FlushMode } from "@fluidframework/runtime-definitions";
|
|
14
13
|
import Deque from "double-ended-queue";
|
|
15
14
|
import { ContainerMessageType } from "./containerRuntime";
|
|
16
15
|
import { pkgVersion } from "./packageVersion";
|
|
@@ -29,15 +28,6 @@ export interface IPendingMessage {
|
|
|
29
28
|
opMetadata: Record<string, unknown> | undefined;
|
|
30
29
|
}
|
|
31
30
|
|
|
32
|
-
/**
|
|
33
|
-
* This represents a FlushMode update and is added to the pending queue when `setFlushMode` is called on the
|
|
34
|
-
* ContainerRuntime and the FlushMode changes.
|
|
35
|
-
*/
|
|
36
|
-
export interface IPendingFlushMode {
|
|
37
|
-
type: "flushMode";
|
|
38
|
-
flushMode: FlushMode;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
31
|
/**
|
|
42
32
|
* This represents an explicit flush call and is added to the pending queue when flush is called on the ContainerRuntime
|
|
43
33
|
* to flush pending messages.
|
|
@@ -46,7 +36,7 @@ export interface IPendingFlush {
|
|
|
46
36
|
type: "flush";
|
|
47
37
|
}
|
|
48
38
|
|
|
49
|
-
export type IPendingState = IPendingMessage |
|
|
39
|
+
export type IPendingState = IPendingMessage | IPendingFlush;
|
|
50
40
|
|
|
51
41
|
export interface IPendingLocalState {
|
|
52
42
|
/**
|
|
@@ -58,8 +48,6 @@ export interface IPendingLocalState {
|
|
|
58
48
|
export interface IRuntimeStateHandler{
|
|
59
49
|
connected(): boolean;
|
|
60
50
|
clientId(): string | undefined;
|
|
61
|
-
flushMode(): FlushMode;
|
|
62
|
-
setFlushMode(mode: FlushMode): void;
|
|
63
51
|
close(error?: ICriticalContainerError): void;
|
|
64
52
|
applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
|
|
65
53
|
flush(): void;
|
|
@@ -68,13 +56,18 @@ export interface IRuntimeStateHandler{
|
|
|
68
56
|
content: any,
|
|
69
57
|
localOpMetadata: unknown,
|
|
70
58
|
opMetadata: Record<string, unknown> | undefined): void;
|
|
59
|
+
rollback(
|
|
60
|
+
type: ContainerMessageType,
|
|
61
|
+
content: any,
|
|
62
|
+
localOpMetadata: unknown): void;
|
|
63
|
+
orderSequentially(callback: () => void): void;
|
|
71
64
|
}
|
|
72
65
|
|
|
73
66
|
/**
|
|
74
67
|
* PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
|
|
75
68
|
* acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
|
|
76
69
|
* batches along with the messages.
|
|
77
|
-
* When the Container reconnects, it replays the pending states, which includes
|
|
70
|
+
* When the Container reconnects, it replays the pending states, which includes manual flushing
|
|
78
71
|
* of messages and triggering resubmission of unacked ops.
|
|
79
72
|
*
|
|
80
73
|
* It verifies that all the ops are acked, are received in the right order and batch information is correct.
|
|
@@ -100,13 +93,6 @@ export class PendingStateManager implements IDisposable {
|
|
|
100
93
|
// the correct batch metadata.
|
|
101
94
|
private pendingBatchBeginMessage: ISequencedDocumentMessage | undefined;
|
|
102
95
|
|
|
103
|
-
/**
|
|
104
|
-
* This tracks the flush mode for the next message in the pending state queue. When replaying messages, we need to
|
|
105
|
-
* first set the flush mode to this value and then send ops. It is important to do this info because the flush
|
|
106
|
-
* mode could have been updated.
|
|
107
|
-
*/
|
|
108
|
-
private flushModeForNextMessage: FlushMode;
|
|
109
|
-
|
|
110
96
|
private clientId: string | undefined;
|
|
111
97
|
|
|
112
98
|
/**
|
|
@@ -131,13 +117,9 @@ export class PendingStateManager implements IDisposable {
|
|
|
131
117
|
|
|
132
118
|
constructor(
|
|
133
119
|
private readonly stateHandler: IRuntimeStateHandler,
|
|
134
|
-
initialFlushMode: FlushMode,
|
|
135
120
|
initialLocalState: IPendingLocalState | undefined,
|
|
136
121
|
) {
|
|
137
122
|
this.initialStates = new Deque<IPendingState>(initialLocalState?.pendingStates ?? []);
|
|
138
|
-
|
|
139
|
-
this.flushModeForNextMessage = initialFlushMode;
|
|
140
|
-
this.onFlushModeUpdated(initialFlushMode);
|
|
141
123
|
}
|
|
142
124
|
|
|
143
125
|
public get disposed() { return this.disposeOnce.evaluated; }
|
|
@@ -174,24 +156,10 @@ export class PendingStateManager implements IDisposable {
|
|
|
174
156
|
this._pendingMessagesCount++;
|
|
175
157
|
}
|
|
176
158
|
|
|
177
|
-
/**
|
|
178
|
-
* Called when the FlushMode is updated. Adds the FlushMode to the pending state queue.
|
|
179
|
-
* @param flushMode - The flushMode that was updated.
|
|
180
|
-
*/
|
|
181
|
-
public onFlushModeUpdated(flushMode: FlushMode) {
|
|
182
|
-
this.pendingStates.push({ type: "flushMode", flushMode });
|
|
183
|
-
}
|
|
184
|
-
|
|
185
159
|
/**
|
|
186
160
|
* Called when flush() is called on the ContainerRuntime to manually flush messages.
|
|
187
161
|
*/
|
|
188
162
|
public onFlush() {
|
|
189
|
-
// If the FlushMode is Immediate, we don't need to track an explicit flush call because every message is
|
|
190
|
-
// automatically flushed. So, flush is a no-op.
|
|
191
|
-
if (this.stateHandler.flushMode() === FlushMode.Immediate) {
|
|
192
|
-
return;
|
|
193
|
-
}
|
|
194
|
-
|
|
195
163
|
// If the previous state is not a message, flush is a no-op.
|
|
196
164
|
const previousState = this.pendingStates.peekBack();
|
|
197
165
|
if (previousState?.type !== "message") {
|
|
@@ -279,11 +247,6 @@ export class PendingStateManager implements IDisposable {
|
|
|
279
247
|
* @param message - The message that is being processed.
|
|
280
248
|
*/
|
|
281
249
|
private maybeProcessBatchBegin(message: ISequencedDocumentMessage) {
|
|
282
|
-
// Tracks the last FlushMode that was set before this message was sent.
|
|
283
|
-
let pendingFlushMode: FlushMode | undefined;
|
|
284
|
-
// Tracks whether a flush was called before this message was sent.
|
|
285
|
-
let pendingFlush: boolean = false;
|
|
286
|
-
|
|
287
250
|
/**
|
|
288
251
|
* We are checking if the next message is the start of a batch. It can happen in the following scenarios:
|
|
289
252
|
*
|
|
@@ -295,34 +258,12 @@ export class PendingStateManager implements IDisposable {
|
|
|
295
258
|
* Keep reading pending states from the queue until we encounter a message. It's possible that the FlushMode was
|
|
296
259
|
* updated a bunch of times without sending any messages.
|
|
297
260
|
*/
|
|
298
|
-
|
|
299
|
-
while (nextPendingState.type !== "message") {
|
|
300
|
-
if (nextPendingState.type === "flushMode") {
|
|
301
|
-
pendingFlushMode = nextPendingState.flushMode;
|
|
302
|
-
}
|
|
303
|
-
if (nextPendingState.type === "flush") {
|
|
304
|
-
pendingFlush = true;
|
|
305
|
-
}
|
|
261
|
+
while (this.peekNextPendingState().type !== "message") {
|
|
306
262
|
this.pendingStates.shift();
|
|
307
|
-
nextPendingState = this.peekNextPendingState();
|
|
308
263
|
}
|
|
309
264
|
|
|
310
|
-
if
|
|
311
|
-
|
|
312
|
-
}
|
|
313
|
-
|
|
314
|
-
// If the FlushMode was set to Immediate before this message was sent, this message won't be a batch message
|
|
315
|
-
// because in Immediate mode, every message is flushed individually.
|
|
316
|
-
if (pendingFlushMode === FlushMode.Immediate) {
|
|
317
|
-
return;
|
|
318
|
-
}
|
|
319
|
-
|
|
320
|
-
/**
|
|
321
|
-
* This message is the first in a batch if before it was sent either the FlushMode was set to TurnBased or there
|
|
322
|
-
* was an explicit flush call. Note that a flush call is tracked only in TurnBased mode and it indicates the end
|
|
323
|
-
* of one batch and beginning of another.
|
|
324
|
-
*/
|
|
325
|
-
if (pendingFlushMode === FlushMode.TurnBased || pendingFlush) {
|
|
265
|
+
// This message is the first in a batch if the "batch" property on the metadata is set to true
|
|
266
|
+
if (message.metadata?.batch) {
|
|
326
267
|
// We should not already be processing a batch and there should be no pending batch begin message.
|
|
327
268
|
assert(!this.isProcessingBatch && this.pendingBatchBeginMessage === undefined,
|
|
328
269
|
0x16b /* "The pending batch state indicates we are already processing a batch" */);
|
|
@@ -347,16 +288,6 @@ export class PendingStateManager implements IDisposable {
|
|
|
347
288
|
return;
|
|
348
289
|
}
|
|
349
290
|
|
|
350
|
-
/**
|
|
351
|
-
* We are in the middle of processing a batch. The batch ends when we see an explicit flush. We should never see
|
|
352
|
-
* a FlushMode before flush. This is true because we track batches only when FlushMode is TurnBased and in this
|
|
353
|
-
* mode, a batch ends either by calling flush or by changing the mode to Immediate which also triggers a flush.
|
|
354
|
-
*/
|
|
355
|
-
assert(
|
|
356
|
-
nextPendingState.type !== "flushMode",
|
|
357
|
-
0x2bd /* "We should not see a pending FlushMode until we see a flush when processing a batch" */,
|
|
358
|
-
);
|
|
359
|
-
|
|
360
291
|
// There should be a pending batch begin message.
|
|
361
292
|
assert(this.pendingBatchBeginMessage !== undefined, 0x16d /* "There is no pending batch begin message" */);
|
|
362
293
|
|
|
@@ -406,7 +337,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
406
337
|
|
|
407
338
|
/**
|
|
408
339
|
* Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
|
|
409
|
-
* states in its queue. This includes
|
|
340
|
+
* states in its queue. This includes triggering resubmission of unacked ops.
|
|
410
341
|
*/
|
|
411
342
|
public replayPendingStates() {
|
|
412
343
|
assert(this.stateHandler.connected(), 0x172 /* "The connection state is not consistent with the runtime" */);
|
|
@@ -426,12 +357,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
426
357
|
// Reset the pending message count because all these messages will be removed from the queue.
|
|
427
358
|
this._pendingMessagesCount = 0;
|
|
428
359
|
|
|
429
|
-
|
|
430
|
-
const savedFlushMode = this.stateHandler.flushMode();
|
|
431
|
-
|
|
432
|
-
// Set the flush mode for the next message. This step is important because the flush mode may have been changed
|
|
433
|
-
// after the next pending message was sent.
|
|
434
|
-
this.stateHandler.setFlushMode(this.flushModeForNextMessage);
|
|
360
|
+
const messageBatchQueue = new Deque<IPendingMessage>();
|
|
435
361
|
|
|
436
362
|
// Process exactly `pendingStatesCount` items in the queue as it represents the number of states that were
|
|
437
363
|
// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
|
|
@@ -441,16 +367,43 @@ export class PendingStateManager implements IDisposable {
|
|
|
441
367
|
const pendingState = this.pendingStates.shift()!;
|
|
442
368
|
switch (pendingState.type) {
|
|
443
369
|
case "message":
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
370
|
+
assert(pendingState.opMetadata?.batch !== false || messageBatchQueue.length > 0,
|
|
371
|
+
0x41b /* We cannot process batches in chunks */);
|
|
372
|
+
/**
|
|
373
|
+
* We want to ensure grouped messages get processed in a batch.
|
|
374
|
+
* Note: It is not possible for the PendingStateManager to receive a partially acked batch. It will
|
|
375
|
+
* either receive the whole batch ack or nothing at all.
|
|
376
|
+
*/
|
|
377
|
+
if (messageBatchQueue.length > 0 || pendingState.opMetadata?.batch) {
|
|
378
|
+
messageBatchQueue.enqueue(pendingState);
|
|
379
|
+
} else {
|
|
380
|
+
this.stateHandler.reSubmit(
|
|
381
|
+
pendingState.messageType,
|
|
382
|
+
pendingState.content,
|
|
383
|
+
pendingState.localOpMetadata,
|
|
384
|
+
pendingState.opMetadata);
|
|
385
|
+
}
|
|
452
386
|
break;
|
|
453
387
|
case "flush":
|
|
388
|
+
/**
|
|
389
|
+
* A "flush" call can indicate the end of a batch.
|
|
390
|
+
* We can't rely on the "batch" property in the message metadata as it gets
|
|
391
|
+
* updated elsewhere and it is not the same object instance that gets updated.
|
|
392
|
+
*/
|
|
393
|
+
if (messageBatchQueue.length > 0) {
|
|
394
|
+
this.stateHandler.orderSequentially(() => {
|
|
395
|
+
while (messageBatchQueue.length > 0) {
|
|
396
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
397
|
+
const message = messageBatchQueue.dequeue()!;
|
|
398
|
+
this.stateHandler.reSubmit(
|
|
399
|
+
message.messageType,
|
|
400
|
+
message.content,
|
|
401
|
+
message.localOpMetadata,
|
|
402
|
+
message.opMetadata);
|
|
403
|
+
}
|
|
404
|
+
});
|
|
405
|
+
}
|
|
406
|
+
assert(messageBatchQueue.length === 0, 0x41c /* cannot flush in the middle of a batch */);
|
|
454
407
|
this.stateHandler.flush();
|
|
455
408
|
break;
|
|
456
409
|
default:
|
|
@@ -459,7 +412,15 @@ export class PendingStateManager implements IDisposable {
|
|
|
459
412
|
pendingStatesCount--;
|
|
460
413
|
}
|
|
461
414
|
|
|
462
|
-
//
|
|
463
|
-
|
|
415
|
+
// There are some cases where ops are stashed but not flushed. We need to ensure they are resubmitted
|
|
416
|
+
while (messageBatchQueue.length > 0) {
|
|
417
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
418
|
+
const message = messageBatchQueue.dequeue()!;
|
|
419
|
+
this.stateHandler.reSubmit(
|
|
420
|
+
message.messageType,
|
|
421
|
+
message.content,
|
|
422
|
+
message.localOpMetadata,
|
|
423
|
+
message.opMetadata);
|
|
424
|
+
}
|
|
464
425
|
}
|
|
465
426
|
}
|
package/src/runningSummarizer.ts
CHANGED
|
@@ -263,9 +263,9 @@ export class RunningSummarizer implements IDisposable {
|
|
|
263
263
|
|
|
264
264
|
/**
|
|
265
265
|
* Can the given op trigger a summary?
|
|
266
|
-
* # Currently
|
|
266
|
+
* # Currently always prevents summaries for Summarize and SummaryAck/Nack ops
|
|
267
267
|
* @param op - op to check
|
|
268
|
-
* @returns true if this
|
|
268
|
+
* @returns true if this op can trigger a summary
|
|
269
269
|
*/
|
|
270
270
|
private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
|
|
271
271
|
switch (op.type) {
|
|
@@ -274,10 +274,18 @@ export class RunningSummarizer implements IDisposable {
|
|
|
274
274
|
case MessageType.SummaryNack:
|
|
275
275
|
return false;
|
|
276
276
|
default:
|
|
277
|
-
return
|
|
277
|
+
return isRuntimeMessage(op) || this.nonRuntimeOpCanTriggerSummary();
|
|
278
278
|
}
|
|
279
279
|
}
|
|
280
280
|
|
|
281
|
+
private nonRuntimeOpCanTriggerSummary(): boolean {
|
|
282
|
+
// eslint-disable-next-line max-len
|
|
283
|
+
const opsSinceLastAck = this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
|
|
284
|
+
return this.configuration.state === "enabled"
|
|
285
|
+
&& (this.configuration.nonRuntimeHeuristicThreshold === undefined
|
|
286
|
+
|| this.configuration.nonRuntimeHeuristicThreshold <= opsSinceLastAck);
|
|
287
|
+
}
|
|
288
|
+
|
|
281
289
|
public async waitStop(allowLastSummary: boolean): Promise<void> {
|
|
282
290
|
if (this.stopping) {
|
|
283
291
|
return;
|
package/src/scheduleManager.ts
CHANGED
package/src/summarizer.ts
CHANGED
|
@@ -334,7 +334,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
334
334
|
const coordinatorCreateP = this.runCoordinatorCreateFn(this.runtime);
|
|
335
335
|
|
|
336
336
|
coordinatorCreateP.then((runCoordinator) => {
|
|
337
|
-
//
|
|
337
|
+
// Successfully created the cancellation token. Start the summarizer.
|
|
338
338
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
339
339
|
const startP = this.start(this.runtime.clientId!, runCoordinator);
|
|
340
340
|
startP.then(async (runningSummarizer) => {
|
|
@@ -382,11 +382,11 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
382
382
|
// executing the refreshLatestSummaryAck.
|
|
383
383
|
// https://dev.azure.com/fluidframework/internal/_workitems/edit/779
|
|
384
384
|
await this.runningSummarizer.lockedRefreshSummaryAckAction(async () =>
|
|
385
|
-
this.internalsProvider.refreshLatestSummaryAck(
|
|
386
|
-
summaryOpHandle,
|
|
387
|
-
summaryAckHandle,
|
|
388
|
-
refSequenceNumber,
|
|
389
|
-
summaryLogger,
|
|
385
|
+
this.internalsProvider.refreshLatestSummaryAck({
|
|
386
|
+
proposalHandle: summaryOpHandle,
|
|
387
|
+
ackHandle: summaryAckHandle,
|
|
388
|
+
summaryRefSeq: refSequenceNumber,
|
|
389
|
+
summaryLogger },
|
|
390
390
|
).catch(async (error) => {
|
|
391
391
|
// If the error is 404, so maybe the fetched version no longer exists on server. We just
|
|
392
392
|
// ignore this error in that case, as that means we will have another summaryAck for the
|
|
@@ -77,7 +77,7 @@ export class SummarizerClientElection
|
|
|
77
77
|
// Log and elect a new summarizer client.
|
|
78
78
|
const opsSinceLastReport = sequenceNumber - this.lastReportedSeq;
|
|
79
79
|
if (opsSinceLastReport > this.maxOpsSinceLastSummary) {
|
|
80
|
-
this.logger.
|
|
80
|
+
this.logger.sendTelemetryEvent({
|
|
81
81
|
eventName: "ElectedClientNotSummarizing",
|
|
82
82
|
electedClientId,
|
|
83
83
|
lastSummaryAckSeqForClient: this.lastSummaryAckSeqForClient,
|
|
@@ -119,9 +119,6 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
|
|
|
119
119
|
}
|
|
120
120
|
|
|
121
121
|
public get idleTime(): number {
|
|
122
|
-
if (this.configuration.idleTime !== undefined) {
|
|
123
|
-
return this.configuration.idleTime;
|
|
124
|
-
}
|
|
125
122
|
const maxIdleTime = this.configuration.maxIdleTime;
|
|
126
123
|
const minIdleTime = this.configuration.minIdleTime;
|
|
127
124
|
const weightedNumOfOps = getWeightedNumberOfOps(
|
package/src/summarizerTypes.ts
CHANGED
|
@@ -61,12 +61,7 @@ export interface ISummarizerInternalsProvider {
|
|
|
61
61
|
submitSummary(options: ISubmitSummaryOptions): Promise<SubmitSummaryResult>;
|
|
62
62
|
|
|
63
63
|
/** Callback whenever a new SummaryAck is received, to update internal tracking state */
|
|
64
|
-
refreshLatestSummaryAck(
|
|
65
|
-
proposalHandle: string,
|
|
66
|
-
ackHandle: string,
|
|
67
|
-
summaryRefSeq: number,
|
|
68
|
-
summaryLogger: ITelemetryLogger,
|
|
69
|
-
): Promise<void>;
|
|
64
|
+
refreshLatestSummaryAck(options: IRefreshSummaryAckOptions): Promise<void>;
|
|
70
65
|
}
|
|
71
66
|
|
|
72
67
|
/**
|
|
@@ -114,6 +109,20 @@ export interface ISummarizeOptions {
|
|
|
114
109
|
readonly refreshLatestAck?: boolean;
|
|
115
110
|
}
|
|
116
111
|
|
|
112
|
+
/**
|
|
113
|
+
* Data required to update internal tracking state after receiving a Summary Ack.
|
|
114
|
+
*/
|
|
115
|
+
export interface IRefreshSummaryAckOptions {
|
|
116
|
+
/** Handle from the ack's summary op. */
|
|
117
|
+
readonly proposalHandle: string | undefined;
|
|
118
|
+
/** Handle from the summary ack just received */
|
|
119
|
+
readonly ackHandle: string;
|
|
120
|
+
/** Reference sequence number from the ack's summary op */
|
|
121
|
+
readonly summaryRefSeq: number;
|
|
122
|
+
/** Telemetry logger to which telemetry events will be forwarded. */
|
|
123
|
+
readonly summaryLogger: ITelemetryLogger;
|
|
124
|
+
}
|
|
125
|
+
|
|
117
126
|
export interface ISubmitSummaryOptions extends ISummarizeOptions {
|
|
118
127
|
/** Logger to use for correlated summary events */
|
|
119
128
|
readonly summaryLogger: ITelemetryLogger;
|
|
@@ -292,7 +301,11 @@ export type SummarizerStopReason =
|
|
|
292
301
|
* client to no longer be elected as responsible for summaries. Then it
|
|
293
302
|
* tries to stop its spawned summarizer client.
|
|
294
303
|
*/
|
|
295
|
-
| "
|
|
304
|
+
| "notElectedParent"
|
|
305
|
+
/**
|
|
306
|
+
* We are not already running the summarizer and we are not the current elected client id.
|
|
307
|
+
*/
|
|
308
|
+
| "notElectedClient"
|
|
296
309
|
/** Summarizer client was disconnected */
|
|
297
310
|
| "summarizerClientDisconnected"
|
|
298
311
|
/* running summarizer threw an exception */
|
package/src/summaryFormat.ts
CHANGED
|
@@ -99,14 +99,16 @@ export interface IGCMetadata {
|
|
|
99
99
|
* - A value greater than 0 means GC is enabled.
|
|
100
100
|
*/
|
|
101
101
|
readonly gcFeature?: GCVersion;
|
|
102
|
-
/** If this is present, the session for this container will expire after this time and the container will close */
|
|
103
|
-
readonly sessionExpiryTimeoutMs?: number;
|
|
104
102
|
/**
|
|
105
103
|
* Tells whether the GC sweep phase is enabled for this container.
|
|
106
104
|
* - True means sweep phase is enabled.
|
|
107
105
|
* - False means sweep phase is disabled. If GC is disabled as per gcFeature, sweep is also disabled.
|
|
108
106
|
*/
|
|
109
107
|
readonly sweepEnabled?: boolean;
|
|
108
|
+
/** If this is present, the session for this container will expire after this time and the container will close */
|
|
109
|
+
readonly sessionExpiryTimeoutMs?: number;
|
|
110
|
+
/** How long to wait after an object is unreferenced before deleting it via GC Sweep */
|
|
111
|
+
readonly sweepTimeoutMs?: number;
|
|
110
112
|
}
|
|
111
113
|
|
|
112
114
|
/** The properties of an ISequencedDocumentMessage to be stored in the metadata blob in summary. */
|
package/src/summaryGenerator.ts
CHANGED
|
@@ -231,13 +231,14 @@ export class SummaryGenerator {
|
|
|
231
231
|
const category = cancellationToken.cancelled || error?.errorType === DriverErrorType.offlineError ?
|
|
232
232
|
"generic" : "error";
|
|
233
233
|
|
|
234
|
+
const message = getFailMessage(errorCode);
|
|
234
235
|
summarizeEvent.cancel({
|
|
235
236
|
...properties,
|
|
236
237
|
reason: errorCode,
|
|
237
238
|
category,
|
|
238
239
|
retryAfterSeconds,
|
|
239
|
-
}, error);
|
|
240
|
-
resultsBuilder.fail(
|
|
240
|
+
}, error ?? message); // disconnect & summaryAckTimeout do not have proper error.
|
|
241
|
+
resultsBuilder.fail(message, error, nackSummaryResult, retryAfterSeconds);
|
|
241
242
|
};
|
|
242
243
|
|
|
243
244
|
// Wait to generate and send summary
|