@fluidframework/container-runtime 1.2.0 → 2.0.0-internal.1.0.0.81589
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts +81 -25
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +301 -100
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +46 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +98 -79
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +32 -26
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -23
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +5 -2
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +11 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/opProperties.d.ts +7 -0
- package/dist/opProperties.d.ts.map +1 -0
- package/dist/opProperties.js +19 -0
- package/dist/opProperties.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +14 -4
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +68 -26
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts +0 -2
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +1 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +26 -4
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +95 -18
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +30 -10
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.js +1 -1
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts +0 -5
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts +1 -0
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +11 -9
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts +81 -25
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +302 -101
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +46 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +99 -80
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +32 -26
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +17 -24
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +5 -2
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +11 -3
- package/lib/dataStores.js.map +1 -1
- package/lib/opProperties.d.ts +7 -0
- package/lib/opProperties.d.ts.map +1 -0
- package/lib/opProperties.js +15 -0
- package/lib/opProperties.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +14 -4
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +68 -26
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts +0 -2
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +1 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +26 -4
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +95 -18
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +30 -10
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.js +1 -1
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts +0 -5
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts +1 -0
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +11 -9
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +45 -20
- package/src/blobManager.ts +360 -119
- package/src/containerRuntime.ts +165 -89
- package/src/dataStore.ts +53 -38
- package/src/dataStoreContext.ts +16 -23
- package/src/dataStores.ts +14 -3
- package/src/opProperties.ts +19 -0
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +75 -22
- package/src/summarizer.ts +1 -18
- package/src/summarizerHeuristics.ts +133 -19
- package/src/summarizerTypes.ts +37 -10
- package/src/summaryCollection.ts +1 -1
- package/src/summaryFormat.ts +0 -6
- package/src/summaryGenerator.ts +40 -22
- package/dist/opTelemetry.d.ts +0 -22
- package/dist/opTelemetry.d.ts.map +0 -1
- package/dist/opTelemetry.js +0 -59
- package/dist/opTelemetry.js.map +0 -1
- package/lib/opTelemetry.d.ts +0 -22
- package/lib/opTelemetry.d.ts.map +0 -1
- package/lib/opTelemetry.js +0 -55
- package/lib/opTelemetry.js.map +0 -1
- package/src/opTelemetry.ts +0 -71
package/src/dataStoreContext.ts
CHANGED
|
@@ -13,7 +13,6 @@ import {
|
|
|
13
13
|
import {
|
|
14
14
|
IAudience,
|
|
15
15
|
IDeltaManager,
|
|
16
|
-
BindState,
|
|
17
16
|
AttachState,
|
|
18
17
|
ILoaderOptions,
|
|
19
18
|
} from "@fluidframework/container-definitions";
|
|
@@ -232,7 +231,9 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
232
231
|
protected registry: IFluidDataStoreRegistry | undefined;
|
|
233
232
|
|
|
234
233
|
protected detachedRuntimeCreation = false;
|
|
235
|
-
|
|
234
|
+
// back-compat (for tests) - can be removed in 2.0.0-alpha.2.0.0, or earlier if compat tests drop n/n-2 coverage
|
|
235
|
+
// @ts-expect-error - This shouldn't be referenced in the current version, but needs to be here for back-compat
|
|
236
|
+
private readonly bindToContext: () => void;
|
|
236
237
|
protected channel: IFluidDataStoreChannel | undefined;
|
|
237
238
|
private loaded = false;
|
|
238
239
|
protected pending: ISequencedDocumentMessage[] | undefined = [];
|
|
@@ -260,7 +261,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
260
261
|
constructor(
|
|
261
262
|
props: IFluidDataStoreContextProps,
|
|
262
263
|
private readonly existing: boolean,
|
|
263
|
-
private bindState: BindState,
|
|
264
264
|
public readonly isLocalDataStore: boolean,
|
|
265
265
|
private readonly makeLocallyVisibleFn: () => void,
|
|
266
266
|
) {
|
|
@@ -282,11 +282,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
282
282
|
this.containerRuntime.attachState : AttachState.Detached;
|
|
283
283
|
|
|
284
284
|
this.bindToContext = () => {
|
|
285
|
-
assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
|
|
286
|
-
this.bindState = BindState.Binding;
|
|
287
285
|
assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
|
|
288
286
|
this.makeLocallyVisible();
|
|
289
|
-
this.bindState = BindState.Bound;
|
|
290
287
|
};
|
|
291
288
|
|
|
292
289
|
const thisSummarizeInternal =
|
|
@@ -319,7 +316,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
319
316
|
}
|
|
320
317
|
|
|
321
318
|
private rejectDeferredRealize(reason: string, packageName?: string): never {
|
|
322
|
-
throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.
|
|
319
|
+
throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
|
|
323
320
|
}
|
|
324
321
|
|
|
325
322
|
public async realize(): Promise<IFluidDataStoreChannel> {
|
|
@@ -328,7 +325,12 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
328
325
|
this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
|
|
329
326
|
this.realizeCore(this.existing).catch((error) => {
|
|
330
327
|
const errorWrapped = DataProcessingError.wrapIfUnrecognized(error, "realizeFluidDataStoreContext");
|
|
331
|
-
errorWrapped.addTelemetryProperties({
|
|
328
|
+
errorWrapped.addTelemetryProperties({
|
|
329
|
+
fluidDataStoreId: {
|
|
330
|
+
value: this.id,
|
|
331
|
+
tag: TelemetryDataTag.CodeArtifact,
|
|
332
|
+
},
|
|
333
|
+
});
|
|
332
334
|
this.channelDeferred?.reject(errorWrapped);
|
|
333
335
|
this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
|
|
334
336
|
});
|
|
@@ -690,7 +692,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
|
|
|
690
692
|
} catch (error) {
|
|
691
693
|
this.channelDeferred?.reject(error);
|
|
692
694
|
this.logger.sendErrorEvent(
|
|
693
|
-
{ eventName: "BindRuntimeError", fluidDataStoreId: {
|
|
695
|
+
{ eventName: "BindRuntimeError", fluidDataStoreId: {
|
|
696
|
+
value: this.id,
|
|
697
|
+
tag: TelemetryDataTag.CodeArtifact,
|
|
698
|
+
} },
|
|
694
699
|
error);
|
|
695
700
|
}
|
|
696
701
|
}
|
|
@@ -783,7 +788,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
783
788
|
super(
|
|
784
789
|
props,
|
|
785
790
|
true /* existing */,
|
|
786
|
-
BindState.Bound,
|
|
787
791
|
false /* isLocalDataStore */,
|
|
788
792
|
() => {
|
|
789
793
|
throw new Error("Already attached");
|
|
@@ -809,11 +813,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
|
|
|
809
813
|
|
|
810
814
|
const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
|
|
811
815
|
if (tree) {
|
|
812
|
-
|
|
813
|
-
tree = loadedSummary.baseSummary;
|
|
814
|
-
// Prepend outstanding ops to pending queue of ops to process.
|
|
815
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
816
|
-
this.pending = loadedSummary.outstandingOps.concat(this.pending!);
|
|
816
|
+
tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
|
|
817
817
|
}
|
|
818
818
|
|
|
819
819
|
if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
|
|
@@ -892,7 +892,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
|
|
|
892
892
|
super(
|
|
893
893
|
props,
|
|
894
894
|
props.snapshotTree !== undefined ? true : false /* existing */,
|
|
895
|
-
props.snapshotTree ? BindState.Bound : BindState.NotBound,
|
|
896
895
|
true /* isLocalDataStore */,
|
|
897
896
|
props.makeLocallyVisibleFn,
|
|
898
897
|
);
|
|
@@ -1040,13 +1039,7 @@ export class LocalDetachedFluidDataStoreContext
|
|
|
1040
1039
|
super.bindRuntime(dataStoreChannel);
|
|
1041
1040
|
|
|
1042
1041
|
if (await this.isRoot()) {
|
|
1043
|
-
|
|
1044
|
-
// For older versions, we still have to call bindToContext.
|
|
1045
|
-
if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
|
|
1046
|
-
dataStoreChannel.makeVisibleAndAttachGraph();
|
|
1047
|
-
} else {
|
|
1048
|
-
dataStoreChannel.bindToContext();
|
|
1049
|
-
}
|
|
1042
|
+
dataStoreChannel.makeVisibleAndAttachGraph();
|
|
1050
1043
|
}
|
|
1051
1044
|
}
|
|
1052
1045
|
|
package/src/dataStores.ts
CHANGED
|
@@ -12,6 +12,7 @@ import {
|
|
|
12
12
|
ISnapshotTree,
|
|
13
13
|
} from "@fluidframework/protocol-definitions";
|
|
14
14
|
import {
|
|
15
|
+
AliasResult,
|
|
15
16
|
channelsTreeName,
|
|
16
17
|
CreateChildSummarizerNodeFn,
|
|
17
18
|
CreateChildSummarizerNodeParam,
|
|
@@ -82,6 +83,7 @@ export class DataStores implements IDisposable {
|
|
|
82
83
|
// The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
|
|
83
84
|
// the container runtime to other nodes.
|
|
84
85
|
private readonly containerRuntimeHandle: IFluidHandle;
|
|
86
|
+
private readonly pendingAliasMap: Map<string, Promise<AliasResult>> = new Map<string, Promise<AliasResult>>();
|
|
85
87
|
|
|
86
88
|
constructor(
|
|
87
89
|
private readonly baseSnapshot: ISnapshotTree | undefined,
|
|
@@ -173,10 +175,19 @@ export class DataStores implements IDisposable {
|
|
|
173
175
|
};
|
|
174
176
|
}
|
|
175
177
|
|
|
176
|
-
public aliases(): ReadonlyMap<string, string> {
|
|
178
|
+
public get aliases(): ReadonlyMap<string, string> {
|
|
177
179
|
return this.aliasMap;
|
|
178
180
|
}
|
|
179
181
|
|
|
182
|
+
public get pendingAliases(): Map<string, Promise<AliasResult>> {
|
|
183
|
+
return this.pendingAliasMap;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
public async waitIfPendingAlias(maybeAlias: string): Promise<AliasResult> {
|
|
187
|
+
const pendingAliasPromise = this.pendingAliases.get(maybeAlias);
|
|
188
|
+
return pendingAliasPromise === undefined ? "Success" : pendingAliasPromise;
|
|
189
|
+
}
|
|
190
|
+
|
|
180
191
|
public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
|
|
181
192
|
const attachMessage = message.contents as InboundAttachMessage;
|
|
182
193
|
|
|
@@ -201,7 +212,7 @@ export class DataStores implements IDisposable {
|
|
|
201
212
|
...extractSafePropertiesFromMessage(message),
|
|
202
213
|
dataStoreId: {
|
|
203
214
|
value: attachMessage.id,
|
|
204
|
-
tag: TelemetryDataTag.
|
|
215
|
+
tag: TelemetryDataTag.CodeArtifact,
|
|
205
216
|
},
|
|
206
217
|
},
|
|
207
218
|
);
|
|
@@ -440,7 +451,7 @@ export class DataStores implements IDisposable {
|
|
|
440
451
|
eventName: "SignalFluidDataStoreNotFound",
|
|
441
452
|
fluidDataStoreId: {
|
|
442
453
|
value: address,
|
|
443
|
-
tag: TelemetryDataTag.
|
|
454
|
+
tag: TelemetryDataTag.CodeArtifact,
|
|
444
455
|
},
|
|
445
456
|
});
|
|
446
457
|
return;
|
|
@@ -0,0 +1,19 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { ISequencedDocumentMessage, ISequencedDocumentSystemMessage } from "@fluidframework/protocol-definitions";
|
|
7
|
+
|
|
8
|
+
export const opSize = (op: ISequencedDocumentMessage): number => {
|
|
9
|
+
// Some messages may already have string contents,
|
|
10
|
+
// so stringifying them again will add inaccurate overhead.
|
|
11
|
+
const content = typeof op.contents === "string" ?
|
|
12
|
+
op.contents :
|
|
13
|
+
JSON.stringify(op.contents);
|
|
14
|
+
const data = opHasData(op) ? op.data : "";
|
|
15
|
+
return content.length + data.length;
|
|
16
|
+
};
|
|
17
|
+
|
|
18
|
+
const opHasData = (op: ISequencedDocumentMessage): op is ISequencedDocumentSystemMessage =>
|
|
19
|
+
(op as ISequencedDocumentSystemMessage).data !== undefined;
|
package/src/packageVersion.ts
CHANGED
package/src/runningSummarizer.ts
CHANGED
|
@@ -6,6 +6,7 @@
|
|
|
6
6
|
import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
7
|
import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
|
|
8
8
|
import { UsageError } from "@fluidframework/container-utils";
|
|
9
|
+
import { isRuntimeMessage } from "@fluidframework/driver-utils";
|
|
9
10
|
import {
|
|
10
11
|
ISequencedDocumentMessage,
|
|
11
12
|
MessageType,
|
|
@@ -14,6 +15,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils";
|
|
|
14
15
|
import {
|
|
15
16
|
ISummaryConfiguration,
|
|
16
17
|
} from "./containerRuntime";
|
|
18
|
+
import { opSize } from "./opProperties";
|
|
17
19
|
import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
|
|
18
20
|
import {
|
|
19
21
|
IEnqueueSummarizeOptions,
|
|
@@ -28,6 +30,7 @@ import {
|
|
|
28
30
|
ISummaryCancellationToken,
|
|
29
31
|
ISummarizeResults,
|
|
30
32
|
ISummarizeTelemetryProperties,
|
|
33
|
+
ISummarizerRuntime,
|
|
31
34
|
ISummarizeRunnerTelemetry,
|
|
32
35
|
} from "./summarizerTypes";
|
|
33
36
|
import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
|
|
@@ -58,6 +61,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
58
61
|
summaryCollection: SummaryCollection,
|
|
59
62
|
cancellationToken: ISummaryCancellationToken,
|
|
60
63
|
stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
64
|
+
runtime: ISummarizerRuntime,
|
|
61
65
|
): Promise<RunningSummarizer> {
|
|
62
66
|
const summarizer = new RunningSummarizer(
|
|
63
67
|
logger,
|
|
@@ -68,12 +72,36 @@ export class RunningSummarizer implements IDisposable {
|
|
|
68
72
|
raiseSummarizingError,
|
|
69
73
|
summaryCollection,
|
|
70
74
|
cancellationToken,
|
|
71
|
-
stopSummarizerCallback
|
|
75
|
+
stopSummarizerCallback,
|
|
76
|
+
runtime);
|
|
72
77
|
|
|
73
78
|
await summarizer.waitStart();
|
|
74
79
|
|
|
75
|
-
//
|
|
80
|
+
// Update heuristic counts
|
|
81
|
+
// By the time we get here, there are potentially ops missing from the heuristic summary counts
|
|
82
|
+
// Examples of where this could happen:
|
|
83
|
+
// 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we
|
|
84
|
+
// listen for the op events (will get missed by the handlers in the current workflow)
|
|
85
|
+
// 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)
|
|
86
|
+
const diff = runtime.deltaManager.lastSequenceNumber - (
|
|
87
|
+
heuristicData.lastSuccessfulSummary.refSequenceNumber
|
|
88
|
+
+ heuristicData.numNonRuntimeOps
|
|
89
|
+
+ heuristicData.numRuntimeOps);
|
|
90
|
+
heuristicData.hasMissingOpData = diff > 0;
|
|
91
|
+
|
|
92
|
+
if (heuristicData.hasMissingOpData) {
|
|
93
|
+
// Split the diff 50-50 and increment the counts appropriately
|
|
94
|
+
heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);
|
|
95
|
+
heuristicData.numRuntimeOps += Math.floor(diff / 2);
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
// Update last seq number (in case the handlers haven't processed anything yet)
|
|
99
|
+
heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
|
|
100
|
+
|
|
101
|
+
// Start heuristics
|
|
102
|
+
summarizer.heuristicRunner?.start();
|
|
76
103
|
summarizer.heuristicRunner?.run();
|
|
104
|
+
|
|
77
105
|
return summarizer;
|
|
78
106
|
}
|
|
79
107
|
|
|
@@ -95,6 +123,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
95
123
|
} | undefined;
|
|
96
124
|
private summarizeCount = 0;
|
|
97
125
|
private totalSuccessfulAttempts = 0;
|
|
126
|
+
private initialized = false;
|
|
98
127
|
|
|
99
128
|
private constructor(
|
|
100
129
|
baseLogger: ITelemetryLogger,
|
|
@@ -106,6 +135,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
106
135
|
private readonly summaryCollection: SummaryCollection,
|
|
107
136
|
private readonly cancellationToken: ISummaryCancellationToken,
|
|
108
137
|
private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
138
|
+
private readonly runtime: ISummarizerRuntime,
|
|
109
139
|
) {
|
|
110
140
|
const telemetryProps: ISummarizeRunnerTelemetry = {
|
|
111
141
|
summarizeCount: () => this.summarizeCount,
|
|
@@ -175,9 +205,13 @@ export class RunningSummarizer implements IDisposable {
|
|
|
175
205
|
this.summaryWatcher,
|
|
176
206
|
this.logger,
|
|
177
207
|
);
|
|
208
|
+
|
|
209
|
+
// Listen for ops
|
|
210
|
+
this.runtime.deltaManager.on("op", (op) => { this.handleOp(op); });
|
|
178
211
|
}
|
|
179
212
|
|
|
180
213
|
public dispose(): void {
|
|
214
|
+
this.runtime.deltaManager.off("op", (op) => { this.handleOp(op); });
|
|
181
215
|
this.summaryWatcher.dispose();
|
|
182
216
|
this.heuristicRunner?.dispose();
|
|
183
217
|
this.heuristicRunner = undefined;
|
|
@@ -199,30 +233,48 @@ export class RunningSummarizer implements IDisposable {
|
|
|
199
233
|
? this.logger
|
|
200
234
|
: undefined;
|
|
201
235
|
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
case MessageType.ClientLeave:
|
|
205
|
-
case MessageType.ClientJoin:
|
|
206
|
-
case MessageType.Propose: {
|
|
207
|
-
// Synchronously handle quorum ops like regular ops
|
|
208
|
-
this.handleOp(undefined, op);
|
|
209
|
-
return;
|
|
210
|
-
}
|
|
211
|
-
default: {
|
|
212
|
-
return;
|
|
213
|
-
}
|
|
214
|
-
}
|
|
215
|
-
}
|
|
236
|
+
/** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
|
|
237
|
+
private heuristicRunnerMicroTaskExists = false;
|
|
216
238
|
|
|
217
|
-
public handleOp(
|
|
218
|
-
|
|
219
|
-
|
|
239
|
+
public handleOp(op: ISequencedDocumentMessage) {
|
|
240
|
+
this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
|
|
241
|
+
|
|
242
|
+
if (op.type !== MessageType.Summarize && isRuntimeMessage(op)) {
|
|
243
|
+
this.heuristicData.numRuntimeOps++;
|
|
244
|
+
} else {
|
|
245
|
+
this.heuristicData.numNonRuntimeOps++;
|
|
220
246
|
}
|
|
221
|
-
|
|
247
|
+
|
|
248
|
+
this.heuristicData.totalOpsSize += opSize(op);
|
|
222
249
|
|
|
223
250
|
// Check for enqueued on-demand summaries; Intentionally do nothing otherwise
|
|
224
|
-
if (
|
|
225
|
-
this.
|
|
251
|
+
if (this.initialized
|
|
252
|
+
&& this.opCanTriggerSummary(op)
|
|
253
|
+
&& !this.tryRunEnqueuedSummary()
|
|
254
|
+
&& !this.heuristicRunnerMicroTaskExists) {
|
|
255
|
+
this.heuristicRunnerMicroTaskExists = true;
|
|
256
|
+
Promise.resolve().then(() => {
|
|
257
|
+
this.heuristicRunner?.run();
|
|
258
|
+
}).finally(() => {
|
|
259
|
+
this.heuristicRunnerMicroTaskExists = false;
|
|
260
|
+
});
|
|
261
|
+
}
|
|
262
|
+
}
|
|
263
|
+
|
|
264
|
+
/**
|
|
265
|
+
* Can the given op trigger a summary?
|
|
266
|
+
* # Currently only prevents summaries for Summarize and SummaryAck ops
|
|
267
|
+
* @param op - op to check
|
|
268
|
+
* @returns true if this type of op can trigger a summary
|
|
269
|
+
*/
|
|
270
|
+
private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
|
|
271
|
+
switch (op.type) {
|
|
272
|
+
case MessageType.Summarize:
|
|
273
|
+
case MessageType.SummaryAck:
|
|
274
|
+
case MessageType.SummaryNack:
|
|
275
|
+
return false;
|
|
276
|
+
default:
|
|
277
|
+
return true;
|
|
226
278
|
}
|
|
227
279
|
}
|
|
228
280
|
|
|
@@ -274,6 +326,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
274
326
|
summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
|
|
275
327
|
});
|
|
276
328
|
}
|
|
329
|
+
this.initialized = true;
|
|
277
330
|
}
|
|
278
331
|
|
|
279
332
|
/**
|
package/src/summarizer.ts
CHANGED
|
@@ -17,9 +17,6 @@ import {
|
|
|
17
17
|
IFluidHandle,
|
|
18
18
|
IRequest,
|
|
19
19
|
} from "@fluidframework/core-interfaces";
|
|
20
|
-
import {
|
|
21
|
-
ISequencedDocumentMessage,
|
|
22
|
-
} from "@fluidframework/protocol-definitions";
|
|
23
20
|
import { ISummaryConfiguration } from "./containerRuntime";
|
|
24
21
|
import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
|
|
25
22
|
import { summarizerClientType } from "./summarizerClientElection";
|
|
@@ -70,8 +67,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
70
67
|
|
|
71
68
|
private readonly logger: ITelemetryLogger;
|
|
72
69
|
private runningSummarizer?: RunningSummarizer;
|
|
73
|
-
private systemOpListener?: (op: ISequencedDocumentMessage) => void;
|
|
74
|
-
private opListener?: (error: any, op: ISequencedDocumentMessage) => void;
|
|
75
70
|
private _disposed: boolean = false;
|
|
76
71
|
private starting: boolean = false;
|
|
77
72
|
|
|
@@ -277,6 +272,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
277
272
|
this.summaryCollection,
|
|
278
273
|
runCoordinator /* cancellationToken */,
|
|
279
274
|
(reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */
|
|
275
|
+
this.runtime,
|
|
280
276
|
);
|
|
281
277
|
this.runningSummarizer = runningSummarizer;
|
|
282
278
|
this.starting = false;
|
|
@@ -287,13 +283,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
287
283
|
this.logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
|
|
288
284
|
});
|
|
289
285
|
|
|
290
|
-
// Listen for ops
|
|
291
|
-
this.systemOpListener = (op: ISequencedDocumentMessage) => runningSummarizer.handleSystemOp(op);
|
|
292
|
-
this.runtime.deltaManager.inbound.on("op", this.systemOpListener);
|
|
293
|
-
|
|
294
|
-
this.opListener = (error: any, op: ISequencedDocumentMessage) => runningSummarizer.handleOp(error, op);
|
|
295
|
-
this.runtime.on("batchEnd", this.opListener);
|
|
296
|
-
|
|
297
286
|
return runningSummarizer;
|
|
298
287
|
}
|
|
299
288
|
|
|
@@ -312,12 +301,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
312
301
|
this.runningSummarizer.dispose();
|
|
313
302
|
this.runningSummarizer = undefined;
|
|
314
303
|
}
|
|
315
|
-
if (this.systemOpListener) {
|
|
316
|
-
this.runtime.deltaManager.inbound.off("op", this.systemOpListener);
|
|
317
|
-
}
|
|
318
|
-
if (this.opListener) {
|
|
319
|
-
this.runtime.removeListener("batchEnd", this.opListener);
|
|
320
|
-
}
|
|
321
304
|
}
|
|
322
305
|
|
|
323
306
|
public readonly summarizeOnDemand: ISummarizer["summarizeOnDemand"] = (...args) => {
|
|
@@ -6,11 +6,11 @@
|
|
|
6
6
|
import { ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
7
|
import { Timer } from "@fluidframework/common-utils";
|
|
8
8
|
import { ISummaryConfigurationHeuristics } from "./containerRuntime";
|
|
9
|
-
|
|
10
9
|
import {
|
|
11
10
|
ISummarizeHeuristicData,
|
|
12
11
|
ISummarizeHeuristicRunner,
|
|
13
12
|
ISummarizeAttempt,
|
|
13
|
+
ISummaryHeuristicStrategy,
|
|
14
14
|
} from "./summarizerTypes";
|
|
15
15
|
import { SummarizeReason } from "./summaryGenerator";
|
|
16
16
|
|
|
@@ -26,6 +26,30 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
|
|
|
26
26
|
return this._lastSuccessfulSummary;
|
|
27
27
|
}
|
|
28
28
|
|
|
29
|
+
public numNonRuntimeOps: number = 0;
|
|
30
|
+
public totalOpsSize: number = 0;
|
|
31
|
+
public hasMissingOpData: boolean = false;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* Cumulative size in bytes of all the ops at the beginning of the summarization attempt.
|
|
35
|
+
* Is used to adjust totalOpsSize appropriately after successful summarization.
|
|
36
|
+
*/
|
|
37
|
+
/** */
|
|
38
|
+
private totalOpsSizeBefore: number = 0;
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Number of system ops at beginning of attempting to summarize.
|
|
42
|
+
* Is used to adjust numSystemOps appropriately after successful summarization.
|
|
43
|
+
*/
|
|
44
|
+
private numSystemOpsBefore: number = 0;
|
|
45
|
+
|
|
46
|
+
public numRuntimeOps: number = 0;
|
|
47
|
+
/**
|
|
48
|
+
* Number of non-system ops at beginning of attempting to summarize.
|
|
49
|
+
* Is used to adjust numNonSystemOps appropriately after successful summarization.
|
|
50
|
+
*/
|
|
51
|
+
private numNonSystemOpsBefore: number = 0;
|
|
52
|
+
|
|
29
53
|
constructor(
|
|
30
54
|
public lastOpSequenceNumber: number,
|
|
31
55
|
/** Baseline attempt data used for comparisons with subsequent attempts/calculations. */
|
|
@@ -45,10 +69,23 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
|
|
|
45
69
|
refSequenceNumber: refSequenceNumber ?? this.lastOpSequenceNumber,
|
|
46
70
|
summaryTime: Date.now(),
|
|
47
71
|
};
|
|
72
|
+
|
|
73
|
+
this.numSystemOpsBefore = this.numNonRuntimeOps;
|
|
74
|
+
this.numNonSystemOpsBefore = this.numRuntimeOps;
|
|
75
|
+
this.totalOpsSizeBefore = this.totalOpsSize;
|
|
48
76
|
}
|
|
49
77
|
|
|
50
78
|
public markLastAttemptAsSuccessful() {
|
|
51
79
|
this._lastSuccessfulSummary = { ...this.lastAttempt };
|
|
80
|
+
|
|
81
|
+
this.numNonRuntimeOps -= this.numSystemOpsBefore;
|
|
82
|
+
this.numSystemOpsBefore = 0;
|
|
83
|
+
|
|
84
|
+
this.numRuntimeOps -= this.numNonSystemOpsBefore;
|
|
85
|
+
this.numNonSystemOpsBefore = 0;
|
|
86
|
+
|
|
87
|
+
this.totalOpsSize -= this.totalOpsSizeBefore;
|
|
88
|
+
this.totalOpsSizeBefore = 0;
|
|
52
89
|
}
|
|
53
90
|
}
|
|
54
91
|
|
|
@@ -56,42 +93,71 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
|
|
|
56
93
|
* This class contains the heuristics for when to summarize.
|
|
57
94
|
*/
|
|
58
95
|
export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
|
|
59
|
-
private readonly idleTimer: Timer;
|
|
60
|
-
private readonly
|
|
96
|
+
private readonly idleTimer: Timer | undefined;
|
|
97
|
+
private readonly runSummarize: (reason: SummarizeReason) => void;
|
|
61
98
|
|
|
62
99
|
public constructor(
|
|
63
100
|
private readonly heuristicData: ISummarizeHeuristicData,
|
|
64
101
|
private readonly configuration: ISummaryConfigurationHeuristics,
|
|
65
|
-
|
|
102
|
+
trySummarize: (reason: SummarizeReason) => void,
|
|
66
103
|
private readonly logger: ITelemetryLogger,
|
|
104
|
+
private readonly summarizeStrategies: ISummaryHeuristicStrategy[] = getDefaultSummaryHeuristicStrategies(),
|
|
67
105
|
) {
|
|
68
106
|
this.idleTimer = new Timer(
|
|
69
|
-
this.
|
|
70
|
-
() => this.
|
|
71
|
-
|
|
107
|
+
this.idleTime,
|
|
108
|
+
() => this.runSummarize("idle"));
|
|
109
|
+
|
|
110
|
+
this.runSummarize = (reason: SummarizeReason) => {
|
|
111
|
+
this.idleTimer?.clear();
|
|
112
|
+
|
|
113
|
+
// We shouldn't attempt a summary if there are no new processed ops
|
|
114
|
+
const opsSinceLastAck = this.opsSinceLastAck;
|
|
115
|
+
if (opsSinceLastAck > 0) {
|
|
116
|
+
trySummarize(reason);
|
|
117
|
+
}
|
|
118
|
+
};
|
|
119
|
+
}
|
|
120
|
+
|
|
121
|
+
public get idleTime(): number {
|
|
122
|
+
const maxIdleTime = this.configuration.maxIdleTime;
|
|
123
|
+
const minIdleTime = this.configuration.minIdleTime;
|
|
124
|
+
const weightedNumOfOps = getWeightedNumberOfOps(
|
|
125
|
+
this.heuristicData.numRuntimeOps,
|
|
126
|
+
this.heuristicData.numNonRuntimeOps,
|
|
127
|
+
this.configuration.runtimeOpWeight,
|
|
128
|
+
this.configuration.nonRuntimeOpWeight,
|
|
129
|
+
);
|
|
130
|
+
const pToMaxOps = weightedNumOfOps * 1.0 / this.configuration.maxOps;
|
|
131
|
+
|
|
132
|
+
if (pToMaxOps >= 1) {
|
|
133
|
+
return minIdleTime;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
// Return a ratioed idle time based on the percentage of ops
|
|
137
|
+
return maxIdleTime - ((maxIdleTime - minIdleTime) * pToMaxOps);
|
|
72
138
|
}
|
|
73
139
|
|
|
74
140
|
public get opsSinceLastAck(): number {
|
|
75
141
|
return this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
|
|
76
142
|
}
|
|
77
143
|
|
|
144
|
+
public start() {
|
|
145
|
+
this.idleTimer?.start(this.idleTime);
|
|
146
|
+
}
|
|
147
|
+
|
|
78
148
|
public run() {
|
|
79
|
-
const
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
this.trySummarize("maxTime");
|
|
84
|
-
} else if (opsSinceLastAck > this.configuration.maxOps) {
|
|
85
|
-
this.idleTimer.clear();
|
|
86
|
-
this.trySummarize("maxOps");
|
|
87
|
-
} else {
|
|
88
|
-
this.idleTimer.restart();
|
|
149
|
+
for (const strategy of this.summarizeStrategies) {
|
|
150
|
+
if (strategy.shouldRunSummary(this.configuration, this.heuristicData)) {
|
|
151
|
+
return this.runSummarize(strategy.summarizeReason);
|
|
152
|
+
}
|
|
89
153
|
}
|
|
154
|
+
|
|
155
|
+
this.idleTimer?.restart(this.idleTime);
|
|
90
156
|
}
|
|
91
157
|
|
|
92
158
|
public shouldRunLastSummary(): boolean {
|
|
93
159
|
const opsSinceLastAck = this.opsSinceLastAck;
|
|
94
|
-
const minOpsForLastSummaryAttempt = this.minOpsForLastSummaryAttempt;
|
|
160
|
+
const minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
|
|
95
161
|
|
|
96
162
|
this.logger.sendTelemetryEvent({
|
|
97
163
|
eventName: "ShouldRunLastSummary",
|
|
@@ -103,6 +169,54 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
|
|
|
103
169
|
}
|
|
104
170
|
|
|
105
171
|
public dispose() {
|
|
106
|
-
this.idleTimer
|
|
172
|
+
this.idleTimer?.clear();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/** Strategy used to run a summary when it's been a while since our last successful summary */
|
|
177
|
+
class MaxTimeSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
|
|
178
|
+
public readonly summarizeReason: Readonly<SummarizeReason> = "maxTime";
|
|
179
|
+
|
|
180
|
+
public shouldRunSummary(
|
|
181
|
+
configuration: ISummaryConfigurationHeuristics,
|
|
182
|
+
heuristicData: ISummarizeHeuristicData,
|
|
183
|
+
): boolean {
|
|
184
|
+
const timeSinceLastSummary = Date.now() - heuristicData.lastSuccessfulSummary.summaryTime;
|
|
185
|
+
return timeSinceLastSummary > configuration.maxTime;
|
|
107
186
|
}
|
|
108
187
|
}
|
|
188
|
+
|
|
189
|
+
function getWeightedNumberOfOps(
|
|
190
|
+
runtimeOpCount: number,
|
|
191
|
+
nonRuntimeOpCount: number,
|
|
192
|
+
runtimeOpWeight: number,
|
|
193
|
+
nonRuntimeOpWeight: number,
|
|
194
|
+
): number {
|
|
195
|
+
return (runtimeOpWeight * runtimeOpCount)
|
|
196
|
+
+ (nonRuntimeOpWeight * nonRuntimeOpCount);
|
|
197
|
+
}
|
|
198
|
+
|
|
199
|
+
/** Strategy used to do a weighted analysis on the ops we've processed since the last successful summary */
|
|
200
|
+
class WeightedOpsSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
|
|
201
|
+
public readonly summarizeReason: Readonly<SummarizeReason> = "maxOps";
|
|
202
|
+
|
|
203
|
+
public shouldRunSummary(
|
|
204
|
+
configuration: ISummaryConfigurationHeuristics,
|
|
205
|
+
heuristicData: ISummarizeHeuristicData,
|
|
206
|
+
): boolean {
|
|
207
|
+
const weightedNumOfOps = getWeightedNumberOfOps(
|
|
208
|
+
heuristicData.numRuntimeOps,
|
|
209
|
+
heuristicData.numNonRuntimeOps,
|
|
210
|
+
configuration.runtimeOpWeight,
|
|
211
|
+
configuration.nonRuntimeOpWeight,
|
|
212
|
+
);
|
|
213
|
+
return weightedNumOfOps > configuration.maxOps;
|
|
214
|
+
}
|
|
215
|
+
}
|
|
216
|
+
|
|
217
|
+
function getDefaultSummaryHeuristicStrategies() {
|
|
218
|
+
return [
|
|
219
|
+
new MaxTimeSummaryHeuristicStrategy(),
|
|
220
|
+
new WeightedOpsSummaryHeuristicStrategy(),
|
|
221
|
+
];
|
|
222
|
+
}
|