@fluidframework/container-runtime 1.2.2 → 2.0.0-internal.1.0.0.81589

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/blobManager.d.ts +81 -25
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +301 -100
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +46 -3
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +98 -79
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStore.d.ts +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +32 -26
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +3 -4
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +16 -23
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStores.d.ts +5 -2
  18. package/dist/dataStores.d.ts.map +1 -1
  19. package/dist/dataStores.js +11 -3
  20. package/dist/dataStores.js.map +1 -1
  21. package/dist/opProperties.d.ts +7 -0
  22. package/dist/opProperties.d.ts.map +1 -0
  23. package/dist/opProperties.js +19 -0
  24. package/dist/opProperties.js.map +1 -0
  25. package/dist/packageVersion.d.ts +1 -1
  26. package/dist/packageVersion.d.ts.map +1 -1
  27. package/dist/packageVersion.js +1 -1
  28. package/dist/packageVersion.js.map +1 -1
  29. package/dist/runningSummarizer.d.ts +14 -4
  30. package/dist/runningSummarizer.d.ts.map +1 -1
  31. package/dist/runningSummarizer.js +68 -26
  32. package/dist/runningSummarizer.js.map +1 -1
  33. package/dist/summarizer.d.ts +0 -2
  34. package/dist/summarizer.d.ts.map +1 -1
  35. package/dist/summarizer.js +1 -12
  36. package/dist/summarizer.js.map +1 -1
  37. package/dist/summarizerHeuristics.d.ts +26 -4
  38. package/dist/summarizerHeuristics.d.ts.map +1 -1
  39. package/dist/summarizerHeuristics.js +95 -18
  40. package/dist/summarizerHeuristics.js.map +1 -1
  41. package/dist/summarizerTypes.d.ts +30 -10
  42. package/dist/summarizerTypes.d.ts.map +1 -1
  43. package/dist/summarizerTypes.js.map +1 -1
  44. package/dist/summaryCollection.js +1 -1
  45. package/dist/summaryCollection.js.map +1 -1
  46. package/dist/summaryFormat.d.ts +0 -5
  47. package/dist/summaryFormat.d.ts.map +1 -1
  48. package/dist/summaryFormat.js.map +1 -1
  49. package/dist/summaryGenerator.d.ts +1 -0
  50. package/dist/summaryGenerator.d.ts.map +1 -1
  51. package/dist/summaryGenerator.js +11 -9
  52. package/dist/summaryGenerator.js.map +1 -1
  53. package/lib/blobManager.d.ts +81 -25
  54. package/lib/blobManager.d.ts.map +1 -1
  55. package/lib/blobManager.js +302 -101
  56. package/lib/blobManager.js.map +1 -1
  57. package/lib/containerRuntime.d.ts +46 -3
  58. package/lib/containerRuntime.d.ts.map +1 -1
  59. package/lib/containerRuntime.js +99 -80
  60. package/lib/containerRuntime.js.map +1 -1
  61. package/lib/dataStore.d.ts +1 -1
  62. package/lib/dataStore.d.ts.map +1 -1
  63. package/lib/dataStore.js +32 -26
  64. package/lib/dataStore.js.map +1 -1
  65. package/lib/dataStoreContext.d.ts +3 -4
  66. package/lib/dataStoreContext.d.ts.map +1 -1
  67. package/lib/dataStoreContext.js +17 -24
  68. package/lib/dataStoreContext.js.map +1 -1
  69. package/lib/dataStores.d.ts +5 -2
  70. package/lib/dataStores.d.ts.map +1 -1
  71. package/lib/dataStores.js +11 -3
  72. package/lib/dataStores.js.map +1 -1
  73. package/lib/opProperties.d.ts +7 -0
  74. package/lib/opProperties.d.ts.map +1 -0
  75. package/lib/opProperties.js +15 -0
  76. package/lib/opProperties.js.map +1 -0
  77. package/lib/packageVersion.d.ts +1 -1
  78. package/lib/packageVersion.d.ts.map +1 -1
  79. package/lib/packageVersion.js +1 -1
  80. package/lib/packageVersion.js.map +1 -1
  81. package/lib/runningSummarizer.d.ts +14 -4
  82. package/lib/runningSummarizer.d.ts.map +1 -1
  83. package/lib/runningSummarizer.js +68 -26
  84. package/lib/runningSummarizer.js.map +1 -1
  85. package/lib/summarizer.d.ts +0 -2
  86. package/lib/summarizer.d.ts.map +1 -1
  87. package/lib/summarizer.js +1 -12
  88. package/lib/summarizer.js.map +1 -1
  89. package/lib/summarizerHeuristics.d.ts +26 -4
  90. package/lib/summarizerHeuristics.d.ts.map +1 -1
  91. package/lib/summarizerHeuristics.js +95 -18
  92. package/lib/summarizerHeuristics.js.map +1 -1
  93. package/lib/summarizerTypes.d.ts +30 -10
  94. package/lib/summarizerTypes.d.ts.map +1 -1
  95. package/lib/summarizerTypes.js.map +1 -1
  96. package/lib/summaryCollection.js +1 -1
  97. package/lib/summaryCollection.js.map +1 -1
  98. package/lib/summaryFormat.d.ts +0 -5
  99. package/lib/summaryFormat.d.ts.map +1 -1
  100. package/lib/summaryFormat.js.map +1 -1
  101. package/lib/summaryGenerator.d.ts +1 -0
  102. package/lib/summaryGenerator.d.ts.map +1 -1
  103. package/lib/summaryGenerator.js +11 -9
  104. package/lib/summaryGenerator.js.map +1 -1
  105. package/package.json +45 -20
  106. package/src/blobManager.ts +360 -119
  107. package/src/containerRuntime.ts +165 -89
  108. package/src/dataStore.ts +53 -38
  109. package/src/dataStoreContext.ts +16 -23
  110. package/src/dataStores.ts +14 -3
  111. package/src/opProperties.ts +19 -0
  112. package/src/packageVersion.ts +1 -1
  113. package/src/runningSummarizer.ts +75 -22
  114. package/src/summarizer.ts +1 -18
  115. package/src/summarizerHeuristics.ts +133 -19
  116. package/src/summarizerTypes.ts +37 -10
  117. package/src/summaryCollection.ts +1 -1
  118. package/src/summaryFormat.ts +0 -6
  119. package/src/summaryGenerator.ts +40 -22
  120. package/dist/opTelemetry.d.ts +0 -22
  121. package/dist/opTelemetry.d.ts.map +0 -1
  122. package/dist/opTelemetry.js +0 -59
  123. package/dist/opTelemetry.js.map +0 -1
  124. package/lib/opTelemetry.d.ts +0 -22
  125. package/lib/opTelemetry.d.ts.map +0 -1
  126. package/lib/opTelemetry.js +0 -55
  127. package/lib/opTelemetry.js.map +0 -1
  128. package/src/opTelemetry.ts +0 -71
@@ -13,7 +13,6 @@ import {
13
13
  import {
14
14
  IAudience,
15
15
  IDeltaManager,
16
- BindState,
17
16
  AttachState,
18
17
  ILoaderOptions,
19
18
  } from "@fluidframework/container-definitions";
@@ -232,7 +231,9 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
232
231
  protected registry: IFluidDataStoreRegistry | undefined;
233
232
 
234
233
  protected detachedRuntimeCreation = false;
235
- public readonly bindToContext: () => void;
234
+ // back-compat (for tests) - can be removed in 2.0.0-alpha.2.0.0, or earlier if compat tests drop n/n-2 coverage
235
+ // @ts-expect-error - This shouldn't be referenced in the current version, but needs to be here for back-compat
236
+ private readonly bindToContext: () => void;
236
237
  protected channel: IFluidDataStoreChannel | undefined;
237
238
  private loaded = false;
238
239
  protected pending: ISequencedDocumentMessage[] | undefined = [];
@@ -260,7 +261,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
260
261
  constructor(
261
262
  props: IFluidDataStoreContextProps,
262
263
  private readonly existing: boolean,
263
- private bindState: BindState,
264
264
  public readonly isLocalDataStore: boolean,
265
265
  private readonly makeLocallyVisibleFn: () => void,
266
266
  ) {
@@ -282,11 +282,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
282
282
  this.containerRuntime.attachState : AttachState.Detached;
283
283
 
284
284
  this.bindToContext = () => {
285
- assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
286
- this.bindState = BindState.Binding;
287
285
  assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
288
286
  this.makeLocallyVisible();
289
- this.bindState = BindState.Bound;
290
287
  };
291
288
 
292
289
  const thisSummarizeInternal =
@@ -319,7 +316,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
319
316
  }
320
317
 
321
318
  private rejectDeferredRealize(reason: string, packageName?: string): never {
322
- throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.PackageData } });
319
+ throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
323
320
  }
324
321
 
325
322
  public async realize(): Promise<IFluidDataStoreChannel> {
@@ -328,7 +325,12 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
328
325
  this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
329
326
  this.realizeCore(this.existing).catch((error) => {
330
327
  const errorWrapped = DataProcessingError.wrapIfUnrecognized(error, "realizeFluidDataStoreContext");
331
- errorWrapped.addTelemetryProperties({ fluidDataStoreId: { value: this.id, tag: "PackageData" } });
328
+ errorWrapped.addTelemetryProperties({
329
+ fluidDataStoreId: {
330
+ value: this.id,
331
+ tag: TelemetryDataTag.CodeArtifact,
332
+ },
333
+ });
332
334
  this.channelDeferred?.reject(errorWrapped);
333
335
  this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
334
336
  });
@@ -690,7 +692,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
690
692
  } catch (error) {
691
693
  this.channelDeferred?.reject(error);
692
694
  this.logger.sendErrorEvent(
693
- { eventName: "BindRuntimeError", fluidDataStoreId: { value: this.id, tag: "PackageData" } },
695
+ { eventName: "BindRuntimeError", fluidDataStoreId: {
696
+ value: this.id,
697
+ tag: TelemetryDataTag.CodeArtifact,
698
+ } },
694
699
  error);
695
700
  }
696
701
  }
@@ -783,7 +788,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
783
788
  super(
784
789
  props,
785
790
  true /* existing */,
786
- BindState.Bound,
787
791
  false /* isLocalDataStore */,
788
792
  () => {
789
793
  throw new Error("Already attached");
@@ -809,11 +813,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
809
813
 
810
814
  const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
811
815
  if (tree) {
812
- const loadedSummary = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
813
- tree = loadedSummary.baseSummary;
814
- // Prepend outstanding ops to pending queue of ops to process.
815
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
816
- this.pending = loadedSummary.outstandingOps.concat(this.pending!);
816
+ tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
817
817
  }
818
818
 
819
819
  if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
@@ -892,7 +892,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
892
892
  super(
893
893
  props,
894
894
  props.snapshotTree !== undefined ? true : false /* existing */,
895
- props.snapshotTree ? BindState.Bound : BindState.NotBound,
896
895
  true /* isLocalDataStore */,
897
896
  props.makeLocallyVisibleFn,
898
897
  );
@@ -1040,13 +1039,7 @@ export class LocalDetachedFluidDataStoreContext
1040
1039
  super.bindRuntime(dataStoreChannel);
1041
1040
 
1042
1041
  if (await this.isRoot()) {
1043
- // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
1044
- // For older versions, we still have to call bindToContext.
1045
- if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
1046
- dataStoreChannel.makeVisibleAndAttachGraph();
1047
- } else {
1048
- dataStoreChannel.bindToContext();
1049
- }
1042
+ dataStoreChannel.makeVisibleAndAttachGraph();
1050
1043
  }
1051
1044
  }
1052
1045
 
package/src/dataStores.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  ISnapshotTree,
13
13
  } from "@fluidframework/protocol-definitions";
14
14
  import {
15
+ AliasResult,
15
16
  channelsTreeName,
16
17
  CreateChildSummarizerNodeFn,
17
18
  CreateChildSummarizerNodeParam,
@@ -82,6 +83,7 @@ export class DataStores implements IDisposable {
82
83
  // The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
83
84
  // the container runtime to other nodes.
84
85
  private readonly containerRuntimeHandle: IFluidHandle;
86
+ private readonly pendingAliasMap: Map<string, Promise<AliasResult>> = new Map<string, Promise<AliasResult>>();
85
87
 
86
88
  constructor(
87
89
  private readonly baseSnapshot: ISnapshotTree | undefined,
@@ -173,10 +175,19 @@ export class DataStores implements IDisposable {
173
175
  };
174
176
  }
175
177
 
176
- public aliases(): ReadonlyMap<string, string> {
178
+ public get aliases(): ReadonlyMap<string, string> {
177
179
  return this.aliasMap;
178
180
  }
179
181
 
182
+ public get pendingAliases(): Map<string, Promise<AliasResult>> {
183
+ return this.pendingAliasMap;
184
+ }
185
+
186
+ public async waitIfPendingAlias(maybeAlias: string): Promise<AliasResult> {
187
+ const pendingAliasPromise = this.pendingAliases.get(maybeAlias);
188
+ return pendingAliasPromise === undefined ? "Success" : pendingAliasPromise;
189
+ }
190
+
180
191
  public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
181
192
  const attachMessage = message.contents as InboundAttachMessage;
182
193
 
@@ -201,7 +212,7 @@ export class DataStores implements IDisposable {
201
212
  ...extractSafePropertiesFromMessage(message),
202
213
  dataStoreId: {
203
214
  value: attachMessage.id,
204
- tag: TelemetryDataTag.PackageData,
215
+ tag: TelemetryDataTag.CodeArtifact,
205
216
  },
206
217
  },
207
218
  );
@@ -440,7 +451,7 @@ export class DataStores implements IDisposable {
440
451
  eventName: "SignalFluidDataStoreNotFound",
441
452
  fluidDataStoreId: {
442
453
  value: address,
443
- tag: TelemetryDataTag.PackageData,
454
+ tag: TelemetryDataTag.CodeArtifact,
444
455
  },
445
456
  });
446
457
  return;
@@ -0,0 +1,19 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { ISequencedDocumentMessage, ISequencedDocumentSystemMessage } from "@fluidframework/protocol-definitions";
7
+
8
+ export const opSize = (op: ISequencedDocumentMessage): number => {
9
+ // Some messages may already have string contents,
10
+ // so stringifying them again will add inaccurate overhead.
11
+ const content = typeof op.contents === "string" ?
12
+ op.contents :
13
+ JSON.stringify(op.contents);
14
+ const data = opHasData(op) ? op.data : "";
15
+ return content.length + data.length;
16
+ };
17
+
18
+ const opHasData = (op: ISequencedDocumentMessage): op is ISequencedDocumentSystemMessage =>
19
+ (op as ISequencedDocumentSystemMessage).data !== undefined;
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "1.2.2";
9
+ export const pkgVersion = "2.0.0-internal.1.0.0.81589";
@@ -6,6 +6,7 @@
6
6
  import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
8
8
  import { UsageError } from "@fluidframework/container-utils";
9
+ import { isRuntimeMessage } from "@fluidframework/driver-utils";
9
10
  import {
10
11
  ISequencedDocumentMessage,
11
12
  MessageType,
@@ -14,6 +15,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils";
14
15
  import {
15
16
  ISummaryConfiguration,
16
17
  } from "./containerRuntime";
18
+ import { opSize } from "./opProperties";
17
19
  import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
18
20
  import {
19
21
  IEnqueueSummarizeOptions,
@@ -28,6 +30,7 @@ import {
28
30
  ISummaryCancellationToken,
29
31
  ISummarizeResults,
30
32
  ISummarizeTelemetryProperties,
33
+ ISummarizerRuntime,
31
34
  ISummarizeRunnerTelemetry,
32
35
  } from "./summarizerTypes";
33
36
  import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
@@ -58,6 +61,7 @@ export class RunningSummarizer implements IDisposable {
58
61
  summaryCollection: SummaryCollection,
59
62
  cancellationToken: ISummaryCancellationToken,
60
63
  stopSummarizerCallback: (reason: SummarizerStopReason) => void,
64
+ runtime: ISummarizerRuntime,
61
65
  ): Promise<RunningSummarizer> {
62
66
  const summarizer = new RunningSummarizer(
63
67
  logger,
@@ -68,12 +72,36 @@ export class RunningSummarizer implements IDisposable {
68
72
  raiseSummarizingError,
69
73
  summaryCollection,
70
74
  cancellationToken,
71
- stopSummarizerCallback);
75
+ stopSummarizerCallback,
76
+ runtime);
72
77
 
73
78
  await summarizer.waitStart();
74
79
 
75
- // Run the heuristics after starting
80
+ // Update heuristic counts
81
+ // By the time we get here, there are potentially ops missing from the heuristic summary counts
82
+ // Examples of where this could happen:
83
+ // 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we
84
+ // listen for the op events (will get missed by the handlers in the current workflow)
85
+ // 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)
86
+ const diff = runtime.deltaManager.lastSequenceNumber - (
87
+ heuristicData.lastSuccessfulSummary.refSequenceNumber
88
+ + heuristicData.numNonRuntimeOps
89
+ + heuristicData.numRuntimeOps);
90
+ heuristicData.hasMissingOpData = diff > 0;
91
+
92
+ if (heuristicData.hasMissingOpData) {
93
+ // Split the diff 50-50 and increment the counts appropriately
94
+ heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);
95
+ heuristicData.numRuntimeOps += Math.floor(diff / 2);
96
+ }
97
+
98
+ // Update last seq number (in case the handlers haven't processed anything yet)
99
+ heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
100
+
101
+ // Start heuristics
102
+ summarizer.heuristicRunner?.start();
76
103
  summarizer.heuristicRunner?.run();
104
+
77
105
  return summarizer;
78
106
  }
79
107
 
@@ -95,6 +123,7 @@ export class RunningSummarizer implements IDisposable {
95
123
  } | undefined;
96
124
  private summarizeCount = 0;
97
125
  private totalSuccessfulAttempts = 0;
126
+ private initialized = false;
98
127
 
99
128
  private constructor(
100
129
  baseLogger: ITelemetryLogger,
@@ -106,6 +135,7 @@ export class RunningSummarizer implements IDisposable {
106
135
  private readonly summaryCollection: SummaryCollection,
107
136
  private readonly cancellationToken: ISummaryCancellationToken,
108
137
  private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
138
+ private readonly runtime: ISummarizerRuntime,
109
139
  ) {
110
140
  const telemetryProps: ISummarizeRunnerTelemetry = {
111
141
  summarizeCount: () => this.summarizeCount,
@@ -175,9 +205,13 @@ export class RunningSummarizer implements IDisposable {
175
205
  this.summaryWatcher,
176
206
  this.logger,
177
207
  );
208
+
209
+ // Listen for ops
210
+ this.runtime.deltaManager.on("op", (op) => { this.handleOp(op); });
178
211
  }
179
212
 
180
213
  public dispose(): void {
214
+ this.runtime.deltaManager.off("op", (op) => { this.handleOp(op); });
181
215
  this.summaryWatcher.dispose();
182
216
  this.heuristicRunner?.dispose();
183
217
  this.heuristicRunner = undefined;
@@ -199,30 +233,48 @@ export class RunningSummarizer implements IDisposable {
199
233
  ? this.logger
200
234
  : undefined;
201
235
 
202
- public handleSystemOp(op: ISequencedDocumentMessage) {
203
- switch (op.type) {
204
- case MessageType.ClientLeave:
205
- case MessageType.ClientJoin:
206
- case MessageType.Propose: {
207
- // Synchronously handle quorum ops like regular ops
208
- this.handleOp(undefined, op);
209
- return;
210
- }
211
- default: {
212
- return;
213
- }
214
- }
215
- }
236
+ /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
237
+ private heuristicRunnerMicroTaskExists = false;
216
238
 
217
- public handleOp(error: any, { sequenceNumber, type, clientId, contents }: ISequencedDocumentMessage) {
218
- if (error !== undefined) {
219
- return;
239
+ public handleOp(op: ISequencedDocumentMessage) {
240
+ this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
241
+
242
+ if (op.type !== MessageType.Summarize && isRuntimeMessage(op)) {
243
+ this.heuristicData.numRuntimeOps++;
244
+ } else {
245
+ this.heuristicData.numNonRuntimeOps++;
220
246
  }
221
- this.heuristicData.lastOpSequenceNumber = sequenceNumber;
247
+
248
+ this.heuristicData.totalOpsSize += opSize(op);
222
249
 
223
250
  // Check for enqueued on-demand summaries; Intentionally do nothing otherwise
224
- if (!this.tryRunEnqueuedSummary()) {
225
- this.heuristicRunner?.run();
251
+ if (this.initialized
252
+ && this.opCanTriggerSummary(op)
253
+ && !this.tryRunEnqueuedSummary()
254
+ && !this.heuristicRunnerMicroTaskExists) {
255
+ this.heuristicRunnerMicroTaskExists = true;
256
+ Promise.resolve().then(() => {
257
+ this.heuristicRunner?.run();
258
+ }).finally(() => {
259
+ this.heuristicRunnerMicroTaskExists = false;
260
+ });
261
+ }
262
+ }
263
+
264
+ /**
265
+ * Can the given op trigger a summary?
266
+ * # Currently only prevents summaries for Summarize and SummaryAck ops
267
+ * @param op - op to check
268
+ * @returns true if this type of op can trigger a summary
269
+ */
270
+ private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
271
+ switch (op.type) {
272
+ case MessageType.Summarize:
273
+ case MessageType.SummaryAck:
274
+ case MessageType.SummaryNack:
275
+ return false;
276
+ default:
277
+ return true;
226
278
  }
227
279
  }
228
280
 
@@ -274,6 +326,7 @@ export class RunningSummarizer implements IDisposable {
274
326
  summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
275
327
  });
276
328
  }
329
+ this.initialized = true;
277
330
  }
278
331
 
279
332
  /**
package/src/summarizer.ts CHANGED
@@ -17,9 +17,6 @@ import {
17
17
  IFluidHandle,
18
18
  IRequest,
19
19
  } from "@fluidframework/core-interfaces";
20
- import {
21
- ISequencedDocumentMessage,
22
- } from "@fluidframework/protocol-definitions";
23
20
  import { ISummaryConfiguration } from "./containerRuntime";
24
21
  import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
25
22
  import { summarizerClientType } from "./summarizerClientElection";
@@ -70,8 +67,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
70
67
 
71
68
  private readonly logger: ITelemetryLogger;
72
69
  private runningSummarizer?: RunningSummarizer;
73
- private systemOpListener?: (op: ISequencedDocumentMessage) => void;
74
- private opListener?: (error: any, op: ISequencedDocumentMessage) => void;
75
70
  private _disposed: boolean = false;
76
71
  private starting: boolean = false;
77
72
 
@@ -277,6 +272,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
277
272
  this.summaryCollection,
278
273
  runCoordinator /* cancellationToken */,
279
274
  (reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */
275
+ this.runtime,
280
276
  );
281
277
  this.runningSummarizer = runningSummarizer;
282
278
  this.starting = false;
@@ -287,13 +283,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
287
283
  this.logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
288
284
  });
289
285
 
290
- // Listen for ops
291
- this.systemOpListener = (op: ISequencedDocumentMessage) => runningSummarizer.handleSystemOp(op);
292
- this.runtime.deltaManager.inbound.on("op", this.systemOpListener);
293
-
294
- this.opListener = (error: any, op: ISequencedDocumentMessage) => runningSummarizer.handleOp(error, op);
295
- this.runtime.on("batchEnd", this.opListener);
296
-
297
286
  return runningSummarizer;
298
287
  }
299
288
 
@@ -312,12 +301,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
312
301
  this.runningSummarizer.dispose();
313
302
  this.runningSummarizer = undefined;
314
303
  }
315
- if (this.systemOpListener) {
316
- this.runtime.deltaManager.inbound.off("op", this.systemOpListener);
317
- }
318
- if (this.opListener) {
319
- this.runtime.removeListener("batchEnd", this.opListener);
320
- }
321
304
  }
322
305
 
323
306
  public readonly summarizeOnDemand: ISummarizer["summarizeOnDemand"] = (...args) => {
@@ -6,11 +6,11 @@
6
6
  import { ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { Timer } from "@fluidframework/common-utils";
8
8
  import { ISummaryConfigurationHeuristics } from "./containerRuntime";
9
-
10
9
  import {
11
10
  ISummarizeHeuristicData,
12
11
  ISummarizeHeuristicRunner,
13
12
  ISummarizeAttempt,
13
+ ISummaryHeuristicStrategy,
14
14
  } from "./summarizerTypes";
15
15
  import { SummarizeReason } from "./summaryGenerator";
16
16
 
@@ -26,6 +26,30 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
26
26
  return this._lastSuccessfulSummary;
27
27
  }
28
28
 
29
+ public numNonRuntimeOps: number = 0;
30
+ public totalOpsSize: number = 0;
31
+ public hasMissingOpData: boolean = false;
32
+
33
+ /**
34
+ * Cumulative size in bytes of all the ops at the beginning of the summarization attempt.
35
+ * Is used to adjust totalOpsSize appropriately after successful summarization.
36
+ */
37
+ /** */
38
+ private totalOpsSizeBefore: number = 0;
39
+
40
+ /**
41
+ * Number of system ops at beginning of attempting to summarize.
42
+ * Is used to adjust numSystemOps appropriately after successful summarization.
43
+ */
44
+ private numSystemOpsBefore: number = 0;
45
+
46
+ public numRuntimeOps: number = 0;
47
+ /**
48
+ * Number of non-system ops at beginning of attempting to summarize.
49
+ * Is used to adjust numNonSystemOps appropriately after successful summarization.
50
+ */
51
+ private numNonSystemOpsBefore: number = 0;
52
+
29
53
  constructor(
30
54
  public lastOpSequenceNumber: number,
31
55
  /** Baseline attempt data used for comparisons with subsequent attempts/calculations. */
@@ -45,10 +69,23 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
45
69
  refSequenceNumber: refSequenceNumber ?? this.lastOpSequenceNumber,
46
70
  summaryTime: Date.now(),
47
71
  };
72
+
73
+ this.numSystemOpsBefore = this.numNonRuntimeOps;
74
+ this.numNonSystemOpsBefore = this.numRuntimeOps;
75
+ this.totalOpsSizeBefore = this.totalOpsSize;
48
76
  }
49
77
 
50
78
  public markLastAttemptAsSuccessful() {
51
79
  this._lastSuccessfulSummary = { ...this.lastAttempt };
80
+
81
+ this.numNonRuntimeOps -= this.numSystemOpsBefore;
82
+ this.numSystemOpsBefore = 0;
83
+
84
+ this.numRuntimeOps -= this.numNonSystemOpsBefore;
85
+ this.numNonSystemOpsBefore = 0;
86
+
87
+ this.totalOpsSize -= this.totalOpsSizeBefore;
88
+ this.totalOpsSizeBefore = 0;
52
89
  }
53
90
  }
54
91
 
@@ -56,42 +93,71 @@ export class SummarizeHeuristicData implements ISummarizeHeuristicData {
56
93
  * This class contains the heuristics for when to summarize.
57
94
  */
58
95
  export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
59
- private readonly idleTimer: Timer;
60
- private readonly minOpsForLastSummaryAttempt: number;
96
+ private readonly idleTimer: Timer | undefined;
97
+ private readonly runSummarize: (reason: SummarizeReason) => void;
61
98
 
62
99
  public constructor(
63
100
  private readonly heuristicData: ISummarizeHeuristicData,
64
101
  private readonly configuration: ISummaryConfigurationHeuristics,
65
- private readonly trySummarize: (reason: SummarizeReason) => void,
102
+ trySummarize: (reason: SummarizeReason) => void,
66
103
  private readonly logger: ITelemetryLogger,
104
+ private readonly summarizeStrategies: ISummaryHeuristicStrategy[] = getDefaultSummaryHeuristicStrategies(),
67
105
  ) {
68
106
  this.idleTimer = new Timer(
69
- this.configuration.idleTime,
70
- () => this.trySummarize("idle"));
71
- this.minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
107
+ this.idleTime,
108
+ () => this.runSummarize("idle"));
109
+
110
+ this.runSummarize = (reason: SummarizeReason) => {
111
+ this.idleTimer?.clear();
112
+
113
+ // We shouldn't attempt a summary if there are no new processed ops
114
+ const opsSinceLastAck = this.opsSinceLastAck;
115
+ if (opsSinceLastAck > 0) {
116
+ trySummarize(reason);
117
+ }
118
+ };
119
+ }
120
+
121
+ public get idleTime(): number {
122
+ const maxIdleTime = this.configuration.maxIdleTime;
123
+ const minIdleTime = this.configuration.minIdleTime;
124
+ const weightedNumOfOps = getWeightedNumberOfOps(
125
+ this.heuristicData.numRuntimeOps,
126
+ this.heuristicData.numNonRuntimeOps,
127
+ this.configuration.runtimeOpWeight,
128
+ this.configuration.nonRuntimeOpWeight,
129
+ );
130
+ const pToMaxOps = weightedNumOfOps * 1.0 / this.configuration.maxOps;
131
+
132
+ if (pToMaxOps >= 1) {
133
+ return minIdleTime;
134
+ }
135
+
136
+ // Return a ratioed idle time based on the percentage of ops
137
+ return maxIdleTime - ((maxIdleTime - minIdleTime) * pToMaxOps);
72
138
  }
73
139
 
74
140
  public get opsSinceLastAck(): number {
75
141
  return this.heuristicData.lastOpSequenceNumber - this.heuristicData.lastSuccessfulSummary.refSequenceNumber;
76
142
  }
77
143
 
144
+ public start() {
145
+ this.idleTimer?.start(this.idleTime);
146
+ }
147
+
78
148
  public run() {
79
- const timeSinceLastSummary = Date.now() - this.heuristicData.lastSuccessfulSummary.summaryTime;
80
- const opsSinceLastAck = this.opsSinceLastAck;
81
- if (timeSinceLastSummary > this.configuration.maxTime) {
82
- this.idleTimer.clear();
83
- this.trySummarize("maxTime");
84
- } else if (opsSinceLastAck > this.configuration.maxOps) {
85
- this.idleTimer.clear();
86
- this.trySummarize("maxOps");
87
- } else {
88
- this.idleTimer.restart();
149
+ for (const strategy of this.summarizeStrategies) {
150
+ if (strategy.shouldRunSummary(this.configuration, this.heuristicData)) {
151
+ return this.runSummarize(strategy.summarizeReason);
152
+ }
89
153
  }
154
+
155
+ this.idleTimer?.restart(this.idleTime);
90
156
  }
91
157
 
92
158
  public shouldRunLastSummary(): boolean {
93
159
  const opsSinceLastAck = this.opsSinceLastAck;
94
- const minOpsForLastSummaryAttempt = this.minOpsForLastSummaryAttempt;
160
+ const minOpsForLastSummaryAttempt = this.configuration.minOpsForLastSummaryAttempt;
95
161
 
96
162
  this.logger.sendTelemetryEvent({
97
163
  eventName: "ShouldRunLastSummary",
@@ -103,6 +169,54 @@ export class SummarizeHeuristicRunner implements ISummarizeHeuristicRunner {
103
169
  }
104
170
 
105
171
  public dispose() {
106
- this.idleTimer.clear();
172
+ this.idleTimer?.clear();
173
+ }
174
+ }
175
+
176
+ /** Strategy used to run a summary when it's been a while since our last successful summary */
177
+ class MaxTimeSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
178
+ public readonly summarizeReason: Readonly<SummarizeReason> = "maxTime";
179
+
180
+ public shouldRunSummary(
181
+ configuration: ISummaryConfigurationHeuristics,
182
+ heuristicData: ISummarizeHeuristicData,
183
+ ): boolean {
184
+ const timeSinceLastSummary = Date.now() - heuristicData.lastSuccessfulSummary.summaryTime;
185
+ return timeSinceLastSummary > configuration.maxTime;
107
186
  }
108
187
  }
188
+
189
+ function getWeightedNumberOfOps(
190
+ runtimeOpCount: number,
191
+ nonRuntimeOpCount: number,
192
+ runtimeOpWeight: number,
193
+ nonRuntimeOpWeight: number,
194
+ ): number {
195
+ return (runtimeOpWeight * runtimeOpCount)
196
+ + (nonRuntimeOpWeight * nonRuntimeOpCount);
197
+ }
198
+
199
+ /** Strategy used to do a weighted analysis on the ops we've processed since the last successful summary */
200
+ class WeightedOpsSummaryHeuristicStrategy implements ISummaryHeuristicStrategy {
201
+ public readonly summarizeReason: Readonly<SummarizeReason> = "maxOps";
202
+
203
+ public shouldRunSummary(
204
+ configuration: ISummaryConfigurationHeuristics,
205
+ heuristicData: ISummarizeHeuristicData,
206
+ ): boolean {
207
+ const weightedNumOfOps = getWeightedNumberOfOps(
208
+ heuristicData.numRuntimeOps,
209
+ heuristicData.numNonRuntimeOps,
210
+ configuration.runtimeOpWeight,
211
+ configuration.nonRuntimeOpWeight,
212
+ );
213
+ return weightedNumOfOps > configuration.maxOps;
214
+ }
215
+ }
216
+
217
+ function getDefaultSummaryHeuristicStrategies() {
218
+ return [
219
+ new MaxTimeSummaryHeuristicStrategy(),
220
+ new WeightedOpsSummaryHeuristicStrategy(),
221
+ ];
222
+ }