@fluidframework/container-runtime 1.2.2 → 2.0.0-internal.1.0.0.82159

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (135) hide show
  1. package/dist/blobManager.d.ts +81 -25
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +301 -100
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +65 -11
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +101 -82
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStore.d.ts +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +32 -26
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +3 -4
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +16 -23
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStores.d.ts +5 -2
  18. package/dist/dataStores.d.ts.map +1 -1
  19. package/dist/dataStores.js +11 -3
  20. package/dist/dataStores.js.map +1 -1
  21. package/dist/garbageCollection.d.ts.map +1 -1
  22. package/dist/garbageCollection.js +17 -12
  23. package/dist/garbageCollection.js.map +1 -1
  24. package/dist/opProperties.d.ts +7 -0
  25. package/dist/opProperties.d.ts.map +1 -0
  26. package/dist/opProperties.js +20 -0
  27. package/dist/opProperties.js.map +1 -0
  28. package/dist/packageVersion.d.ts +1 -1
  29. package/dist/packageVersion.d.ts.map +1 -1
  30. package/dist/packageVersion.js +1 -1
  31. package/dist/packageVersion.js.map +1 -1
  32. package/dist/runningSummarizer.d.ts +14 -4
  33. package/dist/runningSummarizer.d.ts.map +1 -1
  34. package/dist/runningSummarizer.js +68 -26
  35. package/dist/runningSummarizer.js.map +1 -1
  36. package/dist/summarizer.d.ts +0 -2
  37. package/dist/summarizer.d.ts.map +1 -1
  38. package/dist/summarizer.js +1 -12
  39. package/dist/summarizer.js.map +1 -1
  40. package/dist/summarizerHeuristics.d.ts +26 -4
  41. package/dist/summarizerHeuristics.d.ts.map +1 -1
  42. package/dist/summarizerHeuristics.js +95 -18
  43. package/dist/summarizerHeuristics.js.map +1 -1
  44. package/dist/summarizerTypes.d.ts +30 -10
  45. package/dist/summarizerTypes.d.ts.map +1 -1
  46. package/dist/summarizerTypes.js.map +1 -1
  47. package/dist/summaryCollection.js +1 -1
  48. package/dist/summaryCollection.js.map +1 -1
  49. package/dist/summaryFormat.d.ts +0 -5
  50. package/dist/summaryFormat.d.ts.map +1 -1
  51. package/dist/summaryFormat.js.map +1 -1
  52. package/dist/summaryGenerator.d.ts +1 -0
  53. package/dist/summaryGenerator.d.ts.map +1 -1
  54. package/dist/summaryGenerator.js +11 -9
  55. package/dist/summaryGenerator.js.map +1 -1
  56. package/lib/blobManager.d.ts +81 -25
  57. package/lib/blobManager.d.ts.map +1 -1
  58. package/lib/blobManager.js +302 -101
  59. package/lib/blobManager.js.map +1 -1
  60. package/lib/containerRuntime.d.ts +65 -11
  61. package/lib/containerRuntime.d.ts.map +1 -1
  62. package/lib/containerRuntime.js +103 -84
  63. package/lib/containerRuntime.js.map +1 -1
  64. package/lib/dataStore.d.ts +1 -1
  65. package/lib/dataStore.d.ts.map +1 -1
  66. package/lib/dataStore.js +32 -26
  67. package/lib/dataStore.js.map +1 -1
  68. package/lib/dataStoreContext.d.ts +3 -4
  69. package/lib/dataStoreContext.d.ts.map +1 -1
  70. package/lib/dataStoreContext.js +17 -24
  71. package/lib/dataStoreContext.js.map +1 -1
  72. package/lib/dataStores.d.ts +5 -2
  73. package/lib/dataStores.d.ts.map +1 -1
  74. package/lib/dataStores.js +11 -3
  75. package/lib/dataStores.js.map +1 -1
  76. package/lib/garbageCollection.d.ts.map +1 -1
  77. package/lib/garbageCollection.js +17 -12
  78. package/lib/garbageCollection.js.map +1 -1
  79. package/lib/opProperties.d.ts +7 -0
  80. package/lib/opProperties.d.ts.map +1 -0
  81. package/lib/opProperties.js +16 -0
  82. package/lib/opProperties.js.map +1 -0
  83. package/lib/packageVersion.d.ts +1 -1
  84. package/lib/packageVersion.d.ts.map +1 -1
  85. package/lib/packageVersion.js +1 -1
  86. package/lib/packageVersion.js.map +1 -1
  87. package/lib/runningSummarizer.d.ts +14 -4
  88. package/lib/runningSummarizer.d.ts.map +1 -1
  89. package/lib/runningSummarizer.js +68 -26
  90. package/lib/runningSummarizer.js.map +1 -1
  91. package/lib/summarizer.d.ts +0 -2
  92. package/lib/summarizer.d.ts.map +1 -1
  93. package/lib/summarizer.js +1 -12
  94. package/lib/summarizer.js.map +1 -1
  95. package/lib/summarizerHeuristics.d.ts +26 -4
  96. package/lib/summarizerHeuristics.d.ts.map +1 -1
  97. package/lib/summarizerHeuristics.js +95 -18
  98. package/lib/summarizerHeuristics.js.map +1 -1
  99. package/lib/summarizerTypes.d.ts +30 -10
  100. package/lib/summarizerTypes.d.ts.map +1 -1
  101. package/lib/summarizerTypes.js.map +1 -1
  102. package/lib/summaryCollection.js +1 -1
  103. package/lib/summaryCollection.js.map +1 -1
  104. package/lib/summaryFormat.d.ts +0 -5
  105. package/lib/summaryFormat.d.ts.map +1 -1
  106. package/lib/summaryFormat.js.map +1 -1
  107. package/lib/summaryGenerator.d.ts +1 -0
  108. package/lib/summaryGenerator.d.ts.map +1 -1
  109. package/lib/summaryGenerator.js +11 -9
  110. package/lib/summaryGenerator.js.map +1 -1
  111. package/package.json +45 -20
  112. package/src/blobManager.ts +360 -119
  113. package/src/containerRuntime.ts +203 -103
  114. package/src/dataStore.ts +53 -38
  115. package/src/dataStoreContext.ts +16 -23
  116. package/src/dataStores.ts +14 -3
  117. package/src/garbageCollection.ts +13 -7
  118. package/src/opProperties.ts +19 -0
  119. package/src/packageVersion.ts +1 -1
  120. package/src/runningSummarizer.ts +75 -22
  121. package/src/summarizer.ts +1 -18
  122. package/src/summarizerHeuristics.ts +133 -19
  123. package/src/summarizerTypes.ts +37 -10
  124. package/src/summaryCollection.ts +1 -1
  125. package/src/summaryFormat.ts +0 -6
  126. package/src/summaryGenerator.ts +40 -22
  127. package/dist/opTelemetry.d.ts +0 -22
  128. package/dist/opTelemetry.d.ts.map +0 -1
  129. package/dist/opTelemetry.js +0 -59
  130. package/dist/opTelemetry.js.map +0 -1
  131. package/lib/opTelemetry.d.ts +0 -22
  132. package/lib/opTelemetry.d.ts.map +0 -1
  133. package/lib/opTelemetry.js +0 -55
  134. package/lib/opTelemetry.js.map +0 -1
  135. package/src/opTelemetry.ts +0 -71
package/src/dataStore.ts CHANGED
@@ -43,7 +43,8 @@ export const channelToDataStore = (
43
43
  runtime: ContainerRuntime,
44
44
  datastores: DataStores,
45
45
  logger: ITelemetryLogger,
46
- ): IDataStore => new DataStore(fluidDataStoreChannel, internalId, runtime, datastores, logger);
46
+ alreadyAliased: boolean = false,
47
+ ): IDataStore => new DataStore(fluidDataStoreChannel, internalId, runtime, datastores, logger, alreadyAliased);
47
48
 
48
49
  enum AliasState {
49
50
  Aliased = "Aliased",
@@ -54,6 +55,7 @@ enum AliasState {
54
55
  class DataStore implements IDataStore {
55
56
  private aliasState: AliasState = AliasState.None;
56
57
  private alias: string | undefined;
58
+ private readonly pendingAliases: Map<string, Promise<AliasResult>>;
57
59
  private aliasResult: Promise<AliasResult> | undefined;
58
60
 
59
61
  async trySetAlias(alias: string): Promise<AliasResult> {
@@ -75,14 +77,25 @@ class DataStore implements IDataStore {
75
77
  case AliasState.Aliased:
76
78
  return this.alias === alias ? "Success" : "AlreadyAliased";
77
79
 
78
- // There is no current or past alias operation for this datastore,
79
- // it is safe to continue execution
80
- case AliasState.None: break;
80
+ case AliasState.None: {
81
+ const existingAlias = this.pendingAliases.get(alias);
82
+ if (existingAlias !== undefined) {
83
+ // There is already another datastore which will be aliased
84
+ // to the same name
85
+ return "Conflict";
86
+ }
87
+
88
+ // There is no current or past alias operation for this datastore,
89
+ // or for this alias, so it is safe to continue execution
90
+ break;
91
+ }
92
+
81
93
  default: unreachableCase(this.aliasState);
82
94
  }
83
95
 
84
96
  this.aliasState = AliasState.Aliasing;
85
97
  this.aliasResult = this.trySetAliasInternal(alias);
98
+ this.pendingAliases.set(alias, this.aliasResult);
86
99
  return this.aliasResult;
87
100
  }
88
101
 
@@ -92,13 +105,7 @@ class DataStore implements IDataStore {
92
105
  alias,
93
106
  };
94
107
 
95
- // back-compat 0.58.2000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
96
- // older versions, we still have to call bindToContext.
97
- if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
98
- this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
99
- } else {
100
- this.fluidDataStoreChannel.bindToContext();
101
- }
108
+ this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
102
109
 
103
110
  if (this.runtime.attachState === AttachState.Detached) {
104
111
  const localResult = this.datastores.processAliasMessageCore(message);
@@ -108,34 +115,37 @@ class DataStore implements IDataStore {
108
115
  return localResult ? "Success" : "Conflict";
109
116
  }
110
117
 
111
- const aliased = await this.ackBasedPromise<boolean>((resolve) => {
112
- this.runtime.submitDataStoreAliasOp(message, resolve);
113
- }).then((succeeded) => {
114
- // Explicitly Lock-out future attempts of aliasing,
115
- // regardless of result
116
- this.aliasState = AliasState.Aliased;
117
- if (succeeded) {
118
- this.alias = alias;
119
- }
120
-
121
- return succeeded;
122
- }).catch((error) => {
123
- this.logger.sendErrorEvent({
124
- eventName: "AliasingException",
125
- alias: {
126
- value: alias,
127
- tag: TelemetryDataTag.UserData,
128
- },
129
- internalId: {
130
- value: this.internalId,
131
- tag: TelemetryDataTag.PackageData,
132
- },
133
- }, error);
118
+ const aliased = await this
119
+ .ackBasedPromise<boolean>((resolve) => {
120
+ this.runtime.submitDataStoreAliasOp(message, resolve);
121
+ })
122
+ .catch((error) => {
123
+ this.logger.sendErrorEvent({
124
+ eventName: "AliasingException",
125
+ alias: {
126
+ value: alias,
127
+ tag: TelemetryDataTag.UserData,
128
+ },
129
+ internalId: {
130
+ value: this.internalId,
131
+ tag: TelemetryDataTag.CodeArtifact,
132
+ },
133
+ }, error);
134
+
135
+ return false;
136
+ }).finally(() => {
137
+ this.pendingAliases.delete(alias);
138
+ });
139
+
140
+ if (!aliased) {
134
141
  this.aliasState = AliasState.None;
135
- return false;
136
- });
142
+ this.aliasResult = undefined;
143
+ return "Conflict";
144
+ }
137
145
 
138
- return aliased ? "Success" : "Conflict";
146
+ this.alias = alias;
147
+ this.aliasState = AliasState.Aliased;
148
+ return "Success";
139
149
  }
140
150
 
141
151
  async request(request: IRequest): Promise<IResponse> {
@@ -148,7 +158,12 @@ class DataStore implements IDataStore {
148
158
  private readonly runtime: ContainerRuntime,
149
159
  private readonly datastores: DataStores,
150
160
  private readonly logger: ITelemetryLogger,
151
- ) { }
161
+ alreadyAliased: boolean,
162
+ ) {
163
+ this.pendingAliases = datastores.pendingAliases;
164
+ this.aliasState = alreadyAliased ? AliasState.Aliased : AliasState.None;
165
+ }
166
+
152
167
  public get IFluidRouter() { return this.fluidDataStoreChannel; }
153
168
 
154
169
  private async ackBasedPromise<T>(
@@ -13,7 +13,6 @@ import {
13
13
  import {
14
14
  IAudience,
15
15
  IDeltaManager,
16
- BindState,
17
16
  AttachState,
18
17
  ILoaderOptions,
19
18
  } from "@fluidframework/container-definitions";
@@ -232,7 +231,9 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
232
231
  protected registry: IFluidDataStoreRegistry | undefined;
233
232
 
234
233
  protected detachedRuntimeCreation = false;
235
- public readonly bindToContext: () => void;
234
+ // back-compat (for tests) - can be removed in 2.0.0-alpha.2.0.0, or earlier if compat tests drop n/n-2 coverage
235
+ // @ts-expect-error - This shouldn't be referenced in the current version, but needs to be here for back-compat
236
+ private readonly bindToContext: () => void;
236
237
  protected channel: IFluidDataStoreChannel | undefined;
237
238
  private loaded = false;
238
239
  protected pending: ISequencedDocumentMessage[] | undefined = [];
@@ -260,7 +261,6 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
260
261
  constructor(
261
262
  props: IFluidDataStoreContextProps,
262
263
  private readonly existing: boolean,
263
- private bindState: BindState,
264
264
  public readonly isLocalDataStore: boolean,
265
265
  private readonly makeLocallyVisibleFn: () => void,
266
266
  ) {
@@ -282,11 +282,8 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
282
282
  this.containerRuntime.attachState : AttachState.Detached;
283
283
 
284
284
  this.bindToContext = () => {
285
- assert(this.bindState === BindState.NotBound, 0x13b /* "datastore context is already in bound state" */);
286
- this.bindState = BindState.Binding;
287
285
  assert(this.channel !== undefined, 0x13c /* "undefined channel on datastore context" */);
288
286
  this.makeLocallyVisible();
289
- this.bindState = BindState.Bound;
290
287
  };
291
288
 
292
289
  const thisSummarizeInternal =
@@ -319,7 +316,7 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
319
316
  }
320
317
 
321
318
  private rejectDeferredRealize(reason: string, packageName?: string): never {
322
- throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.PackageData } });
319
+ throw new LoggingError(reason, { packageName: { value: packageName, tag: TelemetryDataTag.CodeArtifact } });
323
320
  }
324
321
 
325
322
  public async realize(): Promise<IFluidDataStoreChannel> {
@@ -328,7 +325,12 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
328
325
  this.channelDeferred = new Deferred<IFluidDataStoreChannel>();
329
326
  this.realizeCore(this.existing).catch((error) => {
330
327
  const errorWrapped = DataProcessingError.wrapIfUnrecognized(error, "realizeFluidDataStoreContext");
331
- errorWrapped.addTelemetryProperties({ fluidDataStoreId: { value: this.id, tag: "PackageData" } });
328
+ errorWrapped.addTelemetryProperties({
329
+ fluidDataStoreId: {
330
+ value: this.id,
331
+ tag: TelemetryDataTag.CodeArtifact,
332
+ },
333
+ });
332
334
  this.channelDeferred?.reject(errorWrapped);
333
335
  this.logger.sendErrorEvent({ eventName: "RealizeError" }, errorWrapped);
334
336
  });
@@ -690,7 +692,10 @@ export abstract class FluidDataStoreContext extends TypedEventEmitter<IFluidData
690
692
  } catch (error) {
691
693
  this.channelDeferred?.reject(error);
692
694
  this.logger.sendErrorEvent(
693
- { eventName: "BindRuntimeError", fluidDataStoreId: { value: this.id, tag: "PackageData" } },
695
+ { eventName: "BindRuntimeError", fluidDataStoreId: {
696
+ value: this.id,
697
+ tag: TelemetryDataTag.CodeArtifact,
698
+ } },
694
699
  error);
695
700
  }
696
701
  }
@@ -783,7 +788,6 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
783
788
  super(
784
789
  props,
785
790
  true /* existing */,
786
- BindState.Bound,
787
791
  false /* isLocalDataStore */,
788
792
  () => {
789
793
  throw new Error("Already attached");
@@ -809,11 +813,7 @@ export class RemoteFluidDataStoreContext extends FluidDataStoreContext {
809
813
 
810
814
  const localReadAndParse = async <T>(id: string) => readAndParse<T>(this.storage, id);
811
815
  if (tree) {
812
- const loadedSummary = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
813
- tree = loadedSummary.baseSummary;
814
- // Prepend outstanding ops to pending queue of ops to process.
815
- // eslint-disable-next-line @typescript-eslint/no-non-null-assertion
816
- this.pending = loadedSummary.outstandingOps.concat(this.pending!);
816
+ tree = await this.summarizerNode.loadBaseSummary(tree, localReadAndParse);
817
817
  }
818
818
 
819
819
  if (!!tree && tree.blobs[dataStoreAttributesBlobName] !== undefined) {
@@ -892,7 +892,6 @@ export class LocalFluidDataStoreContextBase extends FluidDataStoreContext {
892
892
  super(
893
893
  props,
894
894
  props.snapshotTree !== undefined ? true : false /* existing */,
895
- props.snapshotTree ? BindState.Bound : BindState.NotBound,
896
895
  true /* isLocalDataStore */,
897
896
  props.makeLocallyVisibleFn,
898
897
  );
@@ -1040,13 +1039,7 @@ export class LocalDetachedFluidDataStoreContext
1040
1039
  super.bindRuntime(dataStoreChannel);
1041
1040
 
1042
1041
  if (await this.isRoot()) {
1043
- // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
1044
- // For older versions, we still have to call bindToContext.
1045
- if (dataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
1046
- dataStoreChannel.makeVisibleAndAttachGraph();
1047
- } else {
1048
- dataStoreChannel.bindToContext();
1049
- }
1042
+ dataStoreChannel.makeVisibleAndAttachGraph();
1050
1043
  }
1051
1044
  }
1052
1045
 
package/src/dataStores.ts CHANGED
@@ -12,6 +12,7 @@ import {
12
12
  ISnapshotTree,
13
13
  } from "@fluidframework/protocol-definitions";
14
14
  import {
15
+ AliasResult,
15
16
  channelsTreeName,
16
17
  CreateChildSummarizerNodeFn,
17
18
  CreateChildSummarizerNodeParam,
@@ -82,6 +83,7 @@ export class DataStores implements IDisposable {
82
83
  // The handle to the container runtime. This is used mainly for GC purposes to represent outbound reference from
83
84
  // the container runtime to other nodes.
84
85
  private readonly containerRuntimeHandle: IFluidHandle;
86
+ private readonly pendingAliasMap: Map<string, Promise<AliasResult>> = new Map<string, Promise<AliasResult>>();
85
87
 
86
88
  constructor(
87
89
  private readonly baseSnapshot: ISnapshotTree | undefined,
@@ -173,10 +175,19 @@ export class DataStores implements IDisposable {
173
175
  };
174
176
  }
175
177
 
176
- public aliases(): ReadonlyMap<string, string> {
178
+ public get aliases(): ReadonlyMap<string, string> {
177
179
  return this.aliasMap;
178
180
  }
179
181
 
182
+ public get pendingAliases(): Map<string, Promise<AliasResult>> {
183
+ return this.pendingAliasMap;
184
+ }
185
+
186
+ public async waitIfPendingAlias(maybeAlias: string): Promise<AliasResult> {
187
+ const pendingAliasPromise = this.pendingAliases.get(maybeAlias);
188
+ return pendingAliasPromise === undefined ? "Success" : pendingAliasPromise;
189
+ }
190
+
180
191
  public processAttachMessage(message: ISequencedDocumentMessage, local: boolean) {
181
192
  const attachMessage = message.contents as InboundAttachMessage;
182
193
 
@@ -201,7 +212,7 @@ export class DataStores implements IDisposable {
201
212
  ...extractSafePropertiesFromMessage(message),
202
213
  dataStoreId: {
203
214
  value: attachMessage.id,
204
- tag: TelemetryDataTag.PackageData,
215
+ tag: TelemetryDataTag.CodeArtifact,
205
216
  },
206
217
  },
207
218
  );
@@ -440,7 +451,7 @@ export class DataStores implements IDisposable {
440
451
  eventName: "SignalFluidDataStoreNotFound",
441
452
  fluidDataStoreId: {
442
453
  value: address,
443
- tag: TelemetryDataTag.PackageData,
454
+ tag: TelemetryDataTag.CodeArtifact,
444
455
  },
445
456
  });
446
457
  return;
@@ -63,7 +63,7 @@ export const gcBlobPrefix = "__gc";
63
63
  // Feature gate key to turn GC on / off.
64
64
  const runGCKey = "Fluid.GarbageCollection.RunGC";
65
65
  // Feature gate key to turn GC sweep on / off.
66
- const runSweepKey = "Fluid.GarbageCollection.RunSweep";
66
+ // const runSweepKey = "Fluid.GarbageCollection.RunSweep";
67
67
  // Feature gate key to turn GC test mode on / off.
68
68
  const gcTestModeKey = "Fluid.GarbageCollection.GCTestMode";
69
69
  // Feature gate key to write GC data at the root of the summary tree.
@@ -486,7 +486,7 @@ export class GarbageCollector implements IGarbageCollector {
486
486
 
487
487
  // Set the Session Expiry only if the flag is enabled or the test option is set.
488
488
  if (this.mc.config.getBoolean(runSessionExpiryKey) && this.gcEnabled) {
489
- this.sessionExpiryTimeoutMs = defaultSessionExpiryDurationMs;
489
+ this.sessionExpiryTimeoutMs = this.gcOptions.sessionExpiryTimeoutMs ?? defaultSessionExpiryDurationMs;
490
490
  }
491
491
  }
492
492
 
@@ -503,14 +503,19 @@ export class GarbageCollector implements IGarbageCollector {
503
503
  (timer) => { this.sessionExpiryTimer = timer; },
504
504
  );
505
505
 
506
+ // TEMPORARY: Hardcode a default of 2 days which is the value used in the ODSP driver.
507
+ // This unblocks the Sweep Log (see logSweepEvents function).
508
+ // This will be removed before sweep is fully implemented.
509
+ const snapshotCacheExpiryMs = createParams.snapshotCacheExpiryMs ?? 2 * 24 * 60 * 60 * 1000;
510
+
506
511
  /**
507
512
  * Sweep timeout is the time after which unreferenced content can be swept.
508
513
  * Sweep timeout = session expiry timeout + snapshot cache expiry timeout + one day buffer. The buffer is
509
514
  * added to account for any clock skew. We use server timestamps throughout so the skew should be minimal
510
515
  * but make it one day to be safe.
511
516
  */
512
- if (createParams.snapshotCacheExpiryMs !== undefined) {
513
- this.sweepTimeoutMs = this.sessionExpiryTimeoutMs + createParams.snapshotCacheExpiryMs + oneDayMs;
517
+ if (snapshotCacheExpiryMs !== undefined) {
518
+ this.sweepTimeoutMs = this.sessionExpiryTimeoutMs + snapshotCacheExpiryMs + oneDayMs;
514
519
  }
515
520
  }
516
521
 
@@ -538,9 +543,10 @@ export class GarbageCollector implements IGarbageCollector {
538
543
  * 3. Sweep should be enabled for this container (this.sweepEnabled). This can be overridden via runSweep
539
544
  * feature flag.
540
545
  */
541
- this.shouldRunSweep = this.shouldRunGC
542
- && this.sweepTimeoutMs !== undefined
543
- && (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
546
+ this.shouldRunSweep = false; // disable while TEMPORARY measure hardcoding snapshotCacheExpiryMs is here
547
+ // this.shouldRunGC
548
+ // && this.sweepTimeoutMs !== undefined
549
+ // && (this.mc.config.getBoolean(runSweepKey) ?? this.sweepEnabled);
544
550
 
545
551
  this.trackGCState = this.mc.config.getBoolean(trackGCStateKey) === true;
546
552
 
@@ -0,0 +1,19 @@
1
+ /*!
2
+ * Copyright (c) Microsoft Corporation and contributors. All rights reserved.
3
+ * Licensed under the MIT License.
4
+ */
5
+
6
+ import { ISequencedDocumentMessage, ISequencedDocumentSystemMessage } from "@fluidframework/protocol-definitions";
7
+
8
+ export const opSize = (op: ISequencedDocumentMessage): number => {
9
+ // Some messages may already have string contents,
10
+ // so stringifying them again will add inaccurate overhead.
11
+ const content = typeof op.contents === "string" ?
12
+ op.contents :
13
+ JSON.stringify(op.contents) ?? "";
14
+ const data = opHasData(op) ? op.data : "";
15
+ return content.length + data.length;
16
+ };
17
+
18
+ const opHasData = (op: ISequencedDocumentMessage): op is ISequencedDocumentSystemMessage =>
19
+ (op as ISequencedDocumentSystemMessage).data !== undefined;
@@ -6,4 +6,4 @@
6
6
  */
7
7
 
8
8
  export const pkgName = "@fluidframework/container-runtime";
9
- export const pkgVersion = "1.2.2";
9
+ export const pkgVersion = "2.0.0-internal.1.0.0.82159";
@@ -6,6 +6,7 @@
6
6
  import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
7
7
  import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-utils";
8
8
  import { UsageError } from "@fluidframework/container-utils";
9
+ import { isRuntimeMessage } from "@fluidframework/driver-utils";
9
10
  import {
10
11
  ISequencedDocumentMessage,
11
12
  MessageType,
@@ -14,6 +15,7 @@ import { ChildLogger } from "@fluidframework/telemetry-utils";
14
15
  import {
15
16
  ISummaryConfiguration,
16
17
  } from "./containerRuntime";
18
+ import { opSize } from "./opProperties";
17
19
  import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
18
20
  import {
19
21
  IEnqueueSummarizeOptions,
@@ -28,6 +30,7 @@ import {
28
30
  ISummaryCancellationToken,
29
31
  ISummarizeResults,
30
32
  ISummarizeTelemetryProperties,
33
+ ISummarizerRuntime,
31
34
  ISummarizeRunnerTelemetry,
32
35
  } from "./summarizerTypes";
33
36
  import { IClientSummaryWatcher, SummaryCollection } from "./summaryCollection";
@@ -58,6 +61,7 @@ export class RunningSummarizer implements IDisposable {
58
61
  summaryCollection: SummaryCollection,
59
62
  cancellationToken: ISummaryCancellationToken,
60
63
  stopSummarizerCallback: (reason: SummarizerStopReason) => void,
64
+ runtime: ISummarizerRuntime,
61
65
  ): Promise<RunningSummarizer> {
62
66
  const summarizer = new RunningSummarizer(
63
67
  logger,
@@ -68,12 +72,36 @@ export class RunningSummarizer implements IDisposable {
68
72
  raiseSummarizingError,
69
73
  summaryCollection,
70
74
  cancellationToken,
71
- stopSummarizerCallback);
75
+ stopSummarizerCallback,
76
+ runtime);
72
77
 
73
78
  await summarizer.waitStart();
74
79
 
75
- // Run the heuristics after starting
80
+ // Update heuristic counts
81
+ // By the time we get here, there are potentially ops missing from the heuristic summary counts
82
+ // Examples of where this could happen:
83
+ // 1. Op is processed during the time that we are initiating the RunningSummarizer instance but before we
84
+ // listen for the op events (will get missed by the handlers in the current workflow)
85
+ // 2. Op was sequenced after the last time we summarized (op sequence number > summarize ref sequence number)
86
+ const diff = runtime.deltaManager.lastSequenceNumber - (
87
+ heuristicData.lastSuccessfulSummary.refSequenceNumber
88
+ + heuristicData.numNonRuntimeOps
89
+ + heuristicData.numRuntimeOps);
90
+ heuristicData.hasMissingOpData = diff > 0;
91
+
92
+ if (heuristicData.hasMissingOpData) {
93
+ // Split the diff 50-50 and increment the counts appropriately
94
+ heuristicData.numNonRuntimeOps += Math.ceil(diff / 2);
95
+ heuristicData.numRuntimeOps += Math.floor(diff / 2);
96
+ }
97
+
98
+ // Update last seq number (in case the handlers haven't processed anything yet)
99
+ heuristicData.lastOpSequenceNumber = runtime.deltaManager.lastSequenceNumber;
100
+
101
+ // Start heuristics
102
+ summarizer.heuristicRunner?.start();
76
103
  summarizer.heuristicRunner?.run();
104
+
77
105
  return summarizer;
78
106
  }
79
107
 
@@ -95,6 +123,7 @@ export class RunningSummarizer implements IDisposable {
95
123
  } | undefined;
96
124
  private summarizeCount = 0;
97
125
  private totalSuccessfulAttempts = 0;
126
+ private initialized = false;
98
127
 
99
128
  private constructor(
100
129
  baseLogger: ITelemetryLogger,
@@ -106,6 +135,7 @@ export class RunningSummarizer implements IDisposable {
106
135
  private readonly summaryCollection: SummaryCollection,
107
136
  private readonly cancellationToken: ISummaryCancellationToken,
108
137
  private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
138
+ private readonly runtime: ISummarizerRuntime,
109
139
  ) {
110
140
  const telemetryProps: ISummarizeRunnerTelemetry = {
111
141
  summarizeCount: () => this.summarizeCount,
@@ -175,9 +205,13 @@ export class RunningSummarizer implements IDisposable {
175
205
  this.summaryWatcher,
176
206
  this.logger,
177
207
  );
208
+
209
+ // Listen for ops
210
+ this.runtime.deltaManager.on("op", (op) => { this.handleOp(op); });
178
211
  }
179
212
 
180
213
  public dispose(): void {
214
+ this.runtime.deltaManager.off("op", (op) => { this.handleOp(op); });
181
215
  this.summaryWatcher.dispose();
182
216
  this.heuristicRunner?.dispose();
183
217
  this.heuristicRunner = undefined;
@@ -199,30 +233,48 @@ export class RunningSummarizer implements IDisposable {
199
233
  ? this.logger
200
234
  : undefined;
201
235
 
202
- public handleSystemOp(op: ISequencedDocumentMessage) {
203
- switch (op.type) {
204
- case MessageType.ClientLeave:
205
- case MessageType.ClientJoin:
206
- case MessageType.Propose: {
207
- // Synchronously handle quorum ops like regular ops
208
- this.handleOp(undefined, op);
209
- return;
210
- }
211
- default: {
212
- return;
213
- }
214
- }
215
- }
236
+ /** We only want a single heuristic runner micro-task (will provide better optimized grouping of ops) */
237
+ private heuristicRunnerMicroTaskExists = false;
216
238
 
217
- public handleOp(error: any, { sequenceNumber, type, clientId, contents }: ISequencedDocumentMessage) {
218
- if (error !== undefined) {
219
- return;
239
+ public handleOp(op: ISequencedDocumentMessage) {
240
+ this.heuristicData.lastOpSequenceNumber = op.sequenceNumber;
241
+
242
+ if (op.type !== MessageType.Summarize && isRuntimeMessage(op)) {
243
+ this.heuristicData.numRuntimeOps++;
244
+ } else {
245
+ this.heuristicData.numNonRuntimeOps++;
220
246
  }
221
- this.heuristicData.lastOpSequenceNumber = sequenceNumber;
247
+
248
+ this.heuristicData.totalOpsSize += opSize(op);
222
249
 
223
250
  // Check for enqueued on-demand summaries; Intentionally do nothing otherwise
224
- if (!this.tryRunEnqueuedSummary()) {
225
- this.heuristicRunner?.run();
251
+ if (this.initialized
252
+ && this.opCanTriggerSummary(op)
253
+ && !this.tryRunEnqueuedSummary()
254
+ && !this.heuristicRunnerMicroTaskExists) {
255
+ this.heuristicRunnerMicroTaskExists = true;
256
+ Promise.resolve().then(() => {
257
+ this.heuristicRunner?.run();
258
+ }).finally(() => {
259
+ this.heuristicRunnerMicroTaskExists = false;
260
+ });
261
+ }
262
+ }
263
+
264
+ /**
265
+ * Can the given op trigger a summary?
266
+ * # Currently only prevents summaries for Summarize and SummaryAck ops
267
+ * @param op - op to check
268
+ * @returns true if this type of op can trigger a summary
269
+ */
270
+ private opCanTriggerSummary(op: ISequencedDocumentMessage): boolean {
271
+ switch (op.type) {
272
+ case MessageType.Summarize:
273
+ case MessageType.SummaryAck:
274
+ case MessageType.SummaryNack:
275
+ return false;
276
+ default:
277
+ return true;
226
278
  }
227
279
  }
228
280
 
@@ -274,6 +326,7 @@ export class RunningSummarizer implements IDisposable {
274
326
  summarySequenceNumber: waitStartResult.value.summaryOp.sequenceNumber,
275
327
  });
276
328
  }
329
+ this.initialized = true;
277
330
  }
278
331
 
279
332
  /**
package/src/summarizer.ts CHANGED
@@ -17,9 +17,6 @@ import {
17
17
  IFluidHandle,
18
18
  IRequest,
19
19
  } from "@fluidframework/core-interfaces";
20
- import {
21
- ISequencedDocumentMessage,
22
- } from "@fluidframework/protocol-definitions";
23
20
  import { ISummaryConfiguration } from "./containerRuntime";
24
21
  import { ICancellableSummarizerController } from "./runWhileConnectedCoordinator";
25
22
  import { summarizerClientType } from "./summarizerClientElection";
@@ -70,8 +67,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
70
67
 
71
68
  private readonly logger: ITelemetryLogger;
72
69
  private runningSummarizer?: RunningSummarizer;
73
- private systemOpListener?: (op: ISequencedDocumentMessage) => void;
74
- private opListener?: (error: any, op: ISequencedDocumentMessage) => void;
75
70
  private _disposed: boolean = false;
76
71
  private starting: boolean = false;
77
72
 
@@ -277,6 +272,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
277
272
  this.summaryCollection,
278
273
  runCoordinator /* cancellationToken */,
279
274
  (reason) => runCoordinator.stop(reason), /* stopSummarizerCallback */
275
+ this.runtime,
280
276
  );
281
277
  this.runningSummarizer = runningSummarizer;
282
278
  this.starting = false;
@@ -287,13 +283,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
287
283
  this.logger.sendErrorEvent({ eventName: "HandleSummaryAckFatalError" }, error);
288
284
  });
289
285
 
290
- // Listen for ops
291
- this.systemOpListener = (op: ISequencedDocumentMessage) => runningSummarizer.handleSystemOp(op);
292
- this.runtime.deltaManager.inbound.on("op", this.systemOpListener);
293
-
294
- this.opListener = (error: any, op: ISequencedDocumentMessage) => runningSummarizer.handleOp(error, op);
295
- this.runtime.on("batchEnd", this.opListener);
296
-
297
286
  return runningSummarizer;
298
287
  }
299
288
 
@@ -312,12 +301,6 @@ export class Summarizer extends EventEmitter implements ISummarizer {
312
301
  this.runningSummarizer.dispose();
313
302
  this.runningSummarizer = undefined;
314
303
  }
315
- if (this.systemOpListener) {
316
- this.runtime.deltaManager.inbound.off("op", this.systemOpListener);
317
- }
318
- if (this.opListener) {
319
- this.runtime.removeListener("batchEnd", this.opListener);
320
- }
321
304
  }
322
305
 
323
306
  public readonly summarizeOnDemand: ISummarizer["summarizeOnDemand"] = (...args) => {