@fluidframework/container-runtime 1.2.0 → 2.0.0-internal.1.0.0.81589

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. package/dist/blobManager.d.ts +81 -25
  2. package/dist/blobManager.d.ts.map +1 -1
  3. package/dist/blobManager.js +301 -100
  4. package/dist/blobManager.js.map +1 -1
  5. package/dist/containerRuntime.d.ts +46 -3
  6. package/dist/containerRuntime.d.ts.map +1 -1
  7. package/dist/containerRuntime.js +98 -79
  8. package/dist/containerRuntime.js.map +1 -1
  9. package/dist/dataStore.d.ts +1 -1
  10. package/dist/dataStore.d.ts.map +1 -1
  11. package/dist/dataStore.js +32 -26
  12. package/dist/dataStore.js.map +1 -1
  13. package/dist/dataStoreContext.d.ts +3 -4
  14. package/dist/dataStoreContext.d.ts.map +1 -1
  15. package/dist/dataStoreContext.js +16 -23
  16. package/dist/dataStoreContext.js.map +1 -1
  17. package/dist/dataStores.d.ts +5 -2
  18. package/dist/dataStores.d.ts.map +1 -1
  19. package/dist/dataStores.js +11 -3
  20. package/dist/dataStores.js.map +1 -1
  21. package/dist/opProperties.d.ts +7 -0
  22. package/dist/opProperties.d.ts.map +1 -0
  23. package/dist/opProperties.js +19 -0
  24. package/dist/opProperties.js.map +1 -0
  25. package/dist/packageVersion.d.ts +1 -1
  26. package/dist/packageVersion.d.ts.map +1 -1
  27. package/dist/packageVersion.js +1 -1
  28. package/dist/packageVersion.js.map +1 -1
  29. package/dist/runningSummarizer.d.ts +14 -4
  30. package/dist/runningSummarizer.d.ts.map +1 -1
  31. package/dist/runningSummarizer.js +68 -26
  32. package/dist/runningSummarizer.js.map +1 -1
  33. package/dist/summarizer.d.ts +0 -2
  34. package/dist/summarizer.d.ts.map +1 -1
  35. package/dist/summarizer.js +1 -12
  36. package/dist/summarizer.js.map +1 -1
  37. package/dist/summarizerHeuristics.d.ts +26 -4
  38. package/dist/summarizerHeuristics.d.ts.map +1 -1
  39. package/dist/summarizerHeuristics.js +95 -18
  40. package/dist/summarizerHeuristics.js.map +1 -1
  41. package/dist/summarizerTypes.d.ts +30 -10
  42. package/dist/summarizerTypes.d.ts.map +1 -1
  43. package/dist/summarizerTypes.js.map +1 -1
  44. package/dist/summaryCollection.js +1 -1
  45. package/dist/summaryCollection.js.map +1 -1
  46. package/dist/summaryFormat.d.ts +0 -5
  47. package/dist/summaryFormat.d.ts.map +1 -1
  48. package/dist/summaryFormat.js.map +1 -1
  49. package/dist/summaryGenerator.d.ts +1 -0
  50. package/dist/summaryGenerator.d.ts.map +1 -1
  51. package/dist/summaryGenerator.js +11 -9
  52. package/dist/summaryGenerator.js.map +1 -1
  53. package/lib/blobManager.d.ts +81 -25
  54. package/lib/blobManager.d.ts.map +1 -1
  55. package/lib/blobManager.js +302 -101
  56. package/lib/blobManager.js.map +1 -1
  57. package/lib/containerRuntime.d.ts +46 -3
  58. package/lib/containerRuntime.d.ts.map +1 -1
  59. package/lib/containerRuntime.js +99 -80
  60. package/lib/containerRuntime.js.map +1 -1
  61. package/lib/dataStore.d.ts +1 -1
  62. package/lib/dataStore.d.ts.map +1 -1
  63. package/lib/dataStore.js +32 -26
  64. package/lib/dataStore.js.map +1 -1
  65. package/lib/dataStoreContext.d.ts +3 -4
  66. package/lib/dataStoreContext.d.ts.map +1 -1
  67. package/lib/dataStoreContext.js +17 -24
  68. package/lib/dataStoreContext.js.map +1 -1
  69. package/lib/dataStores.d.ts +5 -2
  70. package/lib/dataStores.d.ts.map +1 -1
  71. package/lib/dataStores.js +11 -3
  72. package/lib/dataStores.js.map +1 -1
  73. package/lib/opProperties.d.ts +7 -0
  74. package/lib/opProperties.d.ts.map +1 -0
  75. package/lib/opProperties.js +15 -0
  76. package/lib/opProperties.js.map +1 -0
  77. package/lib/packageVersion.d.ts +1 -1
  78. package/lib/packageVersion.d.ts.map +1 -1
  79. package/lib/packageVersion.js +1 -1
  80. package/lib/packageVersion.js.map +1 -1
  81. package/lib/runningSummarizer.d.ts +14 -4
  82. package/lib/runningSummarizer.d.ts.map +1 -1
  83. package/lib/runningSummarizer.js +68 -26
  84. package/lib/runningSummarizer.js.map +1 -1
  85. package/lib/summarizer.d.ts +0 -2
  86. package/lib/summarizer.d.ts.map +1 -1
  87. package/lib/summarizer.js +1 -12
  88. package/lib/summarizer.js.map +1 -1
  89. package/lib/summarizerHeuristics.d.ts +26 -4
  90. package/lib/summarizerHeuristics.d.ts.map +1 -1
  91. package/lib/summarizerHeuristics.js +95 -18
  92. package/lib/summarizerHeuristics.js.map +1 -1
  93. package/lib/summarizerTypes.d.ts +30 -10
  94. package/lib/summarizerTypes.d.ts.map +1 -1
  95. package/lib/summarizerTypes.js.map +1 -1
  96. package/lib/summaryCollection.js +1 -1
  97. package/lib/summaryCollection.js.map +1 -1
  98. package/lib/summaryFormat.d.ts +0 -5
  99. package/lib/summaryFormat.d.ts.map +1 -1
  100. package/lib/summaryFormat.js.map +1 -1
  101. package/lib/summaryGenerator.d.ts +1 -0
  102. package/lib/summaryGenerator.d.ts.map +1 -1
  103. package/lib/summaryGenerator.js +11 -9
  104. package/lib/summaryGenerator.js.map +1 -1
  105. package/package.json +45 -20
  106. package/src/blobManager.ts +360 -119
  107. package/src/containerRuntime.ts +165 -89
  108. package/src/dataStore.ts +53 -38
  109. package/src/dataStoreContext.ts +16 -23
  110. package/src/dataStores.ts +14 -3
  111. package/src/opProperties.ts +19 -0
  112. package/src/packageVersion.ts +1 -1
  113. package/src/runningSummarizer.ts +75 -22
  114. package/src/summarizer.ts +1 -18
  115. package/src/summarizerHeuristics.ts +133 -19
  116. package/src/summarizerTypes.ts +37 -10
  117. package/src/summaryCollection.ts +1 -1
  118. package/src/summaryFormat.ts +0 -6
  119. package/src/summaryGenerator.ts +40 -22
  120. package/dist/opTelemetry.d.ts +0 -22
  121. package/dist/opTelemetry.d.ts.map +0 -1
  122. package/dist/opTelemetry.js +0 -59
  123. package/dist/opTelemetry.js.map +0 -1
  124. package/lib/opTelemetry.d.ts +0 -22
  125. package/lib/opTelemetry.d.ts.map +0 -1
  126. package/lib/opTelemetry.js +0 -55
  127. package/lib/opTelemetry.js.map +0 -1
  128. package/src/opTelemetry.ts +0 -71
@@ -43,7 +43,6 @@ import {
43
43
  TaggedLoggerAdapter,
44
44
  MonitoringContext,
45
45
  loggerToMonitoringContext,
46
- TelemetryDataTag,
47
46
  } from "@fluidframework/telemetry-utils";
48
47
  import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
49
48
  import { readAndParse, isUnpackedRuntimeMessage } from "@fluidframework/driver-utils";
@@ -160,7 +159,6 @@ import {
160
159
  } from "./dataStore";
161
160
  import { BindBatchTracker } from "./batchTracker";
162
161
  import { ISerializedBaseSnapshotBlobs, SerializedSnapshotStorage } from "./serializedSnapshotStorage";
163
- import { OpTracker } from "./opTelemetry";
164
162
 
165
163
  export enum ContainerMessageType {
166
164
  // An op to be delivered to store
@@ -224,12 +222,14 @@ export interface ISummaryBaseConfiguration {
224
222
  export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
225
223
  state: "enabled";
226
224
  /**
227
- * Defines the maximum allowed time in between summarizations.
225
+ * @deprecated - please move all implementation to minIdleTime and maxIdleTime
228
226
  */
229
227
  idleTime: number;
230
228
  /**
231
- * Defines the maximum allowed time, since the last received Ack, before running the summary
229
+ * Defines the maximum allowed time, since the last received Ack, before running the summary
232
230
  * with reason maxTime.
231
+ * For example, say we receive ops one by one just before the idle time is triggered.
232
+ * In this case, we still want to run a summary since it's been a while since the last summary.
233
233
  */
234
234
  maxTime: number;
235
235
  /**
@@ -242,6 +242,34 @@ export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfigurati
242
242
  * before running the last summary.
243
243
  */
244
244
  minOpsForLastSummaryAttempt: number;
245
+ /**
246
+ * Defines the lower boundary for the allowed time in between summarizations.
247
+ * Pairs with maxIdleTime to form a range.
248
+ * For example, if we only receive 1 op, we don't want to have the same idle time as say 100 ops.
249
+ * Based on the boundaries we set in minIdleTime and maxIdleTime, the idle time will change
250
+ * linearly depending on the number of ops we receive.
251
+ */
252
+ minIdleTime: number;
253
+ /**
254
+ * Defines the upper boundary for the allowed time in between summarizations.
255
+ * Pairs with minIdleTime to form a range.
256
+ * For example, if we only receive 1 op, we don't want to have the same idle time as say 100 ops.
257
+ * Based on the boundaries we set in minIdleTime and maxIdleTime, the idle time will change
258
+ * linearly depending on the number of ops we receive.
259
+ */
260
+ maxIdleTime: number;
261
+ /**
262
+ * Runtime op weight to use in heuristic summarizing.
263
+ * This number is a multiplier on the number of runtime ops we process when running summarize heuristics.
264
+ * For example: (multiplier) * (number of runtime ops) = weighted number of runtime ops
265
+ */
266
+ runtimeOpWeight: number;
267
+ /**
268
+ * Non-runtime op weight to use in heuristic summarizing
269
+ * This number is a multiplier on the number of non-runtime ops we process when running summarize heuristics.
270
+ * For example: (multiplier) * (number of non-runtime ops) = weighted number of non-runtime ops
271
+ */
272
+ nonRuntimeOpWeight: number;
245
273
  }
246
274
 
247
275
  export interface ISummaryConfigurationDisableSummarizer {
@@ -260,21 +288,29 @@ export type ISummaryConfiguration =
260
288
  export const DefaultSummaryConfiguration: ISummaryConfiguration = {
261
289
  state: "enabled",
262
290
 
263
- idleTime: 5000 * 3,
291
+ idleTime: 15 * 1000, // 15 secs.
292
+
293
+ minIdleTime: 0,
264
294
 
265
- maxTime: 5000 * 12,
295
+ maxIdleTime: 30 * 1000, // 30 secs.
266
296
 
267
- maxOps: 100, // Summarize if 100 ops received since last snapshot.
297
+ maxTime: 60 * 1000, // 1 min.
298
+
299
+ maxOps: 100, // Summarize if 100 weighted ops received since last snapshot.
268
300
 
269
301
  minOpsForLastSummaryAttempt: 10,
270
302
 
271
- maxAckWaitTime: 6 * 10 * 1000, // 6 min.
303
+ maxAckWaitTime: 10 * 60 * 1000, // 10 mins.
272
304
 
273
305
  maxOpsSinceLastSummary: 7000,
274
306
 
275
- initialSummarizerDelayMs: 5000, // 5 secs.
307
+ initialSummarizerDelayMs: 5 * 1000, // 5 secs.
276
308
 
277
309
  summarizerClientElection: false,
310
+
311
+ nonRuntimeOpWeight: 0.1,
312
+
313
+ runtimeOpWeight: 1.0,
278
314
  };
279
315
 
280
316
  export interface IGCRuntimeOptions {
@@ -318,9 +354,12 @@ export interface ISummaryRuntimeOptions {
318
354
  /** Override summary configurations set by the server. */
319
355
  summaryConfigOverrides?: ISummaryConfiguration;
320
356
 
321
- // Flag that disables putting channels in isolated subtrees for each data store
322
- // and the root node when generating a summary if set to true.
323
- // Defaults to FALSE (enabled) for now.
357
+ /**
358
+ * @deprecated - this option will not be supported on the next versions.
359
+ * Flag that disables putting channels in isolated subtrees for each data store
360
+ * and the root node when generating a summary if set to true.
361
+ * Defaults to FALSE (enabled) for now.
362
+ */
324
363
  disableIsolatedChannels?: boolean;
325
364
 
326
365
  /**
@@ -466,11 +505,6 @@ const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
466
505
  // to not reach the 1MB limits in socket.io and Kafka.
467
506
  const defaultMaxOpSizeInBytes = 768000;
468
507
 
469
- // By default, the size of the contents for the incoming ops is tracked.
470
- // However, in certain situations, this may incur a performance hit.
471
- // The feature-gate below can be used to disable this feature.
472
- const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
473
-
474
508
  const defaultFlushMode = FlushMode.TurnBased;
475
509
 
476
510
  export enum RuntimeMessage {
@@ -1036,6 +1070,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1036
1070
 
1037
1071
  private consecutiveReconnects = 0;
1038
1072
 
1073
+ /**
1074
+ * Used to delay transition to "connected" state while we upload
1075
+ * attachment blobs that were added while disconnected
1076
+ */
1077
+ private delayConnectClientId?: string;
1078
+
1039
1079
  public get connected(): boolean {
1040
1080
  return this._connected;
1041
1081
  }
@@ -1160,7 +1200,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1160
1200
  * a summary is generated.
1161
1201
  */
1162
1202
  private nextSummaryNumber: number;
1163
- private readonly opTracker: OpTracker;
1164
1203
 
1165
1204
  private constructor(
1166
1205
  private readonly context: IContainerContext,
@@ -1197,6 +1236,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1197
1236
  this.mc = loggerToMonitoringContext(
1198
1237
  ChildLogger.create(this.logger, "ContainerRuntime"));
1199
1238
 
1239
+ if (this.summaryConfiguration.state === "enabled") {
1240
+ this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
1241
+ }
1242
+
1200
1243
  this.summariesDisabled = this.isSummariesDisabled();
1201
1244
  this.heuristicsDisabled = this.isHeuristicsDisabled();
1202
1245
  this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
@@ -1288,10 +1331,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1288
1331
  this.handleContext,
1289
1332
  blobManagerSnapshot,
1290
1333
  () => this.storage,
1291
- (blobId: string) => this.submit(ContainerMessageType.BlobAttach, undefined, undefined, { blobId }),
1334
+ (blobId, localId) => this.submit(
1335
+ ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId }),
1292
1336
  (blobPath: string) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
1293
1337
  this,
1294
- this.logger,
1295
1338
  );
1296
1339
 
1297
1340
  this.scheduleManager = new ScheduleManager(
@@ -1442,9 +1485,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1442
1485
  createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
1443
1486
  createContainerTimestamp: metadata?.createContainerTimestamp,
1444
1487
  };
1445
- // back-compat 0.59.3000 - Older document may either write summaryCount or not write it at all. If it does
1446
- // not write it, initialize summaryNumber to 0.
1447
- loadSummaryNumber = metadata?.summaryNumber ?? metadata?.summaryCount ?? 0;
1488
+ // summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
1489
+ // the count is reset to 0.
1490
+ loadSummaryNumber = metadata?.summaryNumber ?? 0;
1448
1491
  } else {
1449
1492
  this.createContainerMetadata = {
1450
1493
  createContainerRuntimeVersion: pkgVersion,
@@ -1466,7 +1509,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1466
1509
 
1467
1510
  ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
1468
1511
  BindBatchTracker(this, this.logger);
1469
- this.opTracker = new OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
1470
1512
  }
1471
1513
 
1472
1514
  public dispose(error?: Error): void {
@@ -1546,12 +1588,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1546
1588
  }
1547
1589
 
1548
1590
  if (id === BlobManager.basePath && requestParser.isLeaf(2)) {
1549
- const handle = await this.blobManager.getBlob(requestParser.pathParts[1]);
1550
- if (handle) {
1591
+ const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
1592
+ if (blob) {
1551
1593
  return {
1552
1594
  status: 200,
1553
1595
  mimeType: "fluid/object",
1554
- value: handle.get(),
1596
+ value: blob,
1555
1597
  };
1556
1598
  } else {
1557
1599
  return create404Response(request);
@@ -1573,7 +1615,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1573
1615
  }
1574
1616
 
1575
1617
  private internalId(maybeAlias: string): string {
1576
- return this.dataStores.aliases().get(maybeAlias) ?? maybeAlias;
1618
+ return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
1577
1619
  }
1578
1620
 
1579
1621
  private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
@@ -1581,6 +1623,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1581
1623
  ? request.headers?.[RuntimeHeaders.wait]
1582
1624
  : true;
1583
1625
 
1626
+ await this.dataStores.waitIfPendingAlias(id);
1584
1627
  const internalId = this.internalId(id);
1585
1628
  const dataStoreContext = await this.dataStores.getDataStore(internalId, wait);
1586
1629
 
@@ -1620,8 +1663,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1620
1663
  private addMetadataToSummary(summaryTree: ISummaryTreeWithStats) {
1621
1664
  const metadata: IContainerRuntimeMetadata = {
1622
1665
  ...this.createContainerMetadata,
1623
- // back-compat 0.59.3000: This is renamed to summaryNumber. Can be removed when 0.59.3000 saturates.
1624
- summaryCount: this.nextSummaryNumber,
1625
1666
  // Increment the summary number for the next summary that will be generated.
1626
1667
  summaryNumber: this.nextSummaryNumber++,
1627
1668
  summaryFormatVersion: 1,
@@ -1647,7 +1688,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1647
1688
  addBlobToSummary(summaryTree, chunksBlobName, content);
1648
1689
  }
1649
1690
 
1650
- const dataStoreAliases = this.dataStores.aliases();
1691
+ const dataStoreAliases = this.dataStores.aliases;
1651
1692
  if (dataStoreAliases.size > 0) {
1652
1693
  addBlobToSummary(summaryTree, aliasBlobName, JSON.stringify([...dataStoreAliases]));
1653
1694
  }
@@ -1757,6 +1798,38 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1757
1798
  }
1758
1799
 
1759
1800
  public setConnectionState(connected: boolean, clientId?: string) {
1801
+ if (connected === false && this.delayConnectClientId !== undefined) {
1802
+ this.delayConnectClientId = undefined;
1803
+ this.mc.logger.sendTelemetryEvent({
1804
+ eventName: "UnsuccessfulConnectedTransition",
1805
+ });
1806
+ // Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
1807
+ return;
1808
+ }
1809
+
1810
+ // If attachment blobs were added while disconnected, we need to delay
1811
+ // propagation of the "connected" event until we have uploaded them to
1812
+ // ensure we don't submit ops referencing a blob that has not been uploaded
1813
+ const connecting = connected && !this._connected && !this.deltaManager.readOnlyInfo.readonly;
1814
+ if (connecting && this.blobManager.hasPendingOfflineUploads) {
1815
+ assert(!this.delayConnectClientId, "Connect event delay must be canceled before subsequent connect event");
1816
+ assert(!!clientId, "Must have clientId when connecting");
1817
+ this.delayConnectClientId = clientId;
1818
+ this.blobManager.onConnected().then(() => {
1819
+ // make sure we didn't reconnect before the promise resolved
1820
+ if (this.delayConnectClientId === clientId && !this.disposed) {
1821
+ this.delayConnectClientId = undefined;
1822
+ this.setConnectionStateCore(connected, clientId);
1823
+ }
1824
+ }, (error) => this.closeFn(error));
1825
+ return;
1826
+ }
1827
+
1828
+ this.setConnectionStateCore(connected, clientId);
1829
+ }
1830
+
1831
+ private setConnectionStateCore(connected: boolean, clientId?: string) {
1832
+ assert(!this.delayConnectClientId, "connect event delay must be cleared before propagating connect event");
1760
1833
  this.verifyNotClosed();
1761
1834
 
1762
1835
  // There might be no change of state due to Container calling this API after loading runtime.
@@ -1854,8 +1927,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1854
1927
  this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
1855
1928
  break;
1856
1929
  case ContainerMessageType.BlobAttach:
1857
- assert(message?.metadata?.blobId, 0x12a /* "Missing blob id on metadata" */);
1858
- this.blobManager.processBlobAttachOp(message.metadata.blobId, local);
1930
+ this.blobManager.processBlobAttachOp(message, local);
1859
1931
  break;
1860
1932
  default:
1861
1933
  }
@@ -1937,6 +2009,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
1937
2009
  }
1938
2010
 
1939
2011
  public async getRootDataStore(id: string, wait = true): Promise<IFluidRouter> {
2012
+ return this.getRootDataStoreChannel(id, wait);
2013
+ }
2014
+
2015
+ private async getRootDataStoreChannel(id: string, wait = true): Promise<IFluidDataStoreChannel> {
2016
+ await this.dataStores.waitIfPendingAlias(id);
1940
2017
  const internalId = this.internalId(id);
1941
2018
  const context = await this.dataStores.getDataStore(internalId, wait);
1942
2019
  assert(await context.isRoot(), 0x12b /* "did not get root data store" */);
@@ -2058,13 +2135,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2058
2135
  */
2059
2136
  private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
2060
2137
  const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
2061
- // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
2062
- // older versions, we still have to call bindToContext.
2063
- if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
2064
- fluidDataStore.makeVisibleAndAttachGraph();
2065
- } else {
2066
- fluidDataStore.bindToContext();
2067
- }
2138
+ fluidDataStore.makeVisibleAndAttachGraph();
2068
2139
  return fluidDataStore;
2069
2140
  }
2070
2141
 
@@ -2093,27 +2164,37 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2093
2164
  */
2094
2165
  private async createAndAliasDataStore(pkg: string | string[], alias: string, props?: any): Promise<IDataStore> {
2095
2166
  const internalId = uuid();
2096
- const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
2097
- const aliasedDataStore = channelToDataStore(dataStore, internalId, this, this.dataStores, this.mc.logger);
2098
- const result = await aliasedDataStore.trySetAlias(alias);
2099
- if (result !== "Success") {
2100
- throw new GenericError(
2101
- "dataStoreAliasFailure",
2102
- undefined /* error */,
2103
- {
2104
- alias: {
2105
- value: alias,
2106
- tag: TelemetryDataTag.UserData,
2107
- },
2108
- internalId: {
2109
- value: internalId,
2110
- tag: TelemetryDataTag.PackageData,
2111
- },
2112
- aliasResult: result,
2113
- });
2114
- }
2115
2167
 
2116
- return aliasedDataStore;
2168
+ try {
2169
+ // A similar call may have been initiated by the same client, so we should try to get
2170
+ // a possible existing aliased datastore first.
2171
+ const existingDataStore = await this.getRootDataStoreChannel(alias, /* wait */ false);
2172
+ return channelToDataStore(
2173
+ existingDataStore,
2174
+ internalId,
2175
+ this,
2176
+ this.dataStores,
2177
+ this.mc.logger,
2178
+ true, // AlreadyAliased. This will block further alias attempts for the datastore
2179
+ );
2180
+ } catch (err) {
2181
+ const newChannel = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
2182
+ const newDataStore = channelToDataStore(newChannel, internalId, this, this.dataStores, this.mc.logger);
2183
+ const aliasResult = await newDataStore.trySetAlias(alias);
2184
+ if (aliasResult === "Success") {
2185
+ return newDataStore;
2186
+ }
2187
+
2188
+ const existingDataStore = await this.getRootDataStoreChannel(alias, /* wait */ false);
2189
+ return channelToDataStore(
2190
+ existingDataStore,
2191
+ internalId,
2192
+ this,
2193
+ this.dataStores,
2194
+ this.mc.logger,
2195
+ true, // AlreadyAliased. This will block further alias attempts for the datastore
2196
+ );
2197
+ }
2117
2198
  }
2118
2199
 
2119
2200
  public createDetachedRootDataStore(
@@ -2144,13 +2225,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2144
2225
  const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
2145
2226
  Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
2146
2227
  if (isRoot) {
2147
- // back-compat 0.59.1000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel.
2148
- // For older versions, we still have to call bindToContext.
2149
- if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
2150
- fluidDataStore.makeVisibleAndAttachGraph();
2151
- } else {
2152
- fluidDataStore.bindToContext();
2153
- }
2228
+ fluidDataStore.makeVisibleAndAttachGraph();
2154
2229
  this.logger.sendTelemetryEvent({
2155
2230
  eventName: "Root datastore with props",
2156
2231
  hasProps: props !== undefined,
@@ -2560,15 +2635,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2560
2635
  const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
2561
2636
  const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
2562
2637
  const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
2563
-
2564
- // We should be here is we haven't processed be here. If we are of if the last message's sequence number
2565
- // doesn't match the last processed sequence number, log an error.
2566
- if (summaryRefSeqNum !== this.deltaManager.lastMessage?.sequenceNumber) {
2567
- summaryNumberLogger.sendErrorEvent({
2568
- eventName: "LastSequenceMismatch",
2569
- error: message,
2570
- });
2571
- }
2638
+ const lastAck = this.summaryCollection.latestAck;
2572
2639
 
2573
2640
  this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
2574
2641
 
@@ -2598,6 +2665,16 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2598
2665
  error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
2599
2666
  };
2600
2667
  }
2668
+ assert(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber,
2669
+ "it's one and the same thing");
2670
+
2671
+ if (lastAck !== this.summaryCollection.latestAck) {
2672
+ return {
2673
+ continue: false,
2674
+ // eslint-disable-next-line max-len
2675
+ error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
2676
+ };
2677
+ }
2601
2678
  return { continue: true };
2602
2679
  };
2603
2680
 
@@ -2654,8 +2731,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2654
2731
  gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
2655
2732
  gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
2656
2733
  gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
2657
- opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
2658
- nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
2659
2734
  summaryNumber,
2660
2735
  ...partialStats,
2661
2736
  };
@@ -2673,7 +2748,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2673
2748
  return { stage: "generate", ...generateSummaryData, error: continueResult.error };
2674
2749
  }
2675
2750
 
2676
- const lastAck = this.summaryCollection.latestAck;
2677
2751
  const summaryContext: ISummaryContext =
2678
2752
  lastAck === undefined
2679
2753
  ? {
@@ -2728,7 +2802,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2728
2802
  } as const;
2729
2803
 
2730
2804
  this.summarizerNode.completeSummary(handle);
2731
- this.opTracker.reset();
2732
2805
  return submitData;
2733
2806
  } finally {
2734
2807
  // Cleanup wip summary in case of failure
@@ -2892,14 +2965,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
2892
2965
  "OpTooLarge",
2893
2966
  /* error */ undefined,
2894
2967
  {
2895
- length: {
2896
- value: serializedContent.length,
2897
- tag: TelemetryDataTag.PackageData,
2898
- },
2899
- limit: {
2900
- value: this._maxOpSizeInBytes,
2901
- tag: TelemetryDataTag.PackageData,
2902
- },
2968
+ length: serializedContent.length,
2969
+ limit: this._maxOpSizeInBytes,
2903
2970
  }));
2904
2971
  return -1;
2905
2972
  }
@@ -3005,7 +3072,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
3005
3072
  case ContainerMessageType.ChunkedOp:
3006
3073
  throw new Error(`chunkedOp not expected here`);
3007
3074
  case ContainerMessageType.BlobAttach:
3008
- this.submit(type, content, localOpMetadata, opMetadata);
3075
+ this.blobManager.reSubmit(opMetadata);
3009
3076
  break;
3010
3077
  case ContainerMessageType.Rejoin:
3011
3078
  this.submit(type, content);
@@ -3218,6 +3285,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
3218
3285
  // don't have any more saved ops
3219
3286
  await this.pendingStateManager.applyStashedOpsAt();
3220
3287
  }
3288
+
3289
+ private validateSummaryHeuristicConfiguration(configuration: ISummaryConfigurationHeuristics) {
3290
+ // eslint-disable-next-line no-restricted-syntax
3291
+ for (const prop in configuration) {
3292
+ if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
3293
+ throw new UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
3294
+ }
3295
+ }
3296
+ }
3221
3297
  }
3222
3298
 
3223
3299
  /**
package/src/dataStore.ts CHANGED
@@ -43,7 +43,8 @@ export const channelToDataStore = (
43
43
  runtime: ContainerRuntime,
44
44
  datastores: DataStores,
45
45
  logger: ITelemetryLogger,
46
- ): IDataStore => new DataStore(fluidDataStoreChannel, internalId, runtime, datastores, logger);
46
+ alreadyAliased: boolean = false,
47
+ ): IDataStore => new DataStore(fluidDataStoreChannel, internalId, runtime, datastores, logger, alreadyAliased);
47
48
 
48
49
  enum AliasState {
49
50
  Aliased = "Aliased",
@@ -54,6 +55,7 @@ enum AliasState {
54
55
  class DataStore implements IDataStore {
55
56
  private aliasState: AliasState = AliasState.None;
56
57
  private alias: string | undefined;
58
+ private readonly pendingAliases: Map<string, Promise<AliasResult>>;
57
59
  private aliasResult: Promise<AliasResult> | undefined;
58
60
 
59
61
  async trySetAlias(alias: string): Promise<AliasResult> {
@@ -75,14 +77,25 @@ class DataStore implements IDataStore {
75
77
  case AliasState.Aliased:
76
78
  return this.alias === alias ? "Success" : "AlreadyAliased";
77
79
 
78
- // There is no current or past alias operation for this datastore,
79
- // it is safe to continue execution
80
- case AliasState.None: break;
80
+ case AliasState.None: {
81
+ const existingAlias = this.pendingAliases.get(alias);
82
+ if (existingAlias !== undefined) {
83
+ // There is already another datastore which will be aliased
84
+ // to the same name
85
+ return "Conflict";
86
+ }
87
+
88
+ // There is no current or past alias operation for this datastore,
89
+ // or for this alias, so it is safe to continue execution
90
+ break;
91
+ }
92
+
81
93
  default: unreachableCase(this.aliasState);
82
94
  }
83
95
 
84
96
  this.aliasState = AliasState.Aliasing;
85
97
  this.aliasResult = this.trySetAliasInternal(alias);
98
+ this.pendingAliases.set(alias, this.aliasResult);
86
99
  return this.aliasResult;
87
100
  }
88
101
 
@@ -92,13 +105,7 @@ class DataStore implements IDataStore {
92
105
  alias,
93
106
  };
94
107
 
95
- // back-compat 0.58.2000 - makeVisibleAndAttachGraph was added in this version to IFluidDataStoreChannel. For
96
- // older versions, we still have to call bindToContext.
97
- if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
98
- this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
99
- } else {
100
- this.fluidDataStoreChannel.bindToContext();
101
- }
108
+ this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
102
109
 
103
110
  if (this.runtime.attachState === AttachState.Detached) {
104
111
  const localResult = this.datastores.processAliasMessageCore(message);
@@ -108,34 +115,37 @@ class DataStore implements IDataStore {
108
115
  return localResult ? "Success" : "Conflict";
109
116
  }
110
117
 
111
- const aliased = await this.ackBasedPromise<boolean>((resolve) => {
112
- this.runtime.submitDataStoreAliasOp(message, resolve);
113
- }).then((succeeded) => {
114
- // Explicitly Lock-out future attempts of aliasing,
115
- // regardless of result
116
- this.aliasState = AliasState.Aliased;
117
- if (succeeded) {
118
- this.alias = alias;
119
- }
120
-
121
- return succeeded;
122
- }).catch((error) => {
123
- this.logger.sendErrorEvent({
124
- eventName: "AliasingException",
125
- alias: {
126
- value: alias,
127
- tag: TelemetryDataTag.UserData,
128
- },
129
- internalId: {
130
- value: this.internalId,
131
- tag: TelemetryDataTag.PackageData,
132
- },
133
- }, error);
118
+ const aliased = await this
119
+ .ackBasedPromise<boolean>((resolve) => {
120
+ this.runtime.submitDataStoreAliasOp(message, resolve);
121
+ })
122
+ .catch((error) => {
123
+ this.logger.sendErrorEvent({
124
+ eventName: "AliasingException",
125
+ alias: {
126
+ value: alias,
127
+ tag: TelemetryDataTag.UserData,
128
+ },
129
+ internalId: {
130
+ value: this.internalId,
131
+ tag: TelemetryDataTag.CodeArtifact,
132
+ },
133
+ }, error);
134
+
135
+ return false;
136
+ }).finally(() => {
137
+ this.pendingAliases.delete(alias);
138
+ });
139
+
140
+ if (!aliased) {
134
141
  this.aliasState = AliasState.None;
135
- return false;
136
- });
142
+ this.aliasResult = undefined;
143
+ return "Conflict";
144
+ }
137
145
 
138
- return aliased ? "Success" : "Conflict";
146
+ this.alias = alias;
147
+ this.aliasState = AliasState.Aliased;
148
+ return "Success";
139
149
  }
140
150
 
141
151
  async request(request: IRequest): Promise<IResponse> {
@@ -148,7 +158,12 @@ class DataStore implements IDataStore {
148
158
  private readonly runtime: ContainerRuntime,
149
159
  private readonly datastores: DataStores,
150
160
  private readonly logger: ITelemetryLogger,
151
- ) { }
161
+ alreadyAliased: boolean,
162
+ ) {
163
+ this.pendingAliases = datastores.pendingAliases;
164
+ this.aliasState = alreadyAliased ? AliasState.Aliased : AliasState.None;
165
+ }
166
+
152
167
  public get IFluidRouter() { return this.fluidDataStoreChannel; }
153
168
 
154
169
  private async ackBasedPromise<T>(