@fluidframework/container-runtime 1.2.2 → 2.0.0-internal.1.0.0.81589
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/blobManager.d.ts +81 -25
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +301 -100
- package/dist/blobManager.js.map +1 -1
- package/dist/containerRuntime.d.ts +46 -3
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +98 -79
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +1 -1
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +32 -26
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +3 -4
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -23
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +5 -2
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +11 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/opProperties.d.ts +7 -0
- package/dist/opProperties.d.ts.map +1 -0
- package/dist/opProperties.js +19 -0
- package/dist/opProperties.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/runningSummarizer.d.ts +14 -4
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +68 -26
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts +0 -2
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +1 -12
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +26 -4
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +95 -18
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +30 -10
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryCollection.js +1 -1
- package/dist/summaryCollection.js.map +1 -1
- package/dist/summaryFormat.d.ts +0 -5
- package/dist/summaryFormat.d.ts.map +1 -1
- package/dist/summaryFormat.js.map +1 -1
- package/dist/summaryGenerator.d.ts +1 -0
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +11 -9
- package/dist/summaryGenerator.js.map +1 -1
- package/lib/blobManager.d.ts +81 -25
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +302 -101
- package/lib/blobManager.js.map +1 -1
- package/lib/containerRuntime.d.ts +46 -3
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +99 -80
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +1 -1
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +32 -26
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +3 -4
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +17 -24
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +5 -2
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +11 -3
- package/lib/dataStores.js.map +1 -1
- package/lib/opProperties.d.ts +7 -0
- package/lib/opProperties.d.ts.map +1 -0
- package/lib/opProperties.js +15 -0
- package/lib/opProperties.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/runningSummarizer.d.ts +14 -4
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +68 -26
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts +0 -2
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +1 -12
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +26 -4
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +95 -18
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +30 -10
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryCollection.js +1 -1
- package/lib/summaryCollection.js.map +1 -1
- package/lib/summaryFormat.d.ts +0 -5
- package/lib/summaryFormat.d.ts.map +1 -1
- package/lib/summaryFormat.js.map +1 -1
- package/lib/summaryGenerator.d.ts +1 -0
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +11 -9
- package/lib/summaryGenerator.js.map +1 -1
- package/package.json +45 -20
- package/src/blobManager.ts +360 -119
- package/src/containerRuntime.ts +165 -89
- package/src/dataStore.ts +53 -38
- package/src/dataStoreContext.ts +16 -23
- package/src/dataStores.ts +14 -3
- package/src/opProperties.ts +19 -0
- package/src/packageVersion.ts +1 -1
- package/src/runningSummarizer.ts +75 -22
- package/src/summarizer.ts +1 -18
- package/src/summarizerHeuristics.ts +133 -19
- package/src/summarizerTypes.ts +37 -10
- package/src/summaryCollection.ts +1 -1
- package/src/summaryFormat.ts +0 -6
- package/src/summaryGenerator.ts +40 -22
- package/dist/opTelemetry.d.ts +0 -22
- package/dist/opTelemetry.d.ts.map +0 -1
- package/dist/opTelemetry.js +0 -59
- package/dist/opTelemetry.js.map +0 -1
- package/lib/opTelemetry.d.ts +0 -22
- package/lib/opTelemetry.d.ts.map +0 -1
- package/lib/opTelemetry.js +0 -55
- package/lib/opTelemetry.js.map +0 -1
- package/src/opTelemetry.ts +0 -71
package/src/containerRuntime.ts
CHANGED
|
@@ -43,7 +43,6 @@ import {
|
|
|
43
43
|
TaggedLoggerAdapter,
|
|
44
44
|
MonitoringContext,
|
|
45
45
|
loggerToMonitoringContext,
|
|
46
|
-
TelemetryDataTag,
|
|
47
46
|
} from "@fluidframework/telemetry-utils";
|
|
48
47
|
import { DriverHeader, IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
49
48
|
import { readAndParse, isUnpackedRuntimeMessage } from "@fluidframework/driver-utils";
|
|
@@ -160,7 +159,6 @@ import {
|
|
|
160
159
|
} from "./dataStore";
|
|
161
160
|
import { BindBatchTracker } from "./batchTracker";
|
|
162
161
|
import { ISerializedBaseSnapshotBlobs, SerializedSnapshotStorage } from "./serializedSnapshotStorage";
|
|
163
|
-
import { OpTracker } from "./opTelemetry";
|
|
164
162
|
|
|
165
163
|
export enum ContainerMessageType {
|
|
166
164
|
// An op to be delivered to store
|
|
@@ -224,12 +222,14 @@ export interface ISummaryBaseConfiguration {
|
|
|
224
222
|
export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfiguration {
|
|
225
223
|
state: "enabled";
|
|
226
224
|
/**
|
|
227
|
-
*
|
|
225
|
+
* @deprecated - please move all implementation to minIdleTime and maxIdleTime
|
|
228
226
|
*/
|
|
229
227
|
idleTime: number;
|
|
230
228
|
/**
|
|
231
|
-
* Defines the maximum allowed time, since the last received Ack,
|
|
229
|
+
* Defines the maximum allowed time, since the last received Ack, before running the summary
|
|
232
230
|
* with reason maxTime.
|
|
231
|
+
* For example, say we receive ops one by one just before the idle time is triggered.
|
|
232
|
+
* In this case, we still want to run a summary since it's been a while since the last summary.
|
|
233
233
|
*/
|
|
234
234
|
maxTime: number;
|
|
235
235
|
/**
|
|
@@ -242,6 +242,34 @@ export interface ISummaryConfigurationHeuristics extends ISummaryBaseConfigurati
|
|
|
242
242
|
* before running the last summary.
|
|
243
243
|
*/
|
|
244
244
|
minOpsForLastSummaryAttempt: number;
|
|
245
|
+
/**
|
|
246
|
+
* Defines the lower boundary for the allowed time in between summarizations.
|
|
247
|
+
* Pairs with maxIdleTime to form a range.
|
|
248
|
+
* For example, if we only receive 1 op, we don't want to have the same idle time as say 100 ops.
|
|
249
|
+
* Based on the boundaries we set in minIdleTime and maxIdleTime, the idle time will change
|
|
250
|
+
* linearly depending on the number of ops we receive.
|
|
251
|
+
*/
|
|
252
|
+
minIdleTime: number;
|
|
253
|
+
/**
|
|
254
|
+
* Defines the upper boundary for the allowed time in between summarizations.
|
|
255
|
+
* Pairs with minIdleTime to form a range.
|
|
256
|
+
* For example, if we only receive 1 op, we don't want to have the same idle time as say 100 ops.
|
|
257
|
+
* Based on the boundaries we set in minIdleTime and maxIdleTime, the idle time will change
|
|
258
|
+
* linearly depending on the number of ops we receive.
|
|
259
|
+
*/
|
|
260
|
+
maxIdleTime: number;
|
|
261
|
+
/**
|
|
262
|
+
* Runtime op weight to use in heuristic summarizing.
|
|
263
|
+
* This number is a multiplier on the number of runtime ops we process when running summarize heuristics.
|
|
264
|
+
* For example: (multiplier) * (number of runtime ops) = weighted number of runtime ops
|
|
265
|
+
*/
|
|
266
|
+
runtimeOpWeight: number;
|
|
267
|
+
/**
|
|
268
|
+
* Non-runtime op weight to use in heuristic summarizing
|
|
269
|
+
* This number is a multiplier on the number of non-runtime ops we process when running summarize heuristics.
|
|
270
|
+
* For example: (multiplier) * (number of non-runtime ops) = weighted number of non-runtime ops
|
|
271
|
+
*/
|
|
272
|
+
nonRuntimeOpWeight: number;
|
|
245
273
|
}
|
|
246
274
|
|
|
247
275
|
export interface ISummaryConfigurationDisableSummarizer {
|
|
@@ -260,21 +288,29 @@ export type ISummaryConfiguration =
|
|
|
260
288
|
export const DefaultSummaryConfiguration: ISummaryConfiguration = {
|
|
261
289
|
state: "enabled",
|
|
262
290
|
|
|
263
|
-
idleTime:
|
|
291
|
+
idleTime: 15 * 1000, // 15 secs.
|
|
292
|
+
|
|
293
|
+
minIdleTime: 0,
|
|
264
294
|
|
|
265
|
-
|
|
295
|
+
maxIdleTime: 30 * 1000, // 30 secs.
|
|
266
296
|
|
|
267
|
-
|
|
297
|
+
maxTime: 60 * 1000, // 1 min.
|
|
298
|
+
|
|
299
|
+
maxOps: 100, // Summarize if 100 weighted ops received since last snapshot.
|
|
268
300
|
|
|
269
301
|
minOpsForLastSummaryAttempt: 10,
|
|
270
302
|
|
|
271
|
-
maxAckWaitTime:
|
|
303
|
+
maxAckWaitTime: 10 * 60 * 1000, // 10 mins.
|
|
272
304
|
|
|
273
305
|
maxOpsSinceLastSummary: 7000,
|
|
274
306
|
|
|
275
|
-
initialSummarizerDelayMs:
|
|
307
|
+
initialSummarizerDelayMs: 5 * 1000, // 5 secs.
|
|
276
308
|
|
|
277
309
|
summarizerClientElection: false,
|
|
310
|
+
|
|
311
|
+
nonRuntimeOpWeight: 0.1,
|
|
312
|
+
|
|
313
|
+
runtimeOpWeight: 1.0,
|
|
278
314
|
};
|
|
279
315
|
|
|
280
316
|
export interface IGCRuntimeOptions {
|
|
@@ -318,9 +354,12 @@ export interface ISummaryRuntimeOptions {
|
|
|
318
354
|
/** Override summary configurations set by the server. */
|
|
319
355
|
summaryConfigOverrides?: ISummaryConfiguration;
|
|
320
356
|
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
357
|
+
/**
|
|
358
|
+
* @deprecated - this option will not be supported on the next versions.
|
|
359
|
+
* Flag that disables putting channels in isolated subtrees for each data store
|
|
360
|
+
* and the root node when generating a summary if set to true.
|
|
361
|
+
* Defaults to FALSE (enabled) for now.
|
|
362
|
+
*/
|
|
324
363
|
disableIsolatedChannels?: boolean;
|
|
325
364
|
|
|
326
365
|
/**
|
|
@@ -466,11 +505,6 @@ const maxOpSizeInBytesKey = "Fluid.ContainerRuntime.MaxOpSizeInBytes";
|
|
|
466
505
|
// to not reach the 1MB limits in socket.io and Kafka.
|
|
467
506
|
const defaultMaxOpSizeInBytes = 768000;
|
|
468
507
|
|
|
469
|
-
// By default, the size of the contents for the incoming ops is tracked.
|
|
470
|
-
// However, in certain situations, this may incur a performance hit.
|
|
471
|
-
// The feature-gate below can be used to disable this feature.
|
|
472
|
-
const disableOpTrackingKey = "Fluid.ContainerRuntime.DisableOpTracking";
|
|
473
|
-
|
|
474
508
|
const defaultFlushMode = FlushMode.TurnBased;
|
|
475
509
|
|
|
476
510
|
export enum RuntimeMessage {
|
|
@@ -1036,6 +1070,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1036
1070
|
|
|
1037
1071
|
private consecutiveReconnects = 0;
|
|
1038
1072
|
|
|
1073
|
+
/**
|
|
1074
|
+
* Used to delay transition to "connected" state while we upload
|
|
1075
|
+
* attachment blobs that were added while disconnected
|
|
1076
|
+
*/
|
|
1077
|
+
private delayConnectClientId?: string;
|
|
1078
|
+
|
|
1039
1079
|
public get connected(): boolean {
|
|
1040
1080
|
return this._connected;
|
|
1041
1081
|
}
|
|
@@ -1160,7 +1200,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1160
1200
|
* a summary is generated.
|
|
1161
1201
|
*/
|
|
1162
1202
|
private nextSummaryNumber: number;
|
|
1163
|
-
private readonly opTracker: OpTracker;
|
|
1164
1203
|
|
|
1165
1204
|
private constructor(
|
|
1166
1205
|
private readonly context: IContainerContext,
|
|
@@ -1197,6 +1236,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1197
1236
|
this.mc = loggerToMonitoringContext(
|
|
1198
1237
|
ChildLogger.create(this.logger, "ContainerRuntime"));
|
|
1199
1238
|
|
|
1239
|
+
if (this.summaryConfiguration.state === "enabled") {
|
|
1240
|
+
this.validateSummaryHeuristicConfiguration(this.summaryConfiguration);
|
|
1241
|
+
}
|
|
1242
|
+
|
|
1200
1243
|
this.summariesDisabled = this.isSummariesDisabled();
|
|
1201
1244
|
this.heuristicsDisabled = this.isHeuristicsDisabled();
|
|
1202
1245
|
this.summarizerClientElectionEnabled = this.isSummarizerClientElectionEnabled();
|
|
@@ -1288,10 +1331,10 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1288
1331
|
this.handleContext,
|
|
1289
1332
|
blobManagerSnapshot,
|
|
1290
1333
|
() => this.storage,
|
|
1291
|
-
(blobId
|
|
1334
|
+
(blobId, localId) => this.submit(
|
|
1335
|
+
ContainerMessageType.BlobAttach, undefined, undefined, { blobId, localId }),
|
|
1292
1336
|
(blobPath: string) => this.garbageCollector.nodeUpdated(blobPath, "Loaded"),
|
|
1293
1337
|
this,
|
|
1294
|
-
this.logger,
|
|
1295
1338
|
);
|
|
1296
1339
|
|
|
1297
1340
|
this.scheduleManager = new ScheduleManager(
|
|
@@ -1442,9 +1485,9 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1442
1485
|
createContainerRuntimeVersion: metadata?.createContainerRuntimeVersion,
|
|
1443
1486
|
createContainerTimestamp: metadata?.createContainerTimestamp,
|
|
1444
1487
|
};
|
|
1445
|
-
//
|
|
1446
|
-
//
|
|
1447
|
-
loadSummaryNumber = metadata?.summaryNumber ??
|
|
1488
|
+
// summaryNumber was renamed from summaryCount. For older docs that haven't been opened for a long time,
|
|
1489
|
+
// the count is reset to 0.
|
|
1490
|
+
loadSummaryNumber = metadata?.summaryNumber ?? 0;
|
|
1448
1491
|
} else {
|
|
1449
1492
|
this.createContainerMetadata = {
|
|
1450
1493
|
createContainerRuntimeVersion: pkgVersion,
|
|
@@ -1466,7 +1509,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1466
1509
|
|
|
1467
1510
|
ReportOpPerfTelemetry(this.context.clientId, this.deltaManager, this.logger);
|
|
1468
1511
|
BindBatchTracker(this, this.logger);
|
|
1469
|
-
this.opTracker = new OpTracker(this.deltaManager, this.mc.config.getBoolean(disableOpTrackingKey) === true);
|
|
1470
1512
|
}
|
|
1471
1513
|
|
|
1472
1514
|
public dispose(error?: Error): void {
|
|
@@ -1546,12 +1588,12 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1546
1588
|
}
|
|
1547
1589
|
|
|
1548
1590
|
if (id === BlobManager.basePath && requestParser.isLeaf(2)) {
|
|
1549
|
-
const
|
|
1550
|
-
if (
|
|
1591
|
+
const blob = await this.blobManager.getBlob(requestParser.pathParts[1]);
|
|
1592
|
+
if (blob) {
|
|
1551
1593
|
return {
|
|
1552
1594
|
status: 200,
|
|
1553
1595
|
mimeType: "fluid/object",
|
|
1554
|
-
value:
|
|
1596
|
+
value: blob,
|
|
1555
1597
|
};
|
|
1556
1598
|
} else {
|
|
1557
1599
|
return create404Response(request);
|
|
@@ -1573,7 +1615,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1573
1615
|
}
|
|
1574
1616
|
|
|
1575
1617
|
private internalId(maybeAlias: string): string {
|
|
1576
|
-
return this.dataStores.aliases
|
|
1618
|
+
return this.dataStores.aliases.get(maybeAlias) ?? maybeAlias;
|
|
1577
1619
|
}
|
|
1578
1620
|
|
|
1579
1621
|
private async getDataStoreFromRequest(id: string, request: IRequest): Promise<IFluidRouter> {
|
|
@@ -1581,6 +1623,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1581
1623
|
? request.headers?.[RuntimeHeaders.wait]
|
|
1582
1624
|
: true;
|
|
1583
1625
|
|
|
1626
|
+
await this.dataStores.waitIfPendingAlias(id);
|
|
1584
1627
|
const internalId = this.internalId(id);
|
|
1585
1628
|
const dataStoreContext = await this.dataStores.getDataStore(internalId, wait);
|
|
1586
1629
|
|
|
@@ -1620,8 +1663,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1620
1663
|
private addMetadataToSummary(summaryTree: ISummaryTreeWithStats) {
|
|
1621
1664
|
const metadata: IContainerRuntimeMetadata = {
|
|
1622
1665
|
...this.createContainerMetadata,
|
|
1623
|
-
// back-compat 0.59.3000: This is renamed to summaryNumber. Can be removed when 0.59.3000 saturates.
|
|
1624
|
-
summaryCount: this.nextSummaryNumber,
|
|
1625
1666
|
// Increment the summary number for the next summary that will be generated.
|
|
1626
1667
|
summaryNumber: this.nextSummaryNumber++,
|
|
1627
1668
|
summaryFormatVersion: 1,
|
|
@@ -1647,7 +1688,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1647
1688
|
addBlobToSummary(summaryTree, chunksBlobName, content);
|
|
1648
1689
|
}
|
|
1649
1690
|
|
|
1650
|
-
const dataStoreAliases = this.dataStores.aliases
|
|
1691
|
+
const dataStoreAliases = this.dataStores.aliases;
|
|
1651
1692
|
if (dataStoreAliases.size > 0) {
|
|
1652
1693
|
addBlobToSummary(summaryTree, aliasBlobName, JSON.stringify([...dataStoreAliases]));
|
|
1653
1694
|
}
|
|
@@ -1757,6 +1798,38 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1757
1798
|
}
|
|
1758
1799
|
|
|
1759
1800
|
public setConnectionState(connected: boolean, clientId?: string) {
|
|
1801
|
+
if (connected === false && this.delayConnectClientId !== undefined) {
|
|
1802
|
+
this.delayConnectClientId = undefined;
|
|
1803
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1804
|
+
eventName: "UnsuccessfulConnectedTransition",
|
|
1805
|
+
});
|
|
1806
|
+
// Don't propagate "disconnected" event because we didn't propagate the previous "connected" event
|
|
1807
|
+
return;
|
|
1808
|
+
}
|
|
1809
|
+
|
|
1810
|
+
// If attachment blobs were added while disconnected, we need to delay
|
|
1811
|
+
// propagation of the "connected" event until we have uploaded them to
|
|
1812
|
+
// ensure we don't submit ops referencing a blob that has not been uploaded
|
|
1813
|
+
const connecting = connected && !this._connected && !this.deltaManager.readOnlyInfo.readonly;
|
|
1814
|
+
if (connecting && this.blobManager.hasPendingOfflineUploads) {
|
|
1815
|
+
assert(!this.delayConnectClientId, "Connect event delay must be canceled before subsequent connect event");
|
|
1816
|
+
assert(!!clientId, "Must have clientId when connecting");
|
|
1817
|
+
this.delayConnectClientId = clientId;
|
|
1818
|
+
this.blobManager.onConnected().then(() => {
|
|
1819
|
+
// make sure we didn't reconnect before the promise resolved
|
|
1820
|
+
if (this.delayConnectClientId === clientId && !this.disposed) {
|
|
1821
|
+
this.delayConnectClientId = undefined;
|
|
1822
|
+
this.setConnectionStateCore(connected, clientId);
|
|
1823
|
+
}
|
|
1824
|
+
}, (error) => this.closeFn(error));
|
|
1825
|
+
return;
|
|
1826
|
+
}
|
|
1827
|
+
|
|
1828
|
+
this.setConnectionStateCore(connected, clientId);
|
|
1829
|
+
}
|
|
1830
|
+
|
|
1831
|
+
private setConnectionStateCore(connected: boolean, clientId?: string) {
|
|
1832
|
+
assert(!this.delayConnectClientId, "connect event delay must be cleared before propagating connect event");
|
|
1760
1833
|
this.verifyNotClosed();
|
|
1761
1834
|
|
|
1762
1835
|
// There might be no change of state due to Container calling this API after loading runtime.
|
|
@@ -1854,8 +1927,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1854
1927
|
this.dataStores.processFluidDataStoreOp(message, local, localOpMetadata);
|
|
1855
1928
|
break;
|
|
1856
1929
|
case ContainerMessageType.BlobAttach:
|
|
1857
|
-
|
|
1858
|
-
this.blobManager.processBlobAttachOp(message.metadata.blobId, local);
|
|
1930
|
+
this.blobManager.processBlobAttachOp(message, local);
|
|
1859
1931
|
break;
|
|
1860
1932
|
default:
|
|
1861
1933
|
}
|
|
@@ -1937,6 +2009,11 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
1937
2009
|
}
|
|
1938
2010
|
|
|
1939
2011
|
public async getRootDataStore(id: string, wait = true): Promise<IFluidRouter> {
|
|
2012
|
+
return this.getRootDataStoreChannel(id, wait);
|
|
2013
|
+
}
|
|
2014
|
+
|
|
2015
|
+
private async getRootDataStoreChannel(id: string, wait = true): Promise<IFluidDataStoreChannel> {
|
|
2016
|
+
await this.dataStores.waitIfPendingAlias(id);
|
|
1940
2017
|
const internalId = this.internalId(id);
|
|
1941
2018
|
const context = await this.dataStores.getDataStore(internalId, wait);
|
|
1942
2019
|
assert(await context.isRoot(), 0x12b /* "did not get root data store" */);
|
|
@@ -2058,13 +2135,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2058
2135
|
*/
|
|
2059
2136
|
private async createRootDataStoreLegacy(pkg: string | string[], rootDataStoreId: string): Promise<IFluidRouter> {
|
|
2060
2137
|
const fluidDataStore = await this._createDataStore(pkg, true /* isRoot */, rootDataStoreId);
|
|
2061
|
-
|
|
2062
|
-
// older versions, we still have to call bindToContext.
|
|
2063
|
-
if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
|
|
2064
|
-
fluidDataStore.makeVisibleAndAttachGraph();
|
|
2065
|
-
} else {
|
|
2066
|
-
fluidDataStore.bindToContext();
|
|
2067
|
-
}
|
|
2138
|
+
fluidDataStore.makeVisibleAndAttachGraph();
|
|
2068
2139
|
return fluidDataStore;
|
|
2069
2140
|
}
|
|
2070
2141
|
|
|
@@ -2093,27 +2164,37 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2093
2164
|
*/
|
|
2094
2165
|
private async createAndAliasDataStore(pkg: string | string[], alias: string, props?: any): Promise<IDataStore> {
|
|
2095
2166
|
const internalId = uuid();
|
|
2096
|
-
const dataStore = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
2097
|
-
const aliasedDataStore = channelToDataStore(dataStore, internalId, this, this.dataStores, this.mc.logger);
|
|
2098
|
-
const result = await aliasedDataStore.trySetAlias(alias);
|
|
2099
|
-
if (result !== "Success") {
|
|
2100
|
-
throw new GenericError(
|
|
2101
|
-
"dataStoreAliasFailure",
|
|
2102
|
-
undefined /* error */,
|
|
2103
|
-
{
|
|
2104
|
-
alias: {
|
|
2105
|
-
value: alias,
|
|
2106
|
-
tag: TelemetryDataTag.UserData,
|
|
2107
|
-
},
|
|
2108
|
-
internalId: {
|
|
2109
|
-
value: internalId,
|
|
2110
|
-
tag: TelemetryDataTag.PackageData,
|
|
2111
|
-
},
|
|
2112
|
-
aliasResult: result,
|
|
2113
|
-
});
|
|
2114
|
-
}
|
|
2115
2167
|
|
|
2116
|
-
|
|
2168
|
+
try {
|
|
2169
|
+
// A similar call may have been initiated by the same client, so we should try to get
|
|
2170
|
+
// a possible existing aliased datastore first.
|
|
2171
|
+
const existingDataStore = await this.getRootDataStoreChannel(alias, /* wait */ false);
|
|
2172
|
+
return channelToDataStore(
|
|
2173
|
+
existingDataStore,
|
|
2174
|
+
internalId,
|
|
2175
|
+
this,
|
|
2176
|
+
this.dataStores,
|
|
2177
|
+
this.mc.logger,
|
|
2178
|
+
true, // AlreadyAliased. This will block further alias attempts for the datastore
|
|
2179
|
+
);
|
|
2180
|
+
} catch (err) {
|
|
2181
|
+
const newChannel = await this._createDataStore(pkg, false /* isRoot */, internalId, props);
|
|
2182
|
+
const newDataStore = channelToDataStore(newChannel, internalId, this, this.dataStores, this.mc.logger);
|
|
2183
|
+
const aliasResult = await newDataStore.trySetAlias(alias);
|
|
2184
|
+
if (aliasResult === "Success") {
|
|
2185
|
+
return newDataStore;
|
|
2186
|
+
}
|
|
2187
|
+
|
|
2188
|
+
const existingDataStore = await this.getRootDataStoreChannel(alias, /* wait */ false);
|
|
2189
|
+
return channelToDataStore(
|
|
2190
|
+
existingDataStore,
|
|
2191
|
+
internalId,
|
|
2192
|
+
this,
|
|
2193
|
+
this.dataStores,
|
|
2194
|
+
this.mc.logger,
|
|
2195
|
+
true, // AlreadyAliased. This will block further alias attempts for the datastore
|
|
2196
|
+
);
|
|
2197
|
+
}
|
|
2117
2198
|
}
|
|
2118
2199
|
|
|
2119
2200
|
public createDetachedRootDataStore(
|
|
@@ -2144,13 +2225,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2144
2225
|
const fluidDataStore = await this.dataStores._createFluidDataStoreContext(
|
|
2145
2226
|
Array.isArray(pkg) ? pkg : [pkg], id, isRoot, props).realize();
|
|
2146
2227
|
if (isRoot) {
|
|
2147
|
-
|
|
2148
|
-
// For older versions, we still have to call bindToContext.
|
|
2149
|
-
if (fluidDataStore.makeVisibleAndAttachGraph !== undefined) {
|
|
2150
|
-
fluidDataStore.makeVisibleAndAttachGraph();
|
|
2151
|
-
} else {
|
|
2152
|
-
fluidDataStore.bindToContext();
|
|
2153
|
-
}
|
|
2228
|
+
fluidDataStore.makeVisibleAndAttachGraph();
|
|
2154
2229
|
this.logger.sendTelemetryEvent({
|
|
2155
2230
|
eventName: "Root datastore with props",
|
|
2156
2231
|
hasProps: props !== undefined,
|
|
@@ -2560,15 +2635,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2560
2635
|
const summaryRefSeqNum = this.deltaManager.lastSequenceNumber;
|
|
2561
2636
|
const minimumSequenceNumber = this.deltaManager.minimumSequenceNumber;
|
|
2562
2637
|
const message = `Summary @${summaryRefSeqNum}:${this.deltaManager.minimumSequenceNumber}`;
|
|
2563
|
-
|
|
2564
|
-
// We should be here is we haven't processed be here. If we are of if the last message's sequence number
|
|
2565
|
-
// doesn't match the last processed sequence number, log an error.
|
|
2566
|
-
if (summaryRefSeqNum !== this.deltaManager.lastMessage?.sequenceNumber) {
|
|
2567
|
-
summaryNumberLogger.sendErrorEvent({
|
|
2568
|
-
eventName: "LastSequenceMismatch",
|
|
2569
|
-
error: message,
|
|
2570
|
-
});
|
|
2571
|
-
}
|
|
2638
|
+
const lastAck = this.summaryCollection.latestAck;
|
|
2572
2639
|
|
|
2573
2640
|
this.summarizerNode.startSummary(summaryRefSeqNum, summaryNumberLogger);
|
|
2574
2641
|
|
|
@@ -2598,6 +2665,16 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2598
2665
|
error: `lastSequenceNumber changed before uploading to storage. ${this.deltaManager.lastSequenceNumber} !== ${summaryRefSeqNum}`,
|
|
2599
2666
|
};
|
|
2600
2667
|
}
|
|
2668
|
+
assert(summaryRefSeqNum === this.deltaManager.lastMessage?.sequenceNumber,
|
|
2669
|
+
"it's one and the same thing");
|
|
2670
|
+
|
|
2671
|
+
if (lastAck !== this.summaryCollection.latestAck) {
|
|
2672
|
+
return {
|
|
2673
|
+
continue: false,
|
|
2674
|
+
// eslint-disable-next-line max-len
|
|
2675
|
+
error: `Last summary changed while summarizing. ${this.summaryCollection.latestAck} !== ${lastAck}`,
|
|
2676
|
+
};
|
|
2677
|
+
}
|
|
2601
2678
|
return { continue: true };
|
|
2602
2679
|
};
|
|
2603
2680
|
|
|
@@ -2654,8 +2731,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2654
2731
|
gcStateUpdatedDataStoreCount: summarizeResult.gcStats?.updatedDataStoreCount,
|
|
2655
2732
|
gcBlobNodeCount: gcSummaryTreeStats?.blobNodeCount,
|
|
2656
2733
|
gcTotalBlobsSize: gcSummaryTreeStats?.totalBlobSize,
|
|
2657
|
-
opsSizesSinceLastSummary: this.opTracker.opsSizeAccumulator,
|
|
2658
|
-
nonSystemOpsSinceLastSummary: this.opTracker.nonSystemOpCount,
|
|
2659
2734
|
summaryNumber,
|
|
2660
2735
|
...partialStats,
|
|
2661
2736
|
};
|
|
@@ -2673,7 +2748,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2673
2748
|
return { stage: "generate", ...generateSummaryData, error: continueResult.error };
|
|
2674
2749
|
}
|
|
2675
2750
|
|
|
2676
|
-
const lastAck = this.summaryCollection.latestAck;
|
|
2677
2751
|
const summaryContext: ISummaryContext =
|
|
2678
2752
|
lastAck === undefined
|
|
2679
2753
|
? {
|
|
@@ -2728,7 +2802,6 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2728
2802
|
} as const;
|
|
2729
2803
|
|
|
2730
2804
|
this.summarizerNode.completeSummary(handle);
|
|
2731
|
-
this.opTracker.reset();
|
|
2732
2805
|
return submitData;
|
|
2733
2806
|
} finally {
|
|
2734
2807
|
// Cleanup wip summary in case of failure
|
|
@@ -2892,14 +2965,8 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
2892
2965
|
"OpTooLarge",
|
|
2893
2966
|
/* error */ undefined,
|
|
2894
2967
|
{
|
|
2895
|
-
length:
|
|
2896
|
-
|
|
2897
|
-
tag: TelemetryDataTag.PackageData,
|
|
2898
|
-
},
|
|
2899
|
-
limit: {
|
|
2900
|
-
value: this._maxOpSizeInBytes,
|
|
2901
|
-
tag: TelemetryDataTag.PackageData,
|
|
2902
|
-
},
|
|
2968
|
+
length: serializedContent.length,
|
|
2969
|
+
limit: this._maxOpSizeInBytes,
|
|
2903
2970
|
}));
|
|
2904
2971
|
return -1;
|
|
2905
2972
|
}
|
|
@@ -3005,7 +3072,7 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
3005
3072
|
case ContainerMessageType.ChunkedOp:
|
|
3006
3073
|
throw new Error(`chunkedOp not expected here`);
|
|
3007
3074
|
case ContainerMessageType.BlobAttach:
|
|
3008
|
-
this.
|
|
3075
|
+
this.blobManager.reSubmit(opMetadata);
|
|
3009
3076
|
break;
|
|
3010
3077
|
case ContainerMessageType.Rejoin:
|
|
3011
3078
|
this.submit(type, content);
|
|
@@ -3218,6 +3285,15 @@ export class ContainerRuntime extends TypedEventEmitter<IContainerRuntimeEvents>
|
|
|
3218
3285
|
// don't have any more saved ops
|
|
3219
3286
|
await this.pendingStateManager.applyStashedOpsAt();
|
|
3220
3287
|
}
|
|
3288
|
+
|
|
3289
|
+
private validateSummaryHeuristicConfiguration(configuration: ISummaryConfigurationHeuristics) {
|
|
3290
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
3291
|
+
for (const prop in configuration) {
|
|
3292
|
+
if (typeof configuration[prop] === "number" && configuration[prop] < 0) {
|
|
3293
|
+
throw new UsageError(`Summary heuristic configuration property "${prop}" cannot be less than 0`);
|
|
3294
|
+
}
|
|
3295
|
+
}
|
|
3296
|
+
}
|
|
3221
3297
|
}
|
|
3222
3298
|
|
|
3223
3299
|
/**
|
package/src/dataStore.ts
CHANGED
|
@@ -43,7 +43,8 @@ export const channelToDataStore = (
|
|
|
43
43
|
runtime: ContainerRuntime,
|
|
44
44
|
datastores: DataStores,
|
|
45
45
|
logger: ITelemetryLogger,
|
|
46
|
-
|
|
46
|
+
alreadyAliased: boolean = false,
|
|
47
|
+
): IDataStore => new DataStore(fluidDataStoreChannel, internalId, runtime, datastores, logger, alreadyAliased);
|
|
47
48
|
|
|
48
49
|
enum AliasState {
|
|
49
50
|
Aliased = "Aliased",
|
|
@@ -54,6 +55,7 @@ enum AliasState {
|
|
|
54
55
|
class DataStore implements IDataStore {
|
|
55
56
|
private aliasState: AliasState = AliasState.None;
|
|
56
57
|
private alias: string | undefined;
|
|
58
|
+
private readonly pendingAliases: Map<string, Promise<AliasResult>>;
|
|
57
59
|
private aliasResult: Promise<AliasResult> | undefined;
|
|
58
60
|
|
|
59
61
|
async trySetAlias(alias: string): Promise<AliasResult> {
|
|
@@ -75,14 +77,25 @@ class DataStore implements IDataStore {
|
|
|
75
77
|
case AliasState.Aliased:
|
|
76
78
|
return this.alias === alias ? "Success" : "AlreadyAliased";
|
|
77
79
|
|
|
78
|
-
|
|
79
|
-
|
|
80
|
-
|
|
80
|
+
case AliasState.None: {
|
|
81
|
+
const existingAlias = this.pendingAliases.get(alias);
|
|
82
|
+
if (existingAlias !== undefined) {
|
|
83
|
+
// There is already another datastore which will be aliased
|
|
84
|
+
// to the same name
|
|
85
|
+
return "Conflict";
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
// There is no current or past alias operation for this datastore,
|
|
89
|
+
// or for this alias, so it is safe to continue execution
|
|
90
|
+
break;
|
|
91
|
+
}
|
|
92
|
+
|
|
81
93
|
default: unreachableCase(this.aliasState);
|
|
82
94
|
}
|
|
83
95
|
|
|
84
96
|
this.aliasState = AliasState.Aliasing;
|
|
85
97
|
this.aliasResult = this.trySetAliasInternal(alias);
|
|
98
|
+
this.pendingAliases.set(alias, this.aliasResult);
|
|
86
99
|
return this.aliasResult;
|
|
87
100
|
}
|
|
88
101
|
|
|
@@ -92,13 +105,7 @@ class DataStore implements IDataStore {
|
|
|
92
105
|
alias,
|
|
93
106
|
};
|
|
94
107
|
|
|
95
|
-
|
|
96
|
-
// older versions, we still have to call bindToContext.
|
|
97
|
-
if (this.fluidDataStoreChannel.makeVisibleAndAttachGraph !== undefined) {
|
|
98
|
-
this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
|
|
99
|
-
} else {
|
|
100
|
-
this.fluidDataStoreChannel.bindToContext();
|
|
101
|
-
}
|
|
108
|
+
this.fluidDataStoreChannel.makeVisibleAndAttachGraph();
|
|
102
109
|
|
|
103
110
|
if (this.runtime.attachState === AttachState.Detached) {
|
|
104
111
|
const localResult = this.datastores.processAliasMessageCore(message);
|
|
@@ -108,34 +115,37 @@ class DataStore implements IDataStore {
|
|
|
108
115
|
return localResult ? "Success" : "Conflict";
|
|
109
116
|
}
|
|
110
117
|
|
|
111
|
-
const aliased = await this
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
118
|
+
const aliased = await this
|
|
119
|
+
.ackBasedPromise<boolean>((resolve) => {
|
|
120
|
+
this.runtime.submitDataStoreAliasOp(message, resolve);
|
|
121
|
+
})
|
|
122
|
+
.catch((error) => {
|
|
123
|
+
this.logger.sendErrorEvent({
|
|
124
|
+
eventName: "AliasingException",
|
|
125
|
+
alias: {
|
|
126
|
+
value: alias,
|
|
127
|
+
tag: TelemetryDataTag.UserData,
|
|
128
|
+
},
|
|
129
|
+
internalId: {
|
|
130
|
+
value: this.internalId,
|
|
131
|
+
tag: TelemetryDataTag.CodeArtifact,
|
|
132
|
+
},
|
|
133
|
+
}, error);
|
|
134
|
+
|
|
135
|
+
return false;
|
|
136
|
+
}).finally(() => {
|
|
137
|
+
this.pendingAliases.delete(alias);
|
|
138
|
+
});
|
|
139
|
+
|
|
140
|
+
if (!aliased) {
|
|
134
141
|
this.aliasState = AliasState.None;
|
|
135
|
-
|
|
136
|
-
|
|
142
|
+
this.aliasResult = undefined;
|
|
143
|
+
return "Conflict";
|
|
144
|
+
}
|
|
137
145
|
|
|
138
|
-
|
|
146
|
+
this.alias = alias;
|
|
147
|
+
this.aliasState = AliasState.Aliased;
|
|
148
|
+
return "Success";
|
|
139
149
|
}
|
|
140
150
|
|
|
141
151
|
async request(request: IRequest): Promise<IResponse> {
|
|
@@ -148,7 +158,12 @@ class DataStore implements IDataStore {
|
|
|
148
158
|
private readonly runtime: ContainerRuntime,
|
|
149
159
|
private readonly datastores: DataStores,
|
|
150
160
|
private readonly logger: ITelemetryLogger,
|
|
151
|
-
|
|
161
|
+
alreadyAliased: boolean,
|
|
162
|
+
) {
|
|
163
|
+
this.pendingAliases = datastores.pendingAliases;
|
|
164
|
+
this.aliasState = alreadyAliased ? AliasState.Aliased : AliasState.None;
|
|
165
|
+
}
|
|
166
|
+
|
|
152
167
|
public get IFluidRouter() { return this.fluidDataStoreChannel; }
|
|
153
168
|
|
|
154
169
|
private async ackBasedPromise<T>(
|