@fluidframework/container-runtime 2.91.0 → 2.92.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/api-report/container-runtime.legacy.beta.api.md +2 -0
- package/container-runtime.test-files.tar +0 -0
- package/dist/containerCompatibility.d.ts +1 -1
- package/dist/containerCompatibility.d.ts.map +1 -1
- package/dist/containerCompatibility.js.map +1 -1
- package/dist/containerRuntime.d.ts +36 -9
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +97 -54
- package/dist/containerRuntime.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +1 -0
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +3 -8
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +4 -0
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +1 -1
- package/dist/opLifecycle/batchManager.d.ts.map +1 -1
- package/dist/opLifecycle/batchManager.js +2 -1
- package/dist/opLifecycle/batchManager.js.map +1 -1
- package/dist/opLifecycle/index.d.ts +1 -1
- package/dist/opLifecycle/index.d.ts.map +1 -1
- package/dist/opLifecycle/index.js +2 -1
- package/dist/opLifecycle/index.js.map +1 -1
- package/dist/opLifecycle/opGroupingManager.d.ts +6 -0
- package/dist/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/dist/opLifecycle/opGroupingManager.js +11 -2
- package/dist/opLifecycle/opGroupingManager.js.map +1 -1
- package/dist/opLifecycle/opSerialization.d.ts +3 -1
- package/dist/opLifecycle/opSerialization.d.ts.map +1 -1
- package/dist/opLifecycle/opSerialization.js +11 -9
- package/dist/opLifecycle/opSerialization.js.map +1 -1
- package/dist/opLifecycle/outbox.d.ts +0 -6
- package/dist/opLifecycle/outbox.d.ts.map +1 -1
- package/dist/opLifecycle/outbox.js +2 -9
- package/dist/opLifecycle/outbox.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +7 -3
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +19 -7
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/public.d.ts +1 -1
- package/dist/runtimeLayerCompatState.d.ts +1 -1
- package/dist/summary/documentSchema.d.ts +9 -3
- package/dist/summary/documentSchema.d.ts.map +1 -1
- package/dist/summary/documentSchema.js +19 -3
- package/dist/summary/documentSchema.js.map +1 -1
- package/dist/summary/orderedClientElection.js +2 -2
- package/dist/summary/orderedClientElection.js.map +1 -1
- package/dist/summary/summaryManager.d.ts +1 -0
- package/dist/summary/summaryManager.d.ts.map +1 -1
- package/dist/summary/summaryManager.js +9 -0
- package/dist/summary/summaryManager.js.map +1 -1
- package/internal.d.ts +1 -1
- package/legacy.d.ts +1 -1
- package/lib/containerCompatibility.d.ts +1 -1
- package/lib/containerCompatibility.d.ts.map +1 -1
- package/lib/containerCompatibility.js.map +1 -1
- package/lib/containerRuntime.d.ts +36 -9
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +97 -55
- package/lib/containerRuntime.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +1 -0
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +3 -8
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +4 -0
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/index.d.ts +1 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -1
- package/lib/opLifecycle/batchManager.d.ts.map +1 -1
- package/lib/opLifecycle/batchManager.js +2 -1
- package/lib/opLifecycle/batchManager.js.map +1 -1
- package/lib/opLifecycle/index.d.ts +1 -1
- package/lib/opLifecycle/index.d.ts.map +1 -1
- package/lib/opLifecycle/index.js +1 -1
- package/lib/opLifecycle/index.js.map +1 -1
- package/lib/opLifecycle/opGroupingManager.d.ts +6 -0
- package/lib/opLifecycle/opGroupingManager.d.ts.map +1 -1
- package/lib/opLifecycle/opGroupingManager.js +10 -1
- package/lib/opLifecycle/opGroupingManager.js.map +1 -1
- package/lib/opLifecycle/opSerialization.d.ts +3 -1
- package/lib/opLifecycle/opSerialization.d.ts.map +1 -1
- package/lib/opLifecycle/opSerialization.js +11 -9
- package/lib/opLifecycle/opSerialization.js.map +1 -1
- package/lib/opLifecycle/outbox.d.ts +0 -6
- package/lib/opLifecycle/outbox.d.ts.map +1 -1
- package/lib/opLifecycle/outbox.js +2 -9
- package/lib/opLifecycle/outbox.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +7 -3
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +19 -7
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/public.d.ts +1 -1
- package/lib/runtimeLayerCompatState.d.ts +1 -1
- package/lib/summary/documentSchema.d.ts +9 -3
- package/lib/summary/documentSchema.d.ts.map +1 -1
- package/lib/summary/documentSchema.js +19 -3
- package/lib/summary/documentSchema.js.map +1 -1
- package/lib/summary/orderedClientElection.js +2 -2
- package/lib/summary/orderedClientElection.js.map +1 -1
- package/lib/summary/summaryManager.d.ts +1 -0
- package/lib/summary/summaryManager.d.ts.map +1 -1
- package/lib/summary/summaryManager.js +9 -0
- package/lib/summary/summaryManager.js.map +1 -1
- package/package.json +27 -23
- package/src/containerCompatibility.ts +2 -0
- package/src/containerRuntime.ts +144 -66
- package/src/gc/garbageCollection.ts +4 -9
- package/src/gc/gcDefinitions.ts +4 -0
- package/src/index.ts +1 -0
- package/src/opLifecycle/batchManager.ts +2 -1
- package/src/opLifecycle/index.ts +1 -0
- package/src/opLifecycle/opGroupingManager.ts +11 -1
- package/src/opLifecycle/opSerialization.ts +14 -12
- package/src/opLifecycle/outbox.ts +2 -17
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +27 -11
- package/src/summary/documentSchema.ts +25 -2
- package/src/summary/orderedClientElection.ts +2 -2
- package/src/summary/summaryManager.ts +11 -0
package/src/containerRuntime.ts
CHANGED
|
@@ -95,6 +95,7 @@ import { FetchSource, MessageType } from "@fluidframework/driver-definitions/int
|
|
|
95
95
|
import { readAndParse } from "@fluidframework/driver-utils/internal";
|
|
96
96
|
import type { IIdCompressor } from "@fluidframework/id-compressor";
|
|
97
97
|
import type {
|
|
98
|
+
// eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
|
|
98
99
|
IIdCompressorCore,
|
|
99
100
|
IdCreationRange,
|
|
100
101
|
SerializedIdCompressorWithNoSession,
|
|
@@ -130,6 +131,7 @@ import type {
|
|
|
130
131
|
IContainerRuntimeBaseInternal,
|
|
131
132
|
MinimumVersionForCollab,
|
|
132
133
|
ContainerExtensionExpectations,
|
|
134
|
+
ContainerRuntimeBaseAlpha,
|
|
133
135
|
} from "@fluidframework/runtime-definitions/internal";
|
|
134
136
|
import {
|
|
135
137
|
addBlobToSummary,
|
|
@@ -219,7 +221,6 @@ import {
|
|
|
219
221
|
type IGCRuntimeOptions,
|
|
220
222
|
type IGCStats,
|
|
221
223
|
type IGarbageCollector,
|
|
222
|
-
gcGenerationOptionName,
|
|
223
224
|
type GarbageCollectionMessage,
|
|
224
225
|
type IGarbageCollectionRuntime,
|
|
225
226
|
} from "./gc/index.js";
|
|
@@ -243,6 +244,7 @@ import {
|
|
|
243
244
|
DuplicateBatchDetector,
|
|
244
245
|
ensureContentsDeserialized,
|
|
245
246
|
type IBatchCheckpoint,
|
|
247
|
+
largeBatchThreshold,
|
|
246
248
|
OpCompressor,
|
|
247
249
|
OpDecompressor,
|
|
248
250
|
OpGroupingManager,
|
|
@@ -258,6 +260,7 @@ import {
|
|
|
258
260
|
type IPendingLocalState,
|
|
259
261
|
PendingStateManager,
|
|
260
262
|
type PendingBatchResubmitMetadata,
|
|
263
|
+
type IPendingMessage,
|
|
261
264
|
} from "./pendingStateManager.js";
|
|
262
265
|
import { BatchRunCounter, RunCounter } from "./runCounter.js";
|
|
263
266
|
import {
|
|
@@ -475,6 +478,25 @@ export interface ContainerRuntimeOptions {
|
|
|
475
478
|
* When enabled (`true`), createBlob will return a handle before the blob upload completes.
|
|
476
479
|
*/
|
|
477
480
|
readonly createBlobPayloadPending: true | undefined;
|
|
481
|
+
|
|
482
|
+
/**
|
|
483
|
+
* Controls automatic batch flushing during staging mode.
|
|
484
|
+
* Normal turn-based/async flush scheduling is suppressed while in staging mode
|
|
485
|
+
* until the accumulated batch reaches this many ops, at which point the batch
|
|
486
|
+
* is flushed. Incoming ops always break the current batch regardless of this setting.
|
|
487
|
+
*
|
|
488
|
+
* Set to Infinity to only break batches on system events (incoming ops).
|
|
489
|
+
*
|
|
490
|
+
* @defaultValue `largeBatchThreshold` (currently 1000)
|
|
491
|
+
*/
|
|
492
|
+
readonly stagingModeAutoFlushThreshold: number;
|
|
493
|
+
|
|
494
|
+
/**
|
|
495
|
+
* When this property is set to true, the runtime will never send DocumentSchemaChange ops
|
|
496
|
+
* and will throw an error if any incoming DocumentSchemaChange ops are received.
|
|
497
|
+
* This effectively freezes the document schema at whatever state it was in when the document was created.
|
|
498
|
+
*/
|
|
499
|
+
readonly disableSchemaUpgrade: boolean;
|
|
478
500
|
}
|
|
479
501
|
|
|
480
502
|
/**
|
|
@@ -607,6 +629,16 @@ const defaultMaxBatchSizeInBytes = 700 * 1024;
|
|
|
607
629
|
|
|
608
630
|
const defaultChunkSizeInBytes = 204800;
|
|
609
631
|
|
|
632
|
+
/**
|
|
633
|
+
* Default maximum ops per staging-mode batch before automatic flush scheduling resumes.
|
|
634
|
+
*
|
|
635
|
+
* Chosen based on production telemetry: copy-paste operations routinely produce batches
|
|
636
|
+
* of 1000+ ops (435K instances over 30 days), and receivers on modern Fluid versions
|
|
637
|
+
* handle them without issues. Uses {@link largeBatchThreshold} to stay aligned with
|
|
638
|
+
* the existing "large batch" telemetry threshold ({@link OpGroupingManager}).
|
|
639
|
+
*/
|
|
640
|
+
const defaultStagingModeAutoFlushThreshold = largeBatchThreshold;
|
|
641
|
+
|
|
610
642
|
/**
|
|
611
643
|
* The default time to wait for pending ops to be processed during summarization
|
|
612
644
|
*/
|
|
@@ -804,6 +836,24 @@ export async function loadContainerRuntime(
|
|
|
804
836
|
return ContainerRuntime.loadRuntime(params);
|
|
805
837
|
}
|
|
806
838
|
|
|
839
|
+
/**
|
|
840
|
+
* Alpha variant of {@link loadContainerRuntime} that returns the runtime in an
|
|
841
|
+
* extendable object, allowing additional properties to be added in the future.
|
|
842
|
+
*
|
|
843
|
+
* @param params - An object which specifies all required and optional params necessary to instantiate a runtime.
|
|
844
|
+
* @returns An object containing the runtime.
|
|
845
|
+
*
|
|
846
|
+
* @legacy @alpha
|
|
847
|
+
*/
|
|
848
|
+
export async function loadContainerRuntimeAlpha(params: LoadContainerRuntimeParams): Promise<{
|
|
849
|
+
runtime: IContainerRuntime & ContainerRuntimeBaseAlpha & IRuntime;
|
|
850
|
+
}> {
|
|
851
|
+
return ContainerRuntime.loadRuntime2({
|
|
852
|
+
...params,
|
|
853
|
+
registry: new FluidDataStoreRegistry(params.registryEntries),
|
|
854
|
+
});
|
|
855
|
+
}
|
|
856
|
+
|
|
807
857
|
const defaultMaxConsecutiveReconnects = 7;
|
|
808
858
|
|
|
809
859
|
/**
|
|
@@ -879,14 +929,15 @@ export class ContainerRuntime
|
|
|
879
929
|
return ContainerRuntime.loadRuntime2({
|
|
880
930
|
...params,
|
|
881
931
|
registry: new FluidDataStoreRegistry(params.registryEntries),
|
|
882
|
-
});
|
|
932
|
+
}).then((r) => r.runtime);
|
|
883
933
|
}
|
|
884
934
|
|
|
885
935
|
/**
|
|
886
|
-
* Load the stores from a snapshot and returns the runtime.
|
|
936
|
+
* Load the stores from a snapshot and returns an object containing the runtime.
|
|
887
937
|
* @remarks
|
|
888
938
|
* Same as {@link ContainerRuntime.loadRuntime},
|
|
889
939
|
* but with `registry` instead of `registryEntries` and more `runtimeOptions`.
|
|
940
|
+
* Returns `{ runtime }` to allow future extensions (e.g. staging mode controls).
|
|
890
941
|
*/
|
|
891
942
|
public static async loadRuntime2(
|
|
892
943
|
params: Omit<LoadContainerRuntimeParams, "registryEntries" | "runtimeOptions"> & {
|
|
@@ -905,7 +956,7 @@ export class ContainerRuntime
|
|
|
905
956
|
*/
|
|
906
957
|
runtimeOptions?: IContainerRuntimeOptionsInternal;
|
|
907
958
|
},
|
|
908
|
-
): Promise<ContainerRuntime> {
|
|
959
|
+
): Promise<{ runtime: ContainerRuntime }> {
|
|
909
960
|
const {
|
|
910
961
|
context,
|
|
911
962
|
registry,
|
|
@@ -961,6 +1012,8 @@ export class ContainerRuntime
|
|
|
961
1012
|
loadSequenceNumberVerification: "close",
|
|
962
1013
|
maxBatchSizeInBytes: defaultMaxBatchSizeInBytes,
|
|
963
1014
|
chunkSizeInBytes: defaultChunkSizeInBytes,
|
|
1015
|
+
stagingModeAutoFlushThreshold: defaultStagingModeAutoFlushThreshold,
|
|
1016
|
+
disableSchemaUpgrade: false,
|
|
964
1017
|
};
|
|
965
1018
|
|
|
966
1019
|
const defaultConfigs = {
|
|
@@ -986,6 +1039,8 @@ export class ContainerRuntime
|
|
|
986
1039
|
? disabledCompressionConfig
|
|
987
1040
|
: defaultConfigs.compressionOptions,
|
|
988
1041
|
createBlobPayloadPending = defaultConfigs.createBlobPayloadPending,
|
|
1042
|
+
stagingModeAutoFlushThreshold = defaultConfigs.stagingModeAutoFlushThreshold,
|
|
1043
|
+
disableSchemaUpgrade = defaultConfigs.disableSchemaUpgrade,
|
|
989
1044
|
}: IContainerRuntimeOptionsInternal = runtimeOptions;
|
|
990
1045
|
|
|
991
1046
|
// If explicitSchemaControl is off, ensure that options which require explicitSchemaControl are not enabled.
|
|
@@ -1130,6 +1185,7 @@ export class ContainerRuntime
|
|
|
1130
1185
|
idCompressorMode = desiredIdCompressorMode;
|
|
1131
1186
|
}
|
|
1132
1187
|
|
|
1188
|
+
// eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
|
|
1133
1189
|
const createIdCompressorFn = (): IIdCompressor & IIdCompressorCore => {
|
|
1134
1190
|
/**
|
|
1135
1191
|
* Because the IdCompressor emits so much telemetry, this function is used to sample
|
|
@@ -1184,6 +1240,7 @@ export class ContainerRuntime
|
|
|
1184
1240
|
},
|
|
1185
1241
|
{ minVersionForCollab },
|
|
1186
1242
|
logger,
|
|
1243
|
+
disableSchemaUpgrade,
|
|
1187
1244
|
);
|
|
1188
1245
|
|
|
1189
1246
|
// If the minVersionForCollab for this client is greater than the existing one, we should use that one going forward.
|
|
@@ -1214,6 +1271,8 @@ export class ContainerRuntime
|
|
|
1214
1271
|
enableGroupedBatching,
|
|
1215
1272
|
explicitSchemaControl,
|
|
1216
1273
|
createBlobPayloadPending,
|
|
1274
|
+
stagingModeAutoFlushThreshold,
|
|
1275
|
+
disableSchemaUpgrade,
|
|
1217
1276
|
};
|
|
1218
1277
|
|
|
1219
1278
|
validateMinimumVersionForCollab(updatedMinVersionForCollab);
|
|
@@ -1249,7 +1308,7 @@ export class ContainerRuntime
|
|
|
1249
1308
|
// or zero. This must be done before Container replays saved ops.
|
|
1250
1309
|
await runtime.pendingStateManager.applyStashedOpsAt(runtimeSequenceNumber ?? 0);
|
|
1251
1310
|
|
|
1252
|
-
return runtime;
|
|
1311
|
+
return { runtime };
|
|
1253
1312
|
}
|
|
1254
1313
|
|
|
1255
1314
|
public readonly options: Record<string | number, unknown>;
|
|
@@ -1319,6 +1378,7 @@ export class ContainerRuntime
|
|
|
1319
1378
|
return this.documentsSchemaController.sessionSchema.runtime;
|
|
1320
1379
|
}
|
|
1321
1380
|
|
|
1381
|
+
// eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
|
|
1322
1382
|
private _idCompressor: (IIdCompressor & IIdCompressorCore) | undefined;
|
|
1323
1383
|
|
|
1324
1384
|
// We accumulate Id compressor Ops while Id compressor is not loaded yet (only for "delayed" mode)
|
|
@@ -1334,6 +1394,7 @@ export class ContainerRuntime
|
|
|
1334
1394
|
/**
|
|
1335
1395
|
* {@inheritDoc @fluidframework/runtime-definitions#IContainerRuntimeBase.idCompressor}
|
|
1336
1396
|
*/
|
|
1397
|
+
// eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
|
|
1337
1398
|
public get idCompressor(): (IIdCompressor & IIdCompressorCore) | undefined {
|
|
1338
1399
|
// Expose ID Compressor only if it's On from the start.
|
|
1339
1400
|
// If container uses delayed mode, then we can only expose generateDocumentUniqueId() and nothing else.
|
|
@@ -1392,6 +1453,7 @@ export class ContainerRuntime
|
|
|
1392
1453
|
|
|
1393
1454
|
private readonly batchRunner = new BatchRunCounter();
|
|
1394
1455
|
private readonly _flushMode: FlushMode;
|
|
1456
|
+
private readonly stagingModeAutoFlushThreshold: number;
|
|
1395
1457
|
/**
|
|
1396
1458
|
* BatchId tracking is needed whenever there's a possibility of a "forked Container",
|
|
1397
1459
|
* where the same local state is pending in two different running Containers, each of
|
|
@@ -1533,13 +1595,6 @@ export class ContainerRuntime
|
|
|
1533
1595
|
return runtimeCompatDetailsForLoader;
|
|
1534
1596
|
}
|
|
1535
1597
|
|
|
1536
|
-
/**
|
|
1537
|
-
* If true, will skip Outbox flushing before processing an incoming message (and on DeltaManager "op" event for loader back-compat),
|
|
1538
|
-
* and instead the Outbox will check for a split batch on every submit.
|
|
1539
|
-
* This is a kill-bit switch for this simplification of logic, in case it causes unexpected issues.
|
|
1540
|
-
*/
|
|
1541
|
-
private readonly skipSafetyFlushDuringProcessStack: boolean;
|
|
1542
|
-
|
|
1543
1598
|
private readonly extensions = new Map<ContainerExtensionId, ExtensionEntry>();
|
|
1544
1599
|
|
|
1545
1600
|
/***/
|
|
@@ -1560,6 +1615,7 @@ export class ContainerRuntime
|
|
|
1560
1615
|
|
|
1561
1616
|
blobManagerLoadInfo: IBlobManagerLoadInfo,
|
|
1562
1617
|
private readonly _storage: IContainerStorageService,
|
|
1618
|
+
// eslint-disable-next-line import-x/no-deprecated -- Will be undeprecated in 2.100.0 when it becomes an internal API
|
|
1563
1619
|
private readonly createIdCompressorFn: () => IIdCompressor & IIdCompressorCore,
|
|
1564
1620
|
|
|
1565
1621
|
private readonly documentsSchemaController: DocumentsSchemaController,
|
|
@@ -1750,15 +1806,6 @@ export class ContainerRuntime
|
|
|
1750
1806
|
this.getConnectionState() === ConnectionState.CatchingUp
|
|
1751
1807
|
: undefined;
|
|
1752
1808
|
|
|
1753
|
-
this.mc.logger.sendTelemetryEvent({
|
|
1754
|
-
eventName: "GCFeatureMatrix",
|
|
1755
|
-
metadataValue: JSON.stringify(metadata?.gcFeatureMatrix),
|
|
1756
|
-
inputs: JSON.stringify({
|
|
1757
|
-
// eslint-disable-next-line @typescript-eslint/no-unsafe-assignment
|
|
1758
|
-
gcOptions_gcGeneration: runtimeOptions.gcOptions[gcGenerationOptionName],
|
|
1759
|
-
}),
|
|
1760
|
-
});
|
|
1761
|
-
|
|
1762
1809
|
this.telemetryDocumentId = metadata?.telemetryDocumentId ?? uuid();
|
|
1763
1810
|
|
|
1764
1811
|
const opGroupingManager = new OpGroupingManager(
|
|
@@ -1840,6 +1887,10 @@ export class ContainerRuntime
|
|
|
1840
1887
|
this.closeFn(error);
|
|
1841
1888
|
throw error;
|
|
1842
1889
|
}
|
|
1890
|
+
this.stagingModeAutoFlushThreshold =
|
|
1891
|
+
this.mc.config.getNumber("Fluid.ContainerRuntime.StagingModeAutoFlushThreshold") ??
|
|
1892
|
+
runtimeOptions.stagingModeAutoFlushThreshold ??
|
|
1893
|
+
defaultStagingModeAutoFlushThreshold;
|
|
1843
1894
|
this.batchIdTrackingEnabled =
|
|
1844
1895
|
this.mc.config.getBoolean("Fluid.Container.enableOfflineFull") ??
|
|
1845
1896
|
this.mc.config.getBoolean("Fluid.ContainerRuntime.enableBatchIdTracking") ??
|
|
@@ -1998,10 +2049,6 @@ export class ContainerRuntime
|
|
|
1998
2049
|
|
|
1999
2050
|
const legacySendBatchFn = makeLegacySendBatchFn(submitFn, this.innerDeltaManager);
|
|
2000
2051
|
|
|
2001
|
-
this.skipSafetyFlushDuringProcessStack =
|
|
2002
|
-
// Keep the old flag name even though we renamed the class member (it shipped in 2.31.0)
|
|
2003
|
-
this.mc.config.getBoolean("Fluid.ContainerRuntime.DisableFlushBeforeProcess") === true;
|
|
2004
|
-
|
|
2005
2052
|
this.outbox = new Outbox({
|
|
2006
2053
|
shouldSend: () => this.shouldSendOps(),
|
|
2007
2054
|
pendingStateManager: this.pendingStateManager,
|
|
@@ -2012,8 +2059,6 @@ export class ContainerRuntime
|
|
|
2012
2059
|
config: {
|
|
2013
2060
|
compressionOptions,
|
|
2014
2061
|
maxBatchSizeInBytes: runtimeOptions.maxBatchSizeInBytes,
|
|
2015
|
-
// If we disable flush before process, we must be ready to flush partial batches
|
|
2016
|
-
flushPartialBatches: this.skipSafetyFlushDuringProcessStack,
|
|
2017
2062
|
},
|
|
2018
2063
|
logger: this.mc.logger,
|
|
2019
2064
|
groupingManager: opGroupingManager,
|
|
@@ -2070,14 +2115,12 @@ export class ContainerRuntime
|
|
|
2070
2115
|
this.lastEmittedDirty = this.computeCurrentDirtyState();
|
|
2071
2116
|
context.updateDirtyContainerState(this.lastEmittedDirty);
|
|
2072
2117
|
|
|
2073
|
-
|
|
2074
|
-
|
|
2075
|
-
|
|
2076
|
-
|
|
2077
|
-
|
|
2078
|
-
|
|
2079
|
-
this.deltaManager.on("op", () => this.flush());
|
|
2080
|
-
}
|
|
2118
|
+
// Reference Sequence Number may have just changed, and it must be consistent across a batch,
|
|
2119
|
+
// so we should flush now to clear the way for the next ops.
|
|
2120
|
+
// NOTE: This will be redundant whenever CR.process was called for the op (since we flush there too) -
|
|
2121
|
+
// But we need this coverage for old loaders that don't call ContainerRuntime.process for non-runtime messages.
|
|
2122
|
+
// (We have to call flush _before_ processing a runtime op, but after is ok for non-runtime op)
|
|
2123
|
+
this.deltaManager.on("op", () => this.flush());
|
|
2081
2124
|
|
|
2082
2125
|
// logging hardware telemetry
|
|
2083
2126
|
this.baseLogger.send({
|
|
@@ -2093,7 +2136,9 @@ export class ContainerRuntime
|
|
|
2093
2136
|
summaryNumber: loadSummaryNumber,
|
|
2094
2137
|
summaryFormatVersion: metadata?.summaryFormatVersion,
|
|
2095
2138
|
disableIsolatedChannels: metadata?.disableIsolatedChannels,
|
|
2139
|
+
// This is useful even for interactive clients since they track unreferenced nodes and log errors.
|
|
2096
2140
|
gcVersion: metadata?.gcFeature,
|
|
2141
|
+
gcConfigs: this.garbageCollector.serializedConfigs,
|
|
2097
2142
|
options: JSON.stringify(runtimeOptions),
|
|
2098
2143
|
idCompressorModeMetadata: metadata?.documentSchema?.runtime?.idCompressorMode,
|
|
2099
2144
|
idCompressorMode: this.sessionSchema.idCompressorMode,
|
|
@@ -2101,7 +2146,6 @@ export class ContainerRuntime
|
|
|
2101
2146
|
featureGates: JSON.stringify({
|
|
2102
2147
|
...featureGatesForTelemetry,
|
|
2103
2148
|
closeSummarizerDelayOverride,
|
|
2104
|
-
disableFlushBeforeProcess: this.skipSafetyFlushDuringProcessStack,
|
|
2105
2149
|
}),
|
|
2106
2150
|
telemetryDocumentId: this.telemetryDocumentId,
|
|
2107
2151
|
groupedBatchingEnabled: this.groupedBatchingEnabled,
|
|
@@ -2723,12 +2767,10 @@ export class ContainerRuntime
|
|
|
2723
2767
|
this.emitDirtyDocumentEvent = false;
|
|
2724
2768
|
|
|
2725
2769
|
try {
|
|
2726
|
-
// Any ID Allocation ops that failed to submit
|
|
2727
|
-
// the
|
|
2728
|
-
//
|
|
2729
|
-
|
|
2730
|
-
this.submitIdAllocationOpIfNeeded({ resubmitOutstandingRanges: true, staged: false });
|
|
2731
|
-
this.scheduleFlush();
|
|
2770
|
+
// Any ID Allocation ops that failed to submit need to have their ranges included
|
|
2771
|
+
// in the next allocation op. Reset the compressor's unfinalized range cursor so that the next
|
|
2772
|
+
// call to takeNextCreationRange (during replay) will include those unfinalized ranges.
|
|
2773
|
+
this._idCompressor?.resetUnfinalizedCreationRange();
|
|
2732
2774
|
|
|
2733
2775
|
// replay the ops
|
|
2734
2776
|
this.pendingStateManager.replayPendingStates();
|
|
@@ -3023,10 +3065,8 @@ export class ContainerRuntime
|
|
|
3023
3065
|
|
|
3024
3066
|
this.verifyNotClosed();
|
|
3025
3067
|
|
|
3026
|
-
|
|
3027
|
-
|
|
3028
|
-
this.flush();
|
|
3029
|
-
}
|
|
3068
|
+
// Reference Sequence Number may be about to change, and it must be consistent across a batch, so flush now
|
|
3069
|
+
this.flush();
|
|
3030
3070
|
|
|
3031
3071
|
this.ensureNoDataModelChanges(() => {
|
|
3032
3072
|
this.processInboundMessageOrBatch(messageCopy, local);
|
|
@@ -3630,20 +3670,39 @@ export class ContainerRuntime
|
|
|
3630
3670
|
// since we mark whole batches as "staged" or not to indicate whether to submit them.
|
|
3631
3671
|
this.flush();
|
|
3632
3672
|
|
|
3633
|
-
const exitStagingMode = (
|
|
3673
|
+
const exitStagingMode = (
|
|
3674
|
+
discardOrCommit: () => IPendingMessage["batchInfo"][],
|
|
3675
|
+
exitMethod: "commit" | "discard",
|
|
3676
|
+
): void => {
|
|
3634
3677
|
try {
|
|
3635
|
-
|
|
3636
|
-
|
|
3637
|
-
|
|
3638
|
-
|
|
3639
|
-
|
|
3640
|
-
|
|
3641
|
-
|
|
3642
|
-
|
|
3643
|
-
|
|
3644
|
-
|
|
3645
|
-
|
|
3646
|
-
|
|
3678
|
+
PerformanceEvent.timedExec(
|
|
3679
|
+
this.mc.logger,
|
|
3680
|
+
{
|
|
3681
|
+
eventName: `ExitStagingMode_${exitMethod}`,
|
|
3682
|
+
},
|
|
3683
|
+
(event) => {
|
|
3684
|
+
// Final flush of any last staged changes
|
|
3685
|
+
// NOTE: We can't use this.flush() here, because orderSequentially uses StagingMode and in the rollback case we'll hit assert 0x24c
|
|
3686
|
+
this.outbox.flush();
|
|
3687
|
+
|
|
3688
|
+
this.stageControls = undefined;
|
|
3689
|
+
|
|
3690
|
+
// During Staging Mode, we avoid submitting any ID Allocation ops (apart from resubmitting pre-staging ops).
|
|
3691
|
+
// Now that we've exited, we need to submit an ID Allocation op for any IDs that were generated while in Staging Mode.
|
|
3692
|
+
this.submitIdAllocationOpIfNeeded({ staged: false });
|
|
3693
|
+
const batchInfos = discardOrCommit();
|
|
3694
|
+
event.reportProgress({
|
|
3695
|
+
details: {
|
|
3696
|
+
autoFlushThreshold: this.stagingModeAutoFlushThreshold,
|
|
3697
|
+
batches: batchInfos.length,
|
|
3698
|
+
batchesAtOrOverThreshold: batchInfos.filter(
|
|
3699
|
+
(b) => b.length >= this.stagingModeAutoFlushThreshold,
|
|
3700
|
+
).length,
|
|
3701
|
+
},
|
|
3702
|
+
});
|
|
3703
|
+
this.channelCollection.notifyStagingMode(false);
|
|
3704
|
+
},
|
|
3705
|
+
);
|
|
3647
3706
|
} catch (error) {
|
|
3648
3707
|
const normalizedError = normalizeError(error);
|
|
3649
3708
|
this.closeFn(normalizedError);
|
|
@@ -3655,21 +3714,24 @@ export class ContainerRuntime
|
|
|
3655
3714
|
discardChanges: () =>
|
|
3656
3715
|
exitStagingMode(() => {
|
|
3657
3716
|
// Pop all staged batches from the PSM and roll them back in LIFO order
|
|
3658
|
-
this.pendingStateManager.popStagedBatches(
|
|
3659
|
-
|
|
3660
|
-
|
|
3717
|
+
const batchInfos = this.pendingStateManager.popStagedBatches(
|
|
3718
|
+
({ runtimeOp, localOpMetadata }) => {
|
|
3719
|
+
this.rollbackStagedChange(runtimeOp, localOpMetadata);
|
|
3720
|
+
},
|
|
3721
|
+
);
|
|
3661
3722
|
this.updateDocumentDirtyState();
|
|
3662
|
-
|
|
3723
|
+
return batchInfos;
|
|
3724
|
+
}, "discard"),
|
|
3663
3725
|
commitChanges: (options) => {
|
|
3664
3726
|
const { squash } = { ...defaultStagingCommitOptions, ...options };
|
|
3665
3727
|
exitStagingMode(() => {
|
|
3666
3728
|
// Replay all staged batches in typical FIFO order.
|
|
3667
3729
|
// We'll be out of staging mode so they'll be sent to the service finally.
|
|
3668
|
-
this.pendingStateManager.replayPendingStates({
|
|
3730
|
+
return this.pendingStateManager.replayPendingStates({
|
|
3669
3731
|
committingStagedBatches: true,
|
|
3670
3732
|
squash,
|
|
3671
3733
|
});
|
|
3672
|
-
});
|
|
3734
|
+
}, "commit");
|
|
3673
3735
|
},
|
|
3674
3736
|
};
|
|
3675
3737
|
|
|
@@ -4800,6 +4862,20 @@ export class ContainerRuntime
|
|
|
4800
4862
|
}
|
|
4801
4863
|
|
|
4802
4864
|
private scheduleFlush(): void {
|
|
4865
|
+
// During staging mode, suppress automatic flush scheduling until the main batch
|
|
4866
|
+
// reaches or exceeds the threshold.
|
|
4867
|
+
// Incoming ops still break the batch via direct this.flush() calls elsewhere
|
|
4868
|
+
// (deltaManager "op" handler, process(), connection changes, getPendingLocalState,
|
|
4869
|
+
// exitStagingMode). Those all bypass scheduleFlush(), so they're unaffected by this check.
|
|
4870
|
+
// Additionally, outbox.maybeFlushPartialBatch() (called on every submit) detects
|
|
4871
|
+
// sequence number changes and throws if unexpected changes are detected.
|
|
4872
|
+
if (
|
|
4873
|
+
this.inStagingMode &&
|
|
4874
|
+
this.outbox.mainBatchMessageCount < this.stagingModeAutoFlushThreshold
|
|
4875
|
+
) {
|
|
4876
|
+
return;
|
|
4877
|
+
}
|
|
4878
|
+
|
|
4803
4879
|
if (this.flushScheduled) {
|
|
4804
4880
|
return;
|
|
4805
4881
|
}
|
|
@@ -5195,7 +5271,9 @@ export class ContainerRuntime
|
|
|
5195
5271
|
eventName: "getPendingLocalState",
|
|
5196
5272
|
},
|
|
5197
5273
|
(event) => {
|
|
5198
|
-
const pending = this.pendingStateManager.getLocalState(
|
|
5274
|
+
const { pending } = this.pendingStateManager.getLocalState(
|
|
5275
|
+
props?.snapshotSequenceNumber,
|
|
5276
|
+
);
|
|
5199
5277
|
const sessionExpiryTimerStarted =
|
|
5200
5278
|
props?.sessionExpiryTimerStarted ?? this.garbageCollector.sessionExpiryTimerStarted;
|
|
5201
5279
|
|
|
@@ -100,6 +100,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
100
100
|
|
|
101
101
|
private readonly configs: IGarbageCollectorConfigs;
|
|
102
102
|
|
|
103
|
+
public get serializedConfigs(): string {
|
|
104
|
+
return JSON.stringify(this.configs);
|
|
105
|
+
}
|
|
106
|
+
|
|
103
107
|
public get shouldRunGC(): boolean {
|
|
104
108
|
return this.configs.gcAllowed;
|
|
105
109
|
}
|
|
@@ -334,15 +338,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
334
338
|
|
|
335
339
|
return { gcData: { gcNodes }, usedRoutes };
|
|
336
340
|
});
|
|
337
|
-
|
|
338
|
-
// Log all the GC options and the state determined by the garbage collector.
|
|
339
|
-
// This is useful even for interactive clients since they track unreferenced nodes and log errors.
|
|
340
|
-
this.mc.logger.sendTelemetryEvent({
|
|
341
|
-
eventName: "GarbageCollectorLoaded",
|
|
342
|
-
gcConfigs: JSON.stringify(this.configs),
|
|
343
|
-
gcOptions: JSON.stringify(createParams.gcOptions),
|
|
344
|
-
...createParams.createContainerMetadata,
|
|
345
|
-
});
|
|
346
341
|
}
|
|
347
342
|
|
|
348
343
|
/**
|
package/src/gc/gcDefinitions.ts
CHANGED
|
@@ -373,6 +373,10 @@ export interface IGarbageCollectionRuntime {
|
|
|
373
373
|
* Defines the contract for the garbage collector.
|
|
374
374
|
*/
|
|
375
375
|
export interface IGarbageCollector {
|
|
376
|
+
/**
|
|
377
|
+
* The GC configurations serialized as a JSON string for telemetry.
|
|
378
|
+
*/
|
|
379
|
+
readonly serializedConfigs: string;
|
|
376
380
|
/**
|
|
377
381
|
* Tells the time at which session expiry timer started in a previous container.
|
|
378
382
|
* This is only set when loading from a stashed container and will be equal to the
|
package/src/index.ts
CHANGED
|
@@ -162,7 +162,8 @@ export class BatchManager {
|
|
|
162
162
|
throw new LoggingError("Ops generated during rollback", {
|
|
163
163
|
count,
|
|
164
164
|
...tagData(TelemetryDataTag.UserData, {
|
|
165
|
-
ops: serializeOp(this.pendingBatch.slice(startPoint).map((b) => b.runtimeOp))
|
|
165
|
+
ops: serializeOp(this.pendingBatch.slice(startPoint).map((b) => b.runtimeOp))
|
|
166
|
+
.content,
|
|
166
167
|
}),
|
|
167
168
|
});
|
|
168
169
|
}
|
package/src/opLifecycle/index.ts
CHANGED
|
@@ -17,6 +17,13 @@ import type {
|
|
|
17
17
|
OutboundSingletonBatch,
|
|
18
18
|
} from "./definitions.js";
|
|
19
19
|
|
|
20
|
+
/**
|
|
21
|
+
* The number of ops in a batch above which the batch is considered "large"
|
|
22
|
+
* for telemetry purposes. Used by both {@link OpGroupingManager} (GroupLargeBatch event)
|
|
23
|
+
* and as the default staging-mode auto-flush threshold.
|
|
24
|
+
*/
|
|
25
|
+
export const largeBatchThreshold = 1000;
|
|
26
|
+
|
|
20
27
|
/**
|
|
21
28
|
* Grouping makes assumptions about the shape of message contents. This interface codifies those assumptions, but does not validate them.
|
|
22
29
|
*/
|
|
@@ -123,7 +130,10 @@ export class OpGroupingManager {
|
|
|
123
130
|
return batch as OutboundSingletonBatch;
|
|
124
131
|
}
|
|
125
132
|
|
|
126
|
-
|
|
133
|
+
// Use > (not >=) so that batches flushed exactly at the staging-mode
|
|
134
|
+
// auto-flush threshold (which defaults to largeBatchThreshold) don't
|
|
135
|
+
// trigger this event. Only genuinely oversized batches are logged.
|
|
136
|
+
if (batch.messages.length > largeBatchThreshold) {
|
|
127
137
|
this.logger.sendTelemetryEvent({
|
|
128
138
|
eventName: "GroupLargeBatch",
|
|
129
139
|
length: batch.messages.length,
|
|
@@ -40,16 +40,18 @@ export function serializeOp(
|
|
|
40
40
|
| EmptyGroupedBatch
|
|
41
41
|
| LocalContainerRuntimeMessage
|
|
42
42
|
| LocalContainerRuntimeMessage[],
|
|
43
|
-
): string {
|
|
44
|
-
return
|
|
45
|
-
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
43
|
+
): { content: string } {
|
|
44
|
+
return {
|
|
45
|
+
content: JSON.stringify(
|
|
46
|
+
toSerialize,
|
|
47
|
+
// replacer:
|
|
48
|
+
(key, value: unknown) => {
|
|
49
|
+
// If 'value' is an IFluidHandle return its encoded form.
|
|
50
|
+
if (isFluidHandle(value)) {
|
|
51
|
+
return encodeHandleForSerialization(toFluidHandleInternal(value));
|
|
52
|
+
}
|
|
53
|
+
return value;
|
|
54
|
+
},
|
|
55
|
+
),
|
|
56
|
+
};
|
|
55
57
|
}
|
|
@@ -48,12 +48,6 @@ export interface IOutboxConfig {
|
|
|
48
48
|
* The maximum size of a batch that we can send over the wire.
|
|
49
49
|
*/
|
|
50
50
|
readonly maxBatchSizeInBytes: number;
|
|
51
|
-
/**
|
|
52
|
-
* If true, maybeFlushPartialBatch will flush the batch if the reference sequence number changed
|
|
53
|
-
* since the batch started. Otherwise, it will throw in this case (apart from reentrancy which is handled elsewhere).
|
|
54
|
-
* Once the new throw-based flow is proved in a production environment, this option will be removed.
|
|
55
|
-
*/
|
|
56
|
-
readonly flushPartialBatches: boolean;
|
|
57
51
|
}
|
|
58
52
|
|
|
59
53
|
export interface IOutboxParameters {
|
|
@@ -142,7 +136,7 @@ export function localBatchToOutboundBatch({
|
|
|
142
136
|
// Shallow copy each message as we switch types
|
|
143
137
|
const outboundMessages = localBatch.messages.map<OutboundBatchMessage>(
|
|
144
138
|
({ runtimeOp, ...message }) => ({
|
|
145
|
-
contents: serializeOp(runtimeOp),
|
|
139
|
+
contents: serializeOp(runtimeOp).content,
|
|
146
140
|
...message,
|
|
147
141
|
}),
|
|
148
142
|
);
|
|
@@ -295,10 +289,7 @@ export class Outbox {
|
|
|
295
289
|
this.logger.sendTelemetryEvent(
|
|
296
290
|
{
|
|
297
291
|
// Only log error if this is truly unexpected
|
|
298
|
-
category:
|
|
299
|
-
expectedDueToReentrancy || this.params.config.flushPartialBatches
|
|
300
|
-
? "generic"
|
|
301
|
-
: "error",
|
|
292
|
+
category: expectedDueToReentrancy ? "generic" : "error",
|
|
302
293
|
eventName: "ReferenceSequenceNumberMismatch",
|
|
303
294
|
details: {
|
|
304
295
|
expectedDueToReentrancy,
|
|
@@ -314,12 +305,6 @@ export class Outbox {
|
|
|
314
305
|
);
|
|
315
306
|
}
|
|
316
307
|
|
|
317
|
-
// If we're configured to flush partial batches, do that now and return (don't throw)
|
|
318
|
-
if (this.params.config.flushPartialBatches) {
|
|
319
|
-
this.flushAll();
|
|
320
|
-
return;
|
|
321
|
-
}
|
|
322
|
-
|
|
323
308
|
// If we are in a reentrant context, we know this can happen without causing any harm.
|
|
324
309
|
if (expectedDueToReentrancy) {
|
|
325
310
|
return;
|
package/src/packageVersion.ts
CHANGED