@fluidframework/container-runtime 0.57.0-51086 → 0.58.0-55561
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/batchTracker.d.ts +26 -0
- package/dist/batchTracker.d.ts.map +1 -0
- package/dist/batchTracker.js +59 -0
- package/dist/batchTracker.js.map +1 -0
- package/dist/containerRuntime.d.ts +12 -7
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +125 -55
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStore.d.ts +1 -36
- package/dist/dataStore.d.ts.map +1 -1
- package/dist/dataStore.js +5 -27
- package/dist/dataStore.js.map +1 -1
- package/dist/dataStoreContext.d.ts +5 -7
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +12 -7
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +1 -1
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +3 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +25 -11
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +100 -57
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +0 -1
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -3
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +1 -6
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runningSummarizer.d.ts +1 -1
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +1 -1
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/summarizer.d.ts +3 -4
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +8 -9
- package/dist/summarizer.js.map +1 -1
- package/dist/summaryGenerator.d.ts +1 -1
- package/dist/summaryGenerator.d.ts.map +1 -1
- package/dist/summaryGenerator.js +1 -1
- package/dist/summaryGenerator.js.map +1 -1
- package/dist/summaryManager.d.ts +2 -6
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +4 -10
- package/dist/summaryManager.js.map +1 -1
- package/lib/batchTracker.d.ts +26 -0
- package/lib/batchTracker.d.ts.map +1 -0
- package/lib/batchTracker.js +54 -0
- package/lib/batchTracker.js.map +1 -0
- package/lib/containerRuntime.d.ts +12 -7
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +126 -56
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStore.d.ts +1 -36
- package/lib/dataStore.d.ts.map +1 -1
- package/lib/dataStore.js +4 -26
- package/lib/dataStore.js.map +1 -1
- package/lib/dataStoreContext.d.ts +5 -7
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +13 -8
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +1 -1
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +3 -3
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +25 -11
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +98 -55
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +0 -1
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +0 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +1 -6
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runningSummarizer.d.ts +1 -1
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +1 -1
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/summarizer.d.ts +3 -4
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +8 -9
- package/lib/summarizer.js.map +1 -1
- package/lib/summaryGenerator.d.ts +1 -1
- package/lib/summaryGenerator.d.ts.map +1 -1
- package/lib/summaryGenerator.js +1 -1
- package/lib/summaryGenerator.js.map +1 -1
- package/lib/summaryManager.d.ts +2 -6
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +5 -11
- package/lib/summaryManager.js.map +1 -1
- package/package.json +12 -12
- package/src/batchTracker.ts +80 -0
- package/src/containerRuntime.ts +180 -63
- package/src/dataStore.ts +6 -42
- package/src/dataStoreContext.ts +17 -15
- package/src/dataStores.ts +9 -4
- package/src/garbageCollection.ts +125 -67
- package/src/index.ts +0 -1
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +4 -8
- package/src/runningSummarizer.ts +3 -3
- package/src/summarizer.ts +8 -8
- package/src/summaryGenerator.ts +2 -2
- package/src/summaryManager.ts +5 -20
package/src/dataStores.ts
CHANGED
|
@@ -91,7 +91,8 @@ export class DataStores implements IDisposable {
|
|
|
91
91
|
private readonly deleteChildSummarizerNodeFn: (id: string) => void,
|
|
92
92
|
baseLogger: ITelemetryBaseLogger,
|
|
93
93
|
getBaseGCDetails: () => Promise<Map<string, IGarbageCollectionDetailsBase>>,
|
|
94
|
-
private readonly dataStoreChanged: (
|
|
94
|
+
private readonly dataStoreChanged: (
|
|
95
|
+
dataStorePath: string, timestampMs: number, packagePath?: readonly string[]) => void,
|
|
95
96
|
private readonly aliasMap: Map<string, string>,
|
|
96
97
|
private readonly writeGCDataAtRoot: boolean,
|
|
97
98
|
private readonly contexts: DataStoreContexts = new DataStoreContexts(baseLogger),
|
|
@@ -193,7 +194,7 @@ export class DataStores implements IDisposable {
|
|
|
193
194
|
if (this.alreadyProcessed(attachMessage.id)) {
|
|
194
195
|
// TODO: dataStoreId may require a different tag from PackageData #7488
|
|
195
196
|
const error = new DataCorruptionError(
|
|
196
|
-
"
|
|
197
|
+
"Duplicate DataStore created with existing id",
|
|
197
198
|
{
|
|
198
199
|
...extractSafePropertiesFromMessage(message),
|
|
199
200
|
dataStoreId: {
|
|
@@ -387,7 +388,11 @@ export class DataStores implements IDisposable {
|
|
|
387
388
|
context.process(transformed, local, localMessageMetadata);
|
|
388
389
|
|
|
389
390
|
// Notify that a data store changed. This is used to detect if a deleted data store is being used.
|
|
390
|
-
this.dataStoreChanged(
|
|
391
|
+
this.dataStoreChanged(
|
|
392
|
+
`/${envelope.address}`,
|
|
393
|
+
message.timestamp,
|
|
394
|
+
context.isLoaded ? context.packagePath : undefined,
|
|
395
|
+
);
|
|
391
396
|
}
|
|
392
397
|
|
|
393
398
|
public async getDataStore(id: string, wait: boolean): Promise<FluidDataStoreContext> {
|
|
@@ -612,7 +617,7 @@ export class DataStores implements IDisposable {
|
|
|
612
617
|
// Currently, only return the data store package path for the node since GC is only interested in data stores.
|
|
613
618
|
const dataStoreId = nodePath.split("/")[1];
|
|
614
619
|
const context = this.contexts.get(dataStoreId);
|
|
615
|
-
assert(context !== undefined, "Data store with given id does not exist");
|
|
620
|
+
assert(context !== undefined, 0x2b9 /* "Data store with given id does not exist" */);
|
|
616
621
|
return context.isLoaded ? context.packagePath : undefined;
|
|
617
622
|
}
|
|
618
623
|
}
|
package/src/garbageCollection.ts
CHANGED
|
@@ -69,7 +69,7 @@ const writeAtRootKey = "Fluid.GarbageCollection.WriteDataAtRoot";
|
|
|
69
69
|
const runSessionExpiry = "Fluid.GarbageCollection.RunSessionExpiry";
|
|
70
70
|
|
|
71
71
|
const defaultDeleteTimeoutMs = 7 * 24 * 60 * 60 * 1000; // 7 days
|
|
72
|
-
const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
72
|
+
export const defaultSessionExpiryDurationMs = 30 * 24 * 60 * 60 * 1000; // 30 days
|
|
73
73
|
|
|
74
74
|
/** The statistics of the system state after a garbage collection run. */
|
|
75
75
|
export interface IGCStats {
|
|
@@ -93,6 +93,7 @@ interface IUnreferencedEvent {
|
|
|
93
93
|
id: string;
|
|
94
94
|
age: number;
|
|
95
95
|
timeout: number;
|
|
96
|
+
lastSummaryTime?: number;
|
|
96
97
|
externalRequest?: boolean;
|
|
97
98
|
viaHandle?: boolean;
|
|
98
99
|
};
|
|
@@ -105,6 +106,8 @@ export interface IGarbageCollectionRuntime {
|
|
|
105
106
|
getGCData(fullGC?: boolean): Promise<IGarbageCollectionData>;
|
|
106
107
|
/** After GC has run, called to notify the runtime of routes that are used in it. */
|
|
107
108
|
updateUsedRoutes(usedRoutes: string[], gcTimestamp?: number): void;
|
|
109
|
+
/** Called when the runtime should close because of an error. */
|
|
110
|
+
closeFn(error?: ICriticalContainerError): void;
|
|
108
111
|
}
|
|
109
112
|
|
|
110
113
|
/** Defines the contract for the garbage collector. */
|
|
@@ -137,6 +140,7 @@ export interface IGarbageCollector {
|
|
|
137
140
|
nodeUpdated(
|
|
138
141
|
nodePath: string,
|
|
139
142
|
reason: "Loaded" | "Changed",
|
|
143
|
+
timestampMs?: number,
|
|
140
144
|
packagePath?: readonly string[],
|
|
141
145
|
requestHeaders?: IRequestHeader,
|
|
142
146
|
): void;
|
|
@@ -155,23 +159,43 @@ class UnreferencedStateTracker {
|
|
|
155
159
|
return this._inactive;
|
|
156
160
|
}
|
|
157
161
|
|
|
158
|
-
private
|
|
162
|
+
private timer: Timer | undefined;
|
|
159
163
|
|
|
160
164
|
constructor(
|
|
161
165
|
public readonly unreferencedTimestampMs: number,
|
|
162
|
-
inactiveTimeoutMs: number,
|
|
166
|
+
private readonly inactiveTimeoutMs: number,
|
|
167
|
+
currentReferenceTimestampMs?: number,
|
|
163
168
|
) {
|
|
164
|
-
// If
|
|
165
|
-
//
|
|
166
|
-
if (
|
|
169
|
+
// If there is no current reference timestamp, don't track the node's inactive state. This will happen later
|
|
170
|
+
// when updateTracking is called with a reference timestamp.
|
|
171
|
+
if (currentReferenceTimestampMs !== undefined) {
|
|
172
|
+
this.updateTracking(currentReferenceTimestampMs);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
/**
|
|
177
|
+
* Updates the tracking state based on the provided timestamp.
|
|
178
|
+
*/
|
|
179
|
+
public updateTracking(currentReferenceTimestampMs: number) {
|
|
180
|
+
const unreferencedDurationMs = currentReferenceTimestampMs - this.unreferencedTimestampMs;
|
|
181
|
+
// If the timeout has already expired, the node has become inactive.
|
|
182
|
+
if (unreferencedDurationMs > this.inactiveTimeoutMs) {
|
|
167
183
|
this._inactive = true;
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
184
|
+
this.timer?.clear();
|
|
185
|
+
return;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
// The node isn't inactive yet. Restart a timer for the duration remaining for it to become inactive.
|
|
189
|
+
const remainingDurationMs = this.inactiveTimeoutMs - unreferencedDurationMs;
|
|
190
|
+
if (this.timer === undefined) {
|
|
191
|
+
this.timer = new Timer(remainingDurationMs, () => { this._inactive = true; });
|
|
171
192
|
}
|
|
193
|
+
this.timer.restart(remainingDurationMs);
|
|
172
194
|
}
|
|
173
195
|
|
|
174
|
-
/**
|
|
196
|
+
/**
|
|
197
|
+
* Stop tracking this node. Reset the unreferenced timer, if any, and reset inactive state.
|
|
198
|
+
*/
|
|
175
199
|
public stopTracking() {
|
|
176
200
|
this.timer?.clear();
|
|
177
201
|
this._inactive = false;
|
|
@@ -188,8 +212,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
188
212
|
gcOptions: IGCRuntimeOptions,
|
|
189
213
|
deleteUnusedRoutes: (unusedRoutes: string[]) => void,
|
|
190
214
|
getNodePackagePath: (nodeId: string) => readonly string[] | undefined,
|
|
191
|
-
|
|
192
|
-
|
|
215
|
+
getCurrentReferenceTimestampMs: () => number | undefined,
|
|
216
|
+
getLastSummaryTimestampMs: () => number | undefined,
|
|
193
217
|
baseSnapshot: ISnapshotTree | undefined,
|
|
194
218
|
readAndParseBlob: ReadAndParseBlob,
|
|
195
219
|
baseLogger: ITelemetryLogger,
|
|
@@ -201,8 +225,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
201
225
|
gcOptions,
|
|
202
226
|
deleteUnusedRoutes,
|
|
203
227
|
getNodePackagePath,
|
|
204
|
-
|
|
205
|
-
|
|
228
|
+
getCurrentReferenceTimestampMs,
|
|
229
|
+
getLastSummaryTimestampMs,
|
|
206
230
|
baseSnapshot,
|
|
207
231
|
readAndParseBlob,
|
|
208
232
|
baseLogger,
|
|
@@ -307,9 +331,14 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
307
331
|
private readonly deleteUnusedRoutes: (unusedRoutes: string[]) => void,
|
|
308
332
|
/** For a given node path, returns the node's package path. */
|
|
309
333
|
private readonly getNodePackagePath: (nodePath: string) => readonly string[] | undefined,
|
|
310
|
-
/**
|
|
311
|
-
|
|
312
|
-
|
|
334
|
+
/**
|
|
335
|
+
* Returns a referenced timestamp to be used to track unreferenced nodes. This is a server generated timestamp
|
|
336
|
+
* and may not be available if there aren't any ops processed yet. If so, we skip tracking unreferenced state
|
|
337
|
+
* such as time when node becomes unreferenced or inactive.
|
|
338
|
+
*/
|
|
339
|
+
private readonly getCurrentReferenceTimestampMs: () => number | undefined,
|
|
340
|
+
/** Returns the timestamp of the last summary generated for this container. */
|
|
341
|
+
private readonly getLastSummaryTimestampMs: () => number | undefined,
|
|
313
342
|
baseSnapshot: ISnapshotTree | undefined,
|
|
314
343
|
readAndParseBlob: ReadAndParseBlob,
|
|
315
344
|
baseLogger: ITelemetryLogger,
|
|
@@ -346,7 +375,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
346
375
|
const timeoutMs = this.sessionExpiryTimeoutMs;
|
|
347
376
|
setLongTimeout(timeoutMs,
|
|
348
377
|
() => {
|
|
349
|
-
this.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
378
|
+
this.provider.closeFn(new ClientSessionExpiredError(`Client session expired.`, timeoutMs));
|
|
350
379
|
},
|
|
351
380
|
(timer) => {
|
|
352
381
|
this.sessionExpiryTimer = timer;
|
|
@@ -448,10 +477,13 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
448
477
|
return Object.keys(gcState.gcNodes).length === 1 ? undefined : gcState;
|
|
449
478
|
});
|
|
450
479
|
|
|
451
|
-
|
|
452
|
-
|
|
480
|
+
/**
|
|
481
|
+
* Set up the initializer which initializes the base GC state from the base snapshot. Note that the reference
|
|
482
|
+
* timestamp maybe from old ops which were not summarized and stored in the file. So, the unreferenced state
|
|
483
|
+
* may be out of date. This is fine because the state is updated every time GC runs based on the time then.
|
|
484
|
+
*/
|
|
453
485
|
this.initializeBaseStateP = new LazyPromise<void>(async () => {
|
|
454
|
-
const
|
|
486
|
+
const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
|
|
455
487
|
const baseState = await baseSummaryStateP;
|
|
456
488
|
if (baseState === undefined) {
|
|
457
489
|
return;
|
|
@@ -459,16 +491,13 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
459
491
|
|
|
460
492
|
const gcNodes: { [ id: string ]: string[] } = {};
|
|
461
493
|
for (const [nodeId, nodeData] of Object.entries(baseState.gcNodes)) {
|
|
462
|
-
|
|
463
|
-
if (unreferencedTimestampMs !== undefined) {
|
|
464
|
-
// Get how long it has been since the node was unreferenced. Start a timeout for the remaining time
|
|
465
|
-
// left for it to be eligible for deletion.
|
|
466
|
-
const unreferencedDurationMs = currentTimestampMs - unreferencedTimestampMs;
|
|
494
|
+
if (nodeData.unreferencedTimestampMs !== undefined) {
|
|
467
495
|
this.unreferencedNodesState.set(
|
|
468
496
|
nodeId,
|
|
469
497
|
new UnreferencedStateTracker(
|
|
470
|
-
unreferencedTimestampMs,
|
|
471
|
-
this.deleteTimeoutMs
|
|
498
|
+
nodeData.unreferencedTimestampMs,
|
|
499
|
+
this.deleteTimeoutMs,
|
|
500
|
+
currentReferenceTimestampMs,
|
|
472
501
|
),
|
|
473
502
|
);
|
|
474
503
|
}
|
|
@@ -516,17 +545,18 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
516
545
|
// used in the container.
|
|
517
546
|
if (this.shouldRunGC) {
|
|
518
547
|
this.initializeBaseStateP.catch((error) => {
|
|
519
|
-
|
|
520
|
-
error
|
|
548
|
+
const dpe = DataProcessingError.wrapIfUnrecognized(
|
|
549
|
+
error,
|
|
521
550
|
"FailedToInitializeGC",
|
|
522
|
-
{
|
|
523
|
-
gcEnabled: this.gcEnabled,
|
|
524
|
-
runSweep: this.shouldRunSweep,
|
|
525
|
-
writeAtRoot: this._writeDataAtRoot,
|
|
526
|
-
testMode: this.testMode,
|
|
527
|
-
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
528
|
-
},
|
|
529
551
|
);
|
|
552
|
+
dpe.addTelemetryProperties({
|
|
553
|
+
gcEnabled: this.gcEnabled,
|
|
554
|
+
runSweep: this.shouldRunSweep,
|
|
555
|
+
writeAtRoot: this._writeDataAtRoot,
|
|
556
|
+
testMode: this.testMode,
|
|
557
|
+
sessionExpiry: this.sessionExpiryTimeoutMs,
|
|
558
|
+
});
|
|
559
|
+
throw dpe;
|
|
530
560
|
});
|
|
531
561
|
}
|
|
532
562
|
}
|
|
@@ -571,10 +601,10 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
571
601
|
this.updateStateSinceLastRun(gcData);
|
|
572
602
|
|
|
573
603
|
// Update the current state of the system based on the GC run.
|
|
574
|
-
const
|
|
575
|
-
this.updateCurrentState(gcData, gcResult,
|
|
604
|
+
const currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs();
|
|
605
|
+
this.updateCurrentState(gcData, gcResult, currentReferenceTimestampMs);
|
|
576
606
|
|
|
577
|
-
this.provider.updateUsedRoutes(gcResult.referencedNodeIds,
|
|
607
|
+
this.provider.updateUsedRoutes(gcResult.referencedNodeIds, currentReferenceTimestampMs);
|
|
578
608
|
|
|
579
609
|
if (runSweep) {
|
|
580
610
|
// Placeholder for running sweep logic.
|
|
@@ -653,12 +683,14 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
653
683
|
* Called when a node with the given id is updated. If the node is inactive, log an error.
|
|
654
684
|
* @param nodePath - The id of the node that changed.
|
|
655
685
|
* @param reason - Whether the node was loaded or changed.
|
|
686
|
+
* @param timestampMs - The timestamp when the node changed.
|
|
656
687
|
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
657
688
|
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
658
689
|
*/
|
|
659
690
|
public nodeUpdated(
|
|
660
691
|
nodePath: string,
|
|
661
692
|
reason: "Loaded" | "Changed",
|
|
693
|
+
timestampMs?: number,
|
|
662
694
|
packagePath?: readonly string[],
|
|
663
695
|
requestHeaders?: IRequestHeader,
|
|
664
696
|
) {
|
|
@@ -669,7 +701,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
669
701
|
this.logIfInactive(
|
|
670
702
|
reason,
|
|
671
703
|
nodePath,
|
|
672
|
-
|
|
704
|
+
timestampMs,
|
|
673
705
|
packagePath,
|
|
674
706
|
requestHeaders,
|
|
675
707
|
);
|
|
@@ -695,8 +727,6 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
695
727
|
this.logIfInactive(
|
|
696
728
|
"Revived",
|
|
697
729
|
toNodePath,
|
|
698
|
-
this.getCurrentTimestampMs(),
|
|
699
|
-
this.getNodePackagePath(toNodePath),
|
|
700
730
|
);
|
|
701
731
|
}
|
|
702
732
|
|
|
@@ -725,28 +755,16 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
725
755
|
* 3. Clears tracking for nodes that were unreferenced but became referenced in this run.
|
|
726
756
|
* @param gcData - The data representing the reference graph on which GC is run.
|
|
727
757
|
* @param gcResult - The result of the GC run on the gcData.
|
|
728
|
-
* @param
|
|
758
|
+
* @param currentReferenceTimestampMs - The timestamp to be used for unreferenced nodes' timestamp.
|
|
729
759
|
*/
|
|
730
|
-
private updateCurrentState(
|
|
760
|
+
private updateCurrentState(
|
|
761
|
+
gcData: IGarbageCollectionData,
|
|
762
|
+
gcResult: IGCResult,
|
|
763
|
+
currentReferenceTimestampMs?: number,
|
|
764
|
+
) {
|
|
731
765
|
this.gcDataFromLastRun = cloneGCData(gcData);
|
|
732
766
|
this.referencesSinceLastRun.clear();
|
|
733
767
|
|
|
734
|
-
// Iterate through the deleted nodes and start tracking if they became unreferenced in this run.
|
|
735
|
-
for (const nodeId of gcResult.deletedNodeIds) {
|
|
736
|
-
// The time when the node became unreferenced. This is added to the current GC state.
|
|
737
|
-
let unreferencedTimestampMs: number = currentTimestampMs;
|
|
738
|
-
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
739
|
-
if (nodeStateTracker !== undefined) {
|
|
740
|
-
unreferencedTimestampMs = nodeStateTracker.unreferencedTimestampMs;
|
|
741
|
-
} else {
|
|
742
|
-
// Start tracking this node as it became unreferenced in this run.
|
|
743
|
-
this.unreferencedNodesState.set(
|
|
744
|
-
nodeId,
|
|
745
|
-
new UnreferencedStateTracker(unreferencedTimestampMs, this.deleteTimeoutMs),
|
|
746
|
-
);
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
|
|
750
768
|
// Iterate through the referenced nodes and stop tracking if they were unreferenced before.
|
|
751
769
|
for (const nodeId of gcResult.referencedNodeIds) {
|
|
752
770
|
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
@@ -757,6 +775,36 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
757
775
|
this.unreferencedNodesState.delete(nodeId);
|
|
758
776
|
}
|
|
759
777
|
}
|
|
778
|
+
|
|
779
|
+
/**
|
|
780
|
+
* If there is no current reference time, skip tracking when a node becomes unreferenced. This would happen
|
|
781
|
+
* if no ops have been processed ever and we still try to run GC. If so, there is nothing interesting to track
|
|
782
|
+
* anyway.
|
|
783
|
+
*/
|
|
784
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
785
|
+
return;
|
|
786
|
+
}
|
|
787
|
+
|
|
788
|
+
/**
|
|
789
|
+
* If a node became unreferenced in this run, start tracking it.
|
|
790
|
+
* If a node was already unreferenced, update its tracking information. Since the current reference time is
|
|
791
|
+
* from the ops seen, this will ensure that we keep updating the unreferenced state as time moves forward.
|
|
792
|
+
*/
|
|
793
|
+
for (const nodeId of gcResult.deletedNodeIds) {
|
|
794
|
+
const nodeStateTracker = this.unreferencedNodesState.get(nodeId);
|
|
795
|
+
if (nodeStateTracker === undefined) {
|
|
796
|
+
this.unreferencedNodesState.set(
|
|
797
|
+
nodeId,
|
|
798
|
+
new UnreferencedStateTracker(
|
|
799
|
+
currentReferenceTimestampMs,
|
|
800
|
+
this.deleteTimeoutMs,
|
|
801
|
+
currentReferenceTimestampMs,
|
|
802
|
+
),
|
|
803
|
+
);
|
|
804
|
+
} else {
|
|
805
|
+
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
806
|
+
}
|
|
807
|
+
}
|
|
760
808
|
}
|
|
761
809
|
|
|
762
810
|
/**
|
|
@@ -939,16 +987,21 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
939
987
|
}
|
|
940
988
|
|
|
941
989
|
/**
|
|
942
|
-
* Logs an event if a node is inactive and is used.
|
|
943
|
-
* queue it and it will be logged the next time GC runs as the package data should be available then.
|
|
990
|
+
* Logs an event if a node is inactive and is used.
|
|
944
991
|
*/
|
|
945
992
|
private logIfInactive(
|
|
946
993
|
eventSuffix: "Changed" | "Loaded" | "Revived",
|
|
947
994
|
nodeId: string,
|
|
948
|
-
|
|
995
|
+
currentReferenceTimestampMs = this.getCurrentReferenceTimestampMs(),
|
|
949
996
|
packagePath?: readonly string[],
|
|
950
997
|
requestHeaders?: IRequestHeader,
|
|
951
998
|
) {
|
|
999
|
+
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
1000
|
+
// logging as nothing interesting would have happened worth logging.
|
|
1001
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
1002
|
+
return;
|
|
1003
|
+
}
|
|
1004
|
+
|
|
952
1005
|
const eventName = `inactiveObject_${eventSuffix}`;
|
|
953
1006
|
// We log a particular event for a given node only once so that it is not too noisy.
|
|
954
1007
|
const uniqueEventId = `${nodeId}-${eventName}`;
|
|
@@ -958,15 +1011,20 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
958
1011
|
const event: IUnreferencedEvent = {
|
|
959
1012
|
eventName,
|
|
960
1013
|
id: nodeId,
|
|
961
|
-
age:
|
|
1014
|
+
age: currentReferenceTimestampMs - nodeState.unreferencedTimestampMs,
|
|
962
1015
|
timeout: this.deleteTimeoutMs,
|
|
1016
|
+
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
963
1017
|
externalRequest: requestHeaders?.[RuntimeHeaders.externalRequest],
|
|
964
1018
|
viaHandle: requestHeaders?.[RuntimeHeaders.viaHandle],
|
|
965
1019
|
};
|
|
966
|
-
|
|
1020
|
+
|
|
1021
|
+
// If the package data for the node exists, log immediately. Otherwise, queue it and it will be logged the
|
|
1022
|
+
// next time GC runs as the package data should be available then.
|
|
1023
|
+
const pkg = packagePath ?? this.getNodePackagePath(nodeId);
|
|
1024
|
+
if (pkg !== undefined) {
|
|
967
1025
|
this.mc.logger.sendErrorEvent({
|
|
968
1026
|
...event,
|
|
969
|
-
pkg: { value: `/${
|
|
1027
|
+
pkg: { value: `/${pkg.join("/")}`, tag: TelemetryDataTag.PackageData },
|
|
970
1028
|
});
|
|
971
1029
|
} else {
|
|
972
1030
|
this.pendingEventsQueue.push(event);
|
package/src/index.ts
CHANGED
|
@@ -19,7 +19,6 @@ export {
|
|
|
19
19
|
ContainerRuntime,
|
|
20
20
|
RuntimeHeaders,
|
|
21
21
|
} from "./containerRuntime";
|
|
22
|
-
export { IDataStore, AliasResult } from "./dataStore";
|
|
23
22
|
export { DeltaScheduler } from "./deltaScheduler";
|
|
24
23
|
export { FluidDataStoreRegistry } from "./dataStoreRegistry";
|
|
25
24
|
export {
|
package/src/packageVersion.ts
CHANGED
|
@@ -339,15 +339,11 @@ export class PendingStateManager implements IDisposable {
|
|
|
339
339
|
// The clientSequenceNumber of the incoming message must match that of the pending message.
|
|
340
340
|
if (pendingState.clientSequenceNumber !== message.clientSequenceNumber) {
|
|
341
341
|
// Close the container because this could indicate data corruption.
|
|
342
|
-
const error =
|
|
342
|
+
const error = DataProcessingError.create(
|
|
343
|
+
"pending local message clientSequenceNumber mismatch",
|
|
343
344
|
"unexpectedAckReceived",
|
|
344
|
-
|
|
345
|
-
{
|
|
346
|
-
clientId: message.clientId,
|
|
347
|
-
sequenceNumber: message.sequenceNumber,
|
|
348
|
-
clientSequenceNumber: message.clientSequenceNumber,
|
|
349
|
-
expectedClientSequenceNumber: pendingState.clientSequenceNumber,
|
|
350
|
-
},
|
|
345
|
+
message,
|
|
346
|
+
{ expectedClientSequenceNumber: pendingState.clientSequenceNumber },
|
|
351
347
|
);
|
|
352
348
|
|
|
353
349
|
this.containerRuntime.closeFn(error);
|
package/src/runningSummarizer.ts
CHANGED
|
@@ -51,7 +51,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
51
51
|
configuration: ISummaryConfiguration,
|
|
52
52
|
submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,
|
|
53
53
|
heuristicData: ISummarizeHeuristicData,
|
|
54
|
-
raiseSummarizingError: (
|
|
54
|
+
raiseSummarizingError: (errorMessage: string) => void,
|
|
55
55
|
summaryCollection: SummaryCollection,
|
|
56
56
|
cancellationToken: ISummaryCancellationToken,
|
|
57
57
|
stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
@@ -101,7 +101,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
101
101
|
private readonly configuration: ISummaryConfiguration,
|
|
102
102
|
private readonly submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,
|
|
103
103
|
private readonly heuristicData: ISummarizeHeuristicData,
|
|
104
|
-
private readonly raiseSummarizingError: (
|
|
104
|
+
private readonly raiseSummarizingError: (errorMessage: string) => void,
|
|
105
105
|
private readonly summaryCollection: SummaryCollection,
|
|
106
106
|
private readonly cancellationToken: ISummaryCancellationToken,
|
|
107
107
|
private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
@@ -131,7 +131,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
131
131
|
this.pendingAckTimer = new PromiseTimer(
|
|
132
132
|
maxAckWaitTime,
|
|
133
133
|
() => {
|
|
134
|
-
this.raiseSummarizingError("
|
|
134
|
+
this.raiseSummarizingError("Pending summary ack not received in time");
|
|
135
135
|
// Note: summarizeCount (from ChildLogger definition) may be 0,
|
|
136
136
|
// since this code path is hit when RunningSummarizer first starts up,
|
|
137
137
|
// before this instance has kicked off a new summarize run.
|
package/src/summarizer.ts
CHANGED
|
@@ -46,20 +46,19 @@ export class SummarizingWarning extends LoggingError implements ISummarizingWarn
|
|
|
46
46
|
|
|
47
47
|
constructor(
|
|
48
48
|
errorMessage: string,
|
|
49
|
-
readonly fluidErrorCode: string,
|
|
50
49
|
readonly logged: boolean = false,
|
|
51
50
|
) {
|
|
52
51
|
super(errorMessage);
|
|
53
52
|
}
|
|
54
53
|
|
|
55
|
-
static wrap(error: any,
|
|
56
|
-
const newErrorFn = (errMsg: string) => new SummarizingWarning(errMsg,
|
|
54
|
+
static wrap(error: any, logged: boolean = false, logger: ITelemetryLogger) {
|
|
55
|
+
const newErrorFn = (errMsg: string) => new SummarizingWarning(errMsg, logged);
|
|
57
56
|
return wrapErrorAndLog<SummarizingWarning>(error, newErrorFn, logger);
|
|
58
57
|
}
|
|
59
58
|
}
|
|
60
59
|
|
|
61
60
|
export const createSummarizingWarning =
|
|
62
|
-
(
|
|
61
|
+
(errorMessage: string, logged: boolean) => new SummarizingWarning(errorMessage, logged);
|
|
63
62
|
|
|
64
63
|
/**
|
|
65
64
|
* Summarizer is responsible for coordinating when to generate and send summaries.
|
|
@@ -144,7 +143,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
144
143
|
return await this.runCore(onBehalfOf, options);
|
|
145
144
|
} catch (error) {
|
|
146
145
|
this.stop("summarizerException");
|
|
147
|
-
throw SummarizingWarning.wrap(error,
|
|
146
|
+
throw SummarizingWarning.wrap(error, false /* logged */, this.logger);
|
|
148
147
|
} finally {
|
|
149
148
|
this.dispose();
|
|
150
149
|
this.runtime.closeFn();
|
|
@@ -257,9 +256,10 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
257
256
|
summaryTime: Date.now(),
|
|
258
257
|
} as const,
|
|
259
258
|
),
|
|
260
|
-
(
|
|
259
|
+
(errorMessage: string) => {
|
|
261
260
|
if (!this._disposed) {
|
|
262
|
-
this.
|
|
261
|
+
this.logger.sendErrorEvent({ eventName: "summarizingError" },
|
|
262
|
+
createSummarizingWarning(errorMessage, true));
|
|
263
263
|
}
|
|
264
264
|
},
|
|
265
265
|
this.summaryCollection,
|
|
@@ -356,7 +356,7 @@ export class Summarizer extends EventEmitter implements ISummarizer {
|
|
|
356
356
|
return builder.build();
|
|
357
357
|
}
|
|
358
358
|
catch (error) {
|
|
359
|
-
throw SummarizingWarning.wrap(error,
|
|
359
|
+
throw SummarizingWarning.wrap(error, false /* logged */, this.logger);
|
|
360
360
|
}
|
|
361
361
|
};
|
|
362
362
|
|
package/src/summaryGenerator.ts
CHANGED
|
@@ -152,7 +152,7 @@ export class SummaryGenerator {
|
|
|
152
152
|
private readonly pendingAckTimer: IPromiseTimer,
|
|
153
153
|
private readonly heuristicData: ISummarizeHeuristicData,
|
|
154
154
|
private readonly submitSummaryCallback: (options: ISubmitSummaryOptions) => Promise<SubmitSummaryResult>,
|
|
155
|
-
private readonly raiseSummarizingError: (
|
|
155
|
+
private readonly raiseSummarizingError: (errorMessage: string) => void,
|
|
156
156
|
private readonly successfulSummaryCallback: () => void,
|
|
157
157
|
private readonly summaryWatcher: Pick<IClientSummaryWatcher, "watchSummary">,
|
|
158
158
|
private readonly logger: ITelemetryLogger,
|
|
@@ -394,7 +394,7 @@ export class SummaryGenerator {
|
|
|
394
394
|
const message = summaryNack?.message;
|
|
395
395
|
const retryAfterSeconds = summaryNack?.retryAfter;
|
|
396
396
|
|
|
397
|
-
const error = new LoggingError(`summaryNack: ${message}`, { retryAfterSeconds });
|
|
397
|
+
const error = new LoggingError(`Received summaryNack: ${message}`, { retryAfterSeconds });
|
|
398
398
|
logger.sendErrorEvent(
|
|
399
399
|
{ eventName: "SummaryNack", ...summarizeTelemetryProps, retryAfterSeconds }, error);
|
|
400
400
|
|
package/src/summaryManager.ts
CHANGED
|
@@ -4,16 +4,14 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { IDisposable, IEvent, IEventProvider, ITelemetryLogger } from "@fluidframework/common-definitions";
|
|
7
|
-
import {
|
|
7
|
+
import { assert } from "@fluidframework/common-utils";
|
|
8
8
|
import { ChildLogger, PerformanceEvent } from "@fluidframework/telemetry-utils";
|
|
9
9
|
import { DriverErrorType } from "@fluidframework/driver-definitions";
|
|
10
|
-
import { createSummarizingWarning } from "./summarizer";
|
|
11
10
|
import { ISummarizerClientElection } from "./summarizerClientElection";
|
|
12
11
|
import { IThrottler } from "./throttler";
|
|
13
12
|
import {
|
|
14
13
|
ISummarizer,
|
|
15
14
|
ISummarizerOptions,
|
|
16
|
-
ISummarizingWarning,
|
|
17
15
|
SummarizerStopReason,
|
|
18
16
|
} from "./summarizerTypes";
|
|
19
17
|
import { SummaryCollection } from "./summaryCollection";
|
|
@@ -60,10 +58,6 @@ export interface IConnectedState extends IEventProvider<IConnectedEvents> {
|
|
|
60
58
|
readonly clientId: string | undefined;
|
|
61
59
|
}
|
|
62
60
|
|
|
63
|
-
export interface ISummaryManagerEvents extends IEvent {
|
|
64
|
-
(event: "summarizerWarning", listener: (warning: ISummarizingWarning) => void);
|
|
65
|
-
}
|
|
66
|
-
|
|
67
61
|
export interface ISummaryManagerConfig {
|
|
68
62
|
initialDelayMs: number;
|
|
69
63
|
opsToBypassInitialDelay: number;
|
|
@@ -74,7 +68,7 @@ export interface ISummaryManagerConfig {
|
|
|
74
68
|
* It observes changes in calculated summarizer and reacts to changes by either creating summarizer client or
|
|
75
69
|
* stopping existing summarizer client.
|
|
76
70
|
*/
|
|
77
|
-
export class SummaryManager
|
|
71
|
+
export class SummaryManager implements IDisposable {
|
|
78
72
|
private readonly logger: ITelemetryLogger;
|
|
79
73
|
private readonly opsToBypassInitialDelay: number;
|
|
80
74
|
private readonly initialDelayMs: number;
|
|
@@ -105,7 +99,6 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
105
99
|
}: Readonly<Partial<ISummaryManagerConfig>> = {},
|
|
106
100
|
private readonly summarizerOptions?: Readonly<Partial<ISummarizerOptions>>,
|
|
107
101
|
) {
|
|
108
|
-
super();
|
|
109
102
|
|
|
110
103
|
this.logger = ChildLogger.create(
|
|
111
104
|
parentLogger,
|
|
@@ -219,8 +212,6 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
219
212
|
assert(this.state === SummaryManagerState.Starting, 0x263 /* "Expected: starting" */);
|
|
220
213
|
this.state = SummaryManagerState.Running;
|
|
221
214
|
|
|
222
|
-
summarizer.on("summarizingError",
|
|
223
|
-
(warning: ISummarizingWarning) => this.emit("summarizerWarning", warning));
|
|
224
215
|
this.summarizer = summarizer;
|
|
225
216
|
|
|
226
217
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
@@ -238,8 +229,8 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
238
229
|
// we ignore blindly, so try to narrow signature we are looking for - skip logging
|
|
239
230
|
// error only if this client should no longer be a summarizer (which in practice
|
|
240
231
|
// means it also lost connection), and error happened on load (we do not have summarizer).
|
|
241
|
-
// We could
|
|
242
|
-
// but that does not seem to be necessary.
|
|
232
|
+
// We could annotate the error raised in Container.load where the container closed during load with no error
|
|
233
|
+
// and check for that case here, but that does not seem to be necessary.
|
|
243
234
|
if (this.getShouldSummarizeState().shouldSummarize || this.summarizer !== undefined) {
|
|
244
235
|
// Report any failure as an error unless it was due to cancellation (like "disconnected" error)
|
|
245
236
|
// If failure happened on container load, we may not yet realized that socket disconnected, so check
|
|
@@ -251,7 +242,6 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
251
242
|
category,
|
|
252
243
|
},
|
|
253
244
|
error);
|
|
254
|
-
this.emit("summarizerWarning", error);
|
|
255
245
|
|
|
256
246
|
// Note that summarizer may keep going (like doing last summary).
|
|
257
247
|
// Ideally we await stopping process, but this code path is due to a bug
|
|
@@ -296,12 +286,6 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
296
286
|
private async delayBeforeCreatingSummarizer(): Promise<boolean> {
|
|
297
287
|
// throttle creation of new summarizer containers to prevent spamming the server with websocket connections
|
|
298
288
|
let delayMs = this.startThrottler.getDelay();
|
|
299
|
-
if (delayMs > 0 && delayMs > this.startThrottler.maxDelayMs) {
|
|
300
|
-
this.emit(
|
|
301
|
-
"summarizerWarning",
|
|
302
|
-
createSummarizingWarning("summaryManagerCreateSummarizerMaxThrottleDelay", false),
|
|
303
|
-
);
|
|
304
|
-
}
|
|
305
289
|
|
|
306
290
|
// We have been elected the summarizer. Some day we may be able to summarize with a live document but for
|
|
307
291
|
// now we play it safe and launch a second copy.
|
|
@@ -309,6 +293,7 @@ export class SummaryManager extends TypedEventEmitter<ISummaryManagerEvents> imp
|
|
|
309
293
|
eventName: "CreatingSummarizer",
|
|
310
294
|
throttlerDelay: delayMs,
|
|
311
295
|
initialDelay: this.initialDelayMs,
|
|
296
|
+
startThrottlerMaxDelayMs: this.startThrottler.maxDelayMs,
|
|
312
297
|
opsSinceLastAck: this.summaryCollection.opsSinceLastAck,
|
|
313
298
|
opsToBypassInitialDelay: this.opsToBypassInitialDelay,
|
|
314
299
|
});
|