@fluidframework/container-runtime 2.0.0-rc.3.0.1 → 2.0.0-rc.3.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api-report/container-runtime.api.md +23 -7
- package/dist/channelCollection.d.ts +5 -3
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +65 -15
- package/dist/channelCollection.js.map +1 -1
- package/dist/containerRuntime.d.ts +5 -0
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +25 -5
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +2 -0
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +7 -0
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +4 -11
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +45 -29
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +26 -5
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +5 -4
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +14 -2
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +13 -2
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +24 -21
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -2
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/legacy.d.ts +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/summary/documentSchema.js +1 -1
- package/dist/summary/documentSchema.js.map +1 -1
- package/lib/channelCollection.d.ts +5 -3
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +67 -17
- package/lib/channelCollection.js.map +1 -1
- package/lib/containerRuntime.d.ts +5 -0
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +24 -4
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +2 -0
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +7 -0
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +4 -11
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +47 -31
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +26 -5
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +5 -4
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +12 -1
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +13 -2
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +24 -21
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/legacy.d.ts +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/summary/documentSchema.js +1 -1
- package/lib/summary/documentSchema.js.map +1 -1
- package/package.json +17 -17
- package/src/channelCollection.ts +85 -35
- package/src/containerRuntime.ts +24 -24
- package/src/dataStoreContexts.ts +12 -0
- package/src/gc/garbageCollection.ts +63 -41
- package/src/gc/gcDefinitions.ts +21 -9
- package/src/gc/gcHelpers.ts +14 -1
- package/src/gc/gcTelemetry.ts +56 -47
- package/src/gc/index.ts +2 -1
- package/src/index.ts +2 -0
- package/src/packageVersion.ts +1 -1
- package/src/summary/documentSchema.ts +1 -1
package/src/containerRuntime.ts
CHANGED
|
@@ -493,6 +493,11 @@ export interface IContainerRuntimeOptions {
|
|
|
493
493
|
readonly explicitSchemaControl?: boolean;
|
|
494
494
|
}
|
|
495
495
|
|
|
496
|
+
/**
|
|
497
|
+
* Error responses when requesting a deleted object will have this header set to true
|
|
498
|
+
* @alpha
|
|
499
|
+
*/
|
|
500
|
+
export const DeletedResponseHeaderKey = "wasDeleted";
|
|
496
501
|
/**
|
|
497
502
|
* Tombstone error responses will have this header set to true
|
|
498
503
|
* @alpha
|
|
@@ -1639,22 +1644,7 @@ export class ContainerRuntime
|
|
|
1639
1644
|
getSummaryForDatastores(baseSnapshot, metadata),
|
|
1640
1645
|
parentContext,
|
|
1641
1646
|
this.mc.logger,
|
|
1642
|
-
(
|
|
1643
|
-
path: string,
|
|
1644
|
-
reason: "Loaded" | "Changed",
|
|
1645
|
-
timestampMs?: number,
|
|
1646
|
-
packagePath?: readonly string[],
|
|
1647
|
-
request?: IRequest,
|
|
1648
|
-
headerData?: RuntimeHeaderData,
|
|
1649
|
-
) =>
|
|
1650
|
-
this.garbageCollector.nodeUpdated(
|
|
1651
|
-
path,
|
|
1652
|
-
reason,
|
|
1653
|
-
timestampMs,
|
|
1654
|
-
packagePath,
|
|
1655
|
-
request,
|
|
1656
|
-
headerData,
|
|
1657
|
-
),
|
|
1647
|
+
(props) => this.garbageCollector.nodeUpdated(props),
|
|
1658
1648
|
(path: string) => this.garbageCollector.isNodeDeleted(path),
|
|
1659
1649
|
new Map<string, string>(dataStoreAliasMap),
|
|
1660
1650
|
async (runtime: ChannelCollection) => provideEntryPoint,
|
|
@@ -1677,7 +1667,10 @@ export class ContainerRuntime
|
|
|
1677
1667
|
}
|
|
1678
1668
|
},
|
|
1679
1669
|
blobRequested: (blobPath: string) =>
|
|
1680
|
-
this.garbageCollector.nodeUpdated(
|
|
1670
|
+
this.garbageCollector.nodeUpdated({
|
|
1671
|
+
node: { type: "Blob", path: blobPath },
|
|
1672
|
+
reason: "Loaded",
|
|
1673
|
+
}),
|
|
1681
1674
|
isBlobDeleted: (blobPath: string) => this.garbageCollector.isNodeDeleted(blobPath),
|
|
1682
1675
|
runtime: this,
|
|
1683
1676
|
stashedBlobs: pendingRuntimeState?.pendingAttachmentBlobs,
|
|
@@ -2901,12 +2894,11 @@ export class ContainerRuntime
|
|
|
2901
2894
|
"entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint",
|
|
2902
2895
|
);
|
|
2903
2896
|
}
|
|
2904
|
-
this.garbageCollector.nodeUpdated(
|
|
2905
|
-
`/${internalId}
|
|
2906
|
-
"Loaded",
|
|
2907
|
-
|
|
2908
|
-
|
|
2909
|
-
);
|
|
2897
|
+
this.garbageCollector.nodeUpdated({
|
|
2898
|
+
node: { type: "DataStore", path: `/${internalId}` },
|
|
2899
|
+
reason: "Loaded",
|
|
2900
|
+
packagePath: context.packagePath,
|
|
2901
|
+
});
|
|
2910
2902
|
return channel.entryPoint;
|
|
2911
2903
|
}
|
|
2912
2904
|
|
|
@@ -3492,11 +3484,19 @@ export class ContainerRuntime
|
|
|
3492
3484
|
latestSummaryRefSeqNum,
|
|
3493
3485
|
);
|
|
3494
3486
|
|
|
3487
|
+
/**
|
|
3488
|
+
* This was added to validate that the summarizer node tree has the same reference sequence number from the
|
|
3489
|
+
* top running summarizer down to the lowest summarizer node.
|
|
3490
|
+
*
|
|
3491
|
+
* The order of mismatch numbers goes (validate sequence number)-(node sequence number).
|
|
3492
|
+
* Generally the validate sequence number comes from the running summarizer and the node sequence number comes from the
|
|
3493
|
+
* summarizer nodes.
|
|
3494
|
+
*/
|
|
3495
3495
|
if (
|
|
3496
3496
|
startSummaryResult.invalidNodes > 0 ||
|
|
3497
3497
|
startSummaryResult.mismatchNumbers.size > 0
|
|
3498
3498
|
) {
|
|
3499
|
-
summaryLogger.
|
|
3499
|
+
summaryLogger.sendTelemetryEvent({
|
|
3500
3500
|
eventName: "LatestSummaryRefSeqNumMismatch",
|
|
3501
3501
|
details: {
|
|
3502
3502
|
...startSummaryResult,
|
package/src/dataStoreContexts.ts
CHANGED
|
@@ -83,9 +83,21 @@ export class DataStoreContexts implements Iterable<[string, FluidDataStoreContex
|
|
|
83
83
|
public delete(id: string): boolean {
|
|
84
84
|
this.deferredContexts.delete(id);
|
|
85
85
|
this.notBoundContexts.delete(id);
|
|
86
|
+
|
|
87
|
+
// Stash the context here in case it's requested in this session, we can log some details about it
|
|
88
|
+
const context = this._contexts.get(id);
|
|
89
|
+
this._recentlyDeletedContexts.set(id, context);
|
|
90
|
+
|
|
86
91
|
return this._contexts.delete(id);
|
|
87
92
|
}
|
|
88
93
|
|
|
94
|
+
private readonly _recentlyDeletedContexts: Map<string, FluidDataStoreContext | undefined> =
|
|
95
|
+
new Map();
|
|
96
|
+
|
|
97
|
+
public getRecentlyDeletedContext(id: string) {
|
|
98
|
+
return this._recentlyDeletedContexts.get(id);
|
|
99
|
+
}
|
|
100
|
+
|
|
89
101
|
/**
|
|
90
102
|
* Return the unbound local context with the given id,
|
|
91
103
|
* or undefined if it's not found or not unbound.
|
|
@@ -23,11 +23,7 @@ import {
|
|
|
23
23
|
} from "@fluidframework/telemetry-utils/internal";
|
|
24
24
|
|
|
25
25
|
import { BlobManager } from "../blobManager.js";
|
|
26
|
-
import {
|
|
27
|
-
InactiveResponseHeaderKey,
|
|
28
|
-
RuntimeHeaderData,
|
|
29
|
-
TombstoneResponseHeaderKey,
|
|
30
|
-
} from "../containerRuntime.js";
|
|
26
|
+
import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
|
|
31
27
|
import { ClientSessionExpiredError } from "../error.js";
|
|
32
28
|
import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
|
|
33
29
|
import { IRefreshSummaryResult } from "../summary/index.js";
|
|
@@ -48,12 +44,15 @@ import {
|
|
|
48
44
|
ISweepPhaseStats,
|
|
49
45
|
UnreferencedState,
|
|
50
46
|
disableAutoRecoveryKey,
|
|
47
|
+
type IGCNodeUpdatedProps,
|
|
51
48
|
} from "./gcDefinitions.js";
|
|
52
49
|
import {
|
|
53
50
|
cloneGCData,
|
|
54
51
|
compatBehaviorAllowsGCMessageType,
|
|
55
52
|
concatGarbageCollectionData,
|
|
53
|
+
dataStoreNodePathOnly,
|
|
56
54
|
getGCDataFromSnapshot,
|
|
55
|
+
urlToGCNodePath,
|
|
57
56
|
} from "./gcHelpers.js";
|
|
58
57
|
import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
|
|
59
58
|
import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions.js";
|
|
@@ -401,17 +400,27 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
401
400
|
return;
|
|
402
401
|
}
|
|
403
402
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
403
|
+
const initialized = this.gcDataFromLastRun !== undefined;
|
|
404
|
+
await PerformanceEvent.timedExecAsync(
|
|
405
|
+
this.mc.logger,
|
|
406
|
+
{
|
|
407
|
+
eventName: "InitializeOrUpdateGCState",
|
|
408
|
+
details: { initialized, unrefNodeCount: this.unreferencedNodesState.size },
|
|
409
|
+
},
|
|
410
|
+
async () => {
|
|
411
|
+
// If the GC state hasn't been initialized yet, initialize it and return.
|
|
412
|
+
if (!initialized) {
|
|
413
|
+
await this.initializeGCStateFromBaseSnapshotP;
|
|
414
|
+
return;
|
|
415
|
+
}
|
|
409
416
|
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
417
|
+
// If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
|
|
418
|
+
// reference timestamp.
|
|
419
|
+
for (const [, nodeStateTracker] of this.unreferencedNodesState) {
|
|
420
|
+
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
421
|
+
}
|
|
422
|
+
},
|
|
423
|
+
);
|
|
415
424
|
}
|
|
416
425
|
|
|
417
426
|
/**
|
|
@@ -978,30 +987,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
978
987
|
/**
|
|
979
988
|
* Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
|
|
980
989
|
* or throw an error if failing on incorrect usage is configured.
|
|
981
|
-
* @param
|
|
982
|
-
* @param reason - Whether the node was loaded or changed.
|
|
983
|
-
* @param timestampMs - The timestamp when the node changed.
|
|
984
|
-
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
985
|
-
* @param request - The original request for loads to preserve it in telemetry.
|
|
986
|
-
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
990
|
+
* @param IGCNodeUpdatedProps - Details about the node and how it was updated
|
|
987
991
|
*/
|
|
988
|
-
public nodeUpdated(
|
|
989
|
-
|
|
990
|
-
reason
|
|
991
|
-
timestampMs
|
|
992
|
-
packagePath
|
|
993
|
-
request
|
|
994
|
-
headerData
|
|
995
|
-
) {
|
|
992
|
+
public nodeUpdated({
|
|
993
|
+
node,
|
|
994
|
+
reason,
|
|
995
|
+
timestampMs,
|
|
996
|
+
packagePath,
|
|
997
|
+
request,
|
|
998
|
+
headerData,
|
|
999
|
+
}: IGCNodeUpdatedProps) {
|
|
996
1000
|
if (!this.configs.shouldRunGC) {
|
|
997
1001
|
return;
|
|
998
1002
|
}
|
|
999
1003
|
|
|
1000
|
-
|
|
1004
|
+
// trackedId will be either DataStore or Blob ID (not sub-DataStore ID, since some of those are unrecognized by GC)
|
|
1005
|
+
const trackedId = node.path;
|
|
1006
|
+
const isTombstoned = this.tombstones.includes(trackedId);
|
|
1007
|
+
const isInactive = this.unreferencedNodesState.get(trackedId)?.state === "Inactive";
|
|
1008
|
+
|
|
1009
|
+
const fullPath = request !== undefined ? urlToGCNodePath(request.url) : trackedId;
|
|
1001
1010
|
|
|
1002
1011
|
// This will log if appropriate
|
|
1003
|
-
this.telemetryTracker.nodeUsed({
|
|
1004
|
-
id:
|
|
1012
|
+
this.telemetryTracker.nodeUsed(trackedId, {
|
|
1013
|
+
id: fullPath,
|
|
1005
1014
|
usageType: reason,
|
|
1006
1015
|
currentReferenceTimestampMs:
|
|
1007
1016
|
timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
|
|
@@ -1010,6 +1019,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1010
1019
|
isTombstoned,
|
|
1011
1020
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1012
1021
|
headers: headerData,
|
|
1022
|
+
requestUrl: request?.url,
|
|
1023
|
+
requestHeaders: JSON.stringify(request?.headers),
|
|
1013
1024
|
});
|
|
1014
1025
|
|
|
1015
1026
|
// Any time we log a Tombstone Loaded error (via Telemetry Tracker),
|
|
@@ -1018,17 +1029,20 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1018
1029
|
// to be loaded by the Summarizer, and auto-recovery will be triggered then.
|
|
1019
1030
|
if (isTombstoned && reason === "Loaded") {
|
|
1020
1031
|
// Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
|
|
1021
|
-
this.triggerAutoRecovery(
|
|
1032
|
+
this.triggerAutoRecovery(fullPath);
|
|
1022
1033
|
}
|
|
1023
1034
|
|
|
1024
|
-
const nodeType = this.runtime.getNodeType(
|
|
1035
|
+
const nodeType = this.runtime.getNodeType(fullPath);
|
|
1025
1036
|
|
|
1026
1037
|
// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
|
|
1027
|
-
|
|
1038
|
+
const loadedBlobOrDataStore =
|
|
1039
|
+
reason === "Loaded" &&
|
|
1040
|
+
(nodeType === GCNodeType.Blob || nodeType === GCNodeType.DataStore);
|
|
1041
|
+
if (!loadedBlobOrDataStore) {
|
|
1028
1042
|
return;
|
|
1029
1043
|
}
|
|
1030
1044
|
|
|
1031
|
-
const errorRequest: IRequest = request ?? { url:
|
|
1045
|
+
const errorRequest: IRequest = request ?? { url: fullPath };
|
|
1032
1046
|
if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
|
|
1033
1047
|
// The requested data store is removed by gc. Create a 404 gc response exception.
|
|
1034
1048
|
throw responseToException(
|
|
@@ -1040,7 +1054,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1040
1054
|
}
|
|
1041
1055
|
|
|
1042
1056
|
// If the object is inactive and inactive enforcement is configured, throw an error.
|
|
1043
|
-
if (
|
|
1057
|
+
if (isInactive) {
|
|
1044
1058
|
const shouldThrowOnInactiveLoad =
|
|
1045
1059
|
!this.isSummarizerClient &&
|
|
1046
1060
|
this.configs.throwOnInactiveLoad === true &&
|
|
@@ -1112,22 +1126,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1112
1126
|
outboundRoutes.push(toNodePath);
|
|
1113
1127
|
this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
1114
1128
|
|
|
1115
|
-
|
|
1129
|
+
// GC won't recognize some subDataStore paths that we encounter (e.g. a path suited for a custom request handler)
|
|
1130
|
+
// So for subDataStore paths we need to check the parent dataStore for current tombstone/inactive status.
|
|
1131
|
+
const trackedId =
|
|
1132
|
+
this.runtime.getNodeType(toNodePath) === "SubDataStore"
|
|
1133
|
+
? dataStoreNodePathOnly(toNodePath)
|
|
1134
|
+
: toNodePath;
|
|
1135
|
+
this.telemetryTracker.nodeUsed(trackedId, {
|
|
1116
1136
|
id: toNodePath,
|
|
1137
|
+
fromId: fromNodePath,
|
|
1117
1138
|
usageType: "Revived",
|
|
1118
1139
|
currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
|
|
1119
1140
|
packagePath: undefined,
|
|
1120
1141
|
completedGCRuns: this.completedRuns,
|
|
1121
|
-
isTombstoned: this.tombstones.includes(
|
|
1142
|
+
isTombstoned: this.tombstones.includes(trackedId),
|
|
1122
1143
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1123
|
-
fromId: fromNodePath,
|
|
1124
1144
|
autorecovery,
|
|
1125
1145
|
});
|
|
1126
1146
|
|
|
1127
|
-
// This node is referenced - Clear its unreferenced state
|
|
1147
|
+
// This node is referenced - Clear its unreferenced state if present
|
|
1128
1148
|
// But don't delete the node id from the map yet.
|
|
1129
1149
|
// When generating GC stats, the set of nodes in here is used as the baseline for
|
|
1130
1150
|
// what was unreferenced in the last GC run.
|
|
1151
|
+
// NOTE: We use toNodePath not trackedId even though it may be an unrecognized subDataStore route (hence no-op),
|
|
1152
|
+
// because a reference to such a path is not sufficient to consider the DataStore referenced.
|
|
1131
1153
|
this.unreferencedNodesState.get(toNodePath)?.stopTracking();
|
|
1132
1154
|
}
|
|
1133
1155
|
|
package/src/gc/gcDefinitions.ts
CHANGED
|
@@ -245,7 +245,7 @@ export const GCNodeType = {
|
|
|
245
245
|
Blob: "Blob",
|
|
246
246
|
// Nodes that are neither of the above. For example, root node.
|
|
247
247
|
Other: "Other",
|
|
248
|
-
};
|
|
248
|
+
} as const;
|
|
249
249
|
|
|
250
250
|
/**
|
|
251
251
|
* @alpha
|
|
@@ -370,14 +370,7 @@ export interface IGarbageCollector {
|
|
|
370
370
|
* Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
|
|
371
371
|
* or throw an error if failing on incorrect usage is configured.
|
|
372
372
|
*/
|
|
373
|
-
nodeUpdated(
|
|
374
|
-
nodePath: string,
|
|
375
|
-
reason: "Loaded" | "Changed",
|
|
376
|
-
timestampMs?: number,
|
|
377
|
-
packagePath?: readonly string[],
|
|
378
|
-
request?: IRequest,
|
|
379
|
-
headerData?: RuntimeHeaderData,
|
|
380
|
-
): void;
|
|
373
|
+
nodeUpdated(props: IGCNodeUpdatedProps): void;
|
|
381
374
|
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
382
375
|
addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true): void;
|
|
383
376
|
/** Called to process a garbage collection message. */
|
|
@@ -388,6 +381,25 @@ export interface IGarbageCollector {
|
|
|
388
381
|
dispose(): void;
|
|
389
382
|
}
|
|
390
383
|
|
|
384
|
+
/**
|
|
385
|
+
* Info needed by GC when notified that a node was updated (loaded or changed)
|
|
386
|
+
* @internal
|
|
387
|
+
*/
|
|
388
|
+
export interface IGCNodeUpdatedProps {
|
|
389
|
+
/** Type and path of the updated node */
|
|
390
|
+
node: { type: (typeof GCNodeType)["DataStore" | "Blob"]; path: string };
|
|
391
|
+
/** Whether the node (or a subpath) was loaded or changed. */
|
|
392
|
+
reason: "Loaded" | "Changed";
|
|
393
|
+
/** The op-based timestamp when the node changed, if applicable */
|
|
394
|
+
timestampMs?: number;
|
|
395
|
+
/** The package path of the node. This may not be available if the node hasn't been loaded yet */
|
|
396
|
+
packagePath?: readonly string[];
|
|
397
|
+
/** The original request for loads to preserve it in telemetry */
|
|
398
|
+
request?: IRequest;
|
|
399
|
+
/** If the node was loaded via request path, the header data. May be modified from the original request */
|
|
400
|
+
headerData?: RuntimeHeaderData;
|
|
401
|
+
}
|
|
402
|
+
|
|
391
403
|
/** Parameters necessary for creating a GarbageCollector. */
|
|
392
404
|
export interface IGarbageCollectorCreateParams {
|
|
393
405
|
readonly runtime: IGarbageCollectionRuntime;
|
package/src/gc/gcHelpers.ts
CHANGED
|
@@ -280,10 +280,23 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
|
|
|
280
280
|
* @param str - A string that may contain leading and / or trailing slashes.
|
|
281
281
|
* @returns A new string without leading and trailing slashes.
|
|
282
282
|
*/
|
|
283
|
-
|
|
283
|
+
function trimLeadingAndTrailingSlashes(str: string) {
|
|
284
284
|
return str.replace(/^\/+|\/+$/g, "");
|
|
285
285
|
}
|
|
286
286
|
|
|
287
|
+
/** Reformats a request URL to match expected format for a GC node path */
|
|
288
|
+
export function urlToGCNodePath(url: string): string {
|
|
289
|
+
return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
|
|
290
|
+
}
|
|
291
|
+
|
|
292
|
+
/**
|
|
293
|
+
* Pulls out the first path segment and formats it as a GC Node path
|
|
294
|
+
* e.g. "/dataStoreId/ddsId" yields "/dataStoreId"
|
|
295
|
+
*/
|
|
296
|
+
export function dataStoreNodePathOnly(subDataStorePath: string): string {
|
|
297
|
+
return `/${subDataStorePath.split("/")[1]}`;
|
|
298
|
+
}
|
|
299
|
+
|
|
287
300
|
/**
|
|
288
301
|
* Utility to implement compat behaviors given an unknown message type
|
|
289
302
|
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|
package/src/gc/gcTelemetry.ts
CHANGED
|
@@ -14,6 +14,7 @@ import {
|
|
|
14
14
|
tagCodeArtifacts,
|
|
15
15
|
} from "@fluidframework/telemetry-utils/internal";
|
|
16
16
|
|
|
17
|
+
import type { Tagged } from "@fluidframework/core-interfaces";
|
|
17
18
|
import { RuntimeHeaderData } from "../containerRuntime.js";
|
|
18
19
|
import { ICreateContainerMetadata } from "../summary/index.js";
|
|
19
20
|
|
|
@@ -43,11 +44,12 @@ interface ICommonProps {
|
|
|
43
44
|
|
|
44
45
|
/** The event that is logged when unreferenced node is used after a certain time. */
|
|
45
46
|
interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps {
|
|
47
|
+
/** The id that GC uses to track the node. May or may not match id */
|
|
48
|
+
trackedId: string;
|
|
46
49
|
state: UnreferencedState;
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
};
|
|
50
|
+
/** The full path (in GC Path format) to the node in question */
|
|
51
|
+
id: Tagged<string>;
|
|
52
|
+
fromId?: Tagged<string>;
|
|
51
53
|
type: GCNodeType;
|
|
52
54
|
unrefTime: number;
|
|
53
55
|
age: number;
|
|
@@ -56,19 +58,24 @@ interface IUnreferencedEventProps extends ICreateContainerMetadata, ICommonProps
|
|
|
56
58
|
[K in keyof GCFeatureMatrix]: GCFeatureMatrix[K];
|
|
57
59
|
};
|
|
58
60
|
timeout?: number;
|
|
59
|
-
fromId?: {
|
|
60
|
-
value: string;
|
|
61
|
-
tag: string;
|
|
62
|
-
};
|
|
63
61
|
}
|
|
64
62
|
|
|
65
63
|
/** Properties passed to nodeUsed function when a node is used. */
|
|
66
64
|
interface INodeUsageProps extends ICommonProps {
|
|
65
|
+
/** The full path (in GC Path format) to the node in question */
|
|
67
66
|
id: string;
|
|
67
|
+
/** Latest timestamp received from the server, as a baseline for computing GC state/age */
|
|
68
68
|
currentReferenceTimestampMs: number | undefined;
|
|
69
|
+
/** The package path of the node. This may not be available if the node hasn't been loaded yet */
|
|
69
70
|
packagePath: readonly string[] | undefined;
|
|
71
|
+
/** In case of Revived - what node added the reference? */
|
|
70
72
|
fromId?: string;
|
|
73
|
+
/** In case of Revived - was it revived due to autorecovery? */
|
|
71
74
|
autorecovery?: true;
|
|
75
|
+
/** URL (including query string) if this usage came from a request */
|
|
76
|
+
requestUrl?: string;
|
|
77
|
+
/** Original request headers if this usage came from a request or handle.get */
|
|
78
|
+
requestHeaders?: string;
|
|
72
79
|
}
|
|
73
80
|
|
|
74
81
|
/**
|
|
@@ -143,18 +150,34 @@ export class GCTelemetryTracker {
|
|
|
143
150
|
}
|
|
144
151
|
|
|
145
152
|
/**
|
|
146
|
-
* Called when a node is used. If the node is
|
|
153
|
+
* Called when a node is used. If the node is inactive or tombstoned, log telemetry indicating object is used
|
|
147
154
|
* when it should not have been.
|
|
155
|
+
* @param trackedId - The id that GC uses to track the node. For SubDataStore nodes, this should be the DataStore ID.
|
|
156
|
+
* @param INodeUsageProps - All kind of details about this event to be logged
|
|
148
157
|
*/
|
|
149
|
-
public nodeUsed(
|
|
158
|
+
public nodeUsed(
|
|
159
|
+
trackedId: string,
|
|
160
|
+
{
|
|
161
|
+
usageType,
|
|
162
|
+
currentReferenceTimestampMs,
|
|
163
|
+
packagePath,
|
|
164
|
+
id: untaggedId,
|
|
165
|
+
fromId: untaggedFromId,
|
|
166
|
+
isTombstoned,
|
|
167
|
+
...otherNodeUsageProps
|
|
168
|
+
}: INodeUsageProps,
|
|
169
|
+
) {
|
|
150
170
|
// If there is no reference timestamp to work with, no ops have been processed after creation. If so, skip
|
|
151
171
|
// logging as nothing interesting would have happened worth logging.
|
|
152
|
-
if (
|
|
172
|
+
if (currentReferenceTimestampMs === undefined) {
|
|
153
173
|
return;
|
|
154
174
|
}
|
|
155
175
|
|
|
156
|
-
|
|
157
|
-
|
|
176
|
+
// Note: For SubDataStore Load usage, trackedId will be the DataStore's id, not the full path in question.
|
|
177
|
+
// This is necessary because the SubDataStore path may be unrecognized by GC (if suited for a custom request handler)
|
|
178
|
+
const nodeStateTracker = this.getNodeStateTracker(trackedId);
|
|
179
|
+
const nodeType = this.getNodeType(untaggedId);
|
|
180
|
+
|
|
158
181
|
const timeout = (() => {
|
|
159
182
|
switch (nodeStateTracker?.state) {
|
|
160
183
|
case UnreferencedState.Inactive:
|
|
@@ -170,33 +193,27 @@ export class GCTelemetryTracker {
|
|
|
170
193
|
return undefined;
|
|
171
194
|
}
|
|
172
195
|
})();
|
|
173
|
-
const {
|
|
174
|
-
usageType,
|
|
175
|
-
currentReferenceTimestampMs,
|
|
176
|
-
packagePath,
|
|
177
|
-
id: untaggedId,
|
|
178
|
-
fromId: untaggedFromId,
|
|
179
|
-
...propsToLog
|
|
180
|
-
} = nodeUsageProps;
|
|
181
196
|
const { persistedGcFeatureMatrix, ...configs } = this.configs;
|
|
182
|
-
const unrefEventProps
|
|
197
|
+
const unrefEventProps = {
|
|
198
|
+
trackedId,
|
|
183
199
|
type: nodeType,
|
|
184
200
|
unrefTime: nodeStateTracker?.unreferencedTimestampMs ?? -1,
|
|
185
201
|
age:
|
|
186
202
|
nodeStateTracker !== undefined
|
|
187
|
-
?
|
|
188
|
-
nodeStateTracker.unreferencedTimestampMs
|
|
203
|
+
? currentReferenceTimestampMs - nodeStateTracker.unreferencedTimestampMs
|
|
189
204
|
: -1,
|
|
190
205
|
timeout,
|
|
206
|
+
isTombstoned,
|
|
191
207
|
...tagCodeArtifacts({ id: untaggedId, fromId: untaggedFromId }),
|
|
192
|
-
...
|
|
208
|
+
...otherNodeUsageProps,
|
|
193
209
|
...this.createContainerMetadata,
|
|
194
210
|
gcConfigs: { ...configs, ...persistedGcFeatureMatrix },
|
|
195
|
-
}
|
|
211
|
+
} satisfies Omit<IUnreferencedEventProps, "state" | "usageType"> &
|
|
212
|
+
typeof otherNodeUsageProps;
|
|
196
213
|
|
|
197
214
|
// If the node that is used is tombstoned, log a tombstone telemetry.
|
|
198
|
-
if (
|
|
199
|
-
this.logTombstoneUsageTelemetry(
|
|
215
|
+
if (isTombstoned) {
|
|
216
|
+
this.logTombstoneUsageTelemetry(unrefEventProps, nodeType, usageType, packagePath);
|
|
200
217
|
}
|
|
201
218
|
|
|
202
219
|
// After logging tombstone telemetry, if the node's unreferenced state is not tracked, there is nothing
|
|
@@ -206,16 +223,9 @@ export class GCTelemetryTracker {
|
|
|
206
223
|
}
|
|
207
224
|
|
|
208
225
|
const state = nodeStateTracker.state;
|
|
209
|
-
const uniqueEventId = `${state}-${
|
|
226
|
+
const uniqueEventId = `${state}-${untaggedId}-${usageType}`;
|
|
210
227
|
|
|
211
|
-
if (
|
|
212
|
-
!this.shouldLogNonActiveEvent(
|
|
213
|
-
nodeType,
|
|
214
|
-
nodeUsageProps.usageType,
|
|
215
|
-
nodeStateTracker,
|
|
216
|
-
uniqueEventId,
|
|
217
|
-
)
|
|
218
|
-
) {
|
|
228
|
+
if (!this.shouldLogNonActiveEvent(nodeType, usageType, nodeStateTracker, uniqueEventId)) {
|
|
219
229
|
return;
|
|
220
230
|
}
|
|
221
231
|
|
|
@@ -229,8 +239,8 @@ export class GCTelemetryTracker {
|
|
|
229
239
|
// SweepReady errors are usages of Objects that will be deleted by GC Sweep!
|
|
230
240
|
if (this.isSummarizerClient) {
|
|
231
241
|
this.pendingEventsQueue.push({
|
|
232
|
-
...unrefEventProps,
|
|
233
|
-
usageType
|
|
242
|
+
...unrefEventProps, // Note: Contains some properties from INodeUsageProps as well
|
|
243
|
+
usageType,
|
|
234
244
|
state,
|
|
235
245
|
});
|
|
236
246
|
} else {
|
|
@@ -238,11 +248,11 @@ export class GCTelemetryTracker {
|
|
|
238
248
|
// summarizer clients if they are based off of user actions (such as scrolling to content for these objects)
|
|
239
249
|
// Events generated:
|
|
240
250
|
// InactiveObject_Loaded, SweepReadyObject_Loaded
|
|
241
|
-
if (
|
|
251
|
+
if (usageType === "Loaded") {
|
|
242
252
|
const { id, fromId, headers, gcConfigs, ...detailedProps } = unrefEventProps;
|
|
243
253
|
const event = {
|
|
244
|
-
eventName: `${state}Object_${
|
|
245
|
-
...tagCodeArtifacts({ pkg:
|
|
254
|
+
eventName: `${state}Object_${usageType}`,
|
|
255
|
+
...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
|
|
246
256
|
stack: generateStack(),
|
|
247
257
|
id,
|
|
248
258
|
fromId,
|
|
@@ -262,10 +272,10 @@ export class GCTelemetryTracker {
|
|
|
262
272
|
* Logs telemetry when a tombstoned object is changed, revived or loaded.
|
|
263
273
|
*/
|
|
264
274
|
private logTombstoneUsageTelemetry(
|
|
265
|
-
nodeUsageProps: INodeUsageProps,
|
|
266
275
|
unrefEventProps: Omit<IUnreferencedEventProps, "state" | "usageType">,
|
|
267
276
|
nodeType: GCNodeType,
|
|
268
277
|
usageType: NodeUsageType,
|
|
278
|
+
packagePath?: readonly string[],
|
|
269
279
|
) {
|
|
270
280
|
// This will log the following events:
|
|
271
281
|
// GC_Tombstone_DataStore_Requested, GC_Tombstone_DataStore_Changed, GC_Tombstone_DataStore_Revived
|
|
@@ -275,12 +285,12 @@ export class GCTelemetryTracker {
|
|
|
275
285
|
const eventUsageName = usageType === "Loaded" ? "Requested" : usageType;
|
|
276
286
|
const event = {
|
|
277
287
|
eventName: `GC_Tombstone_${nodeType}_${eventUsageName}`,
|
|
278
|
-
|
|
288
|
+
...tagCodeArtifacts({ pkg: packagePath?.join("/") }),
|
|
279
289
|
stack: generateStack(),
|
|
280
290
|
id,
|
|
281
291
|
fromId,
|
|
282
292
|
headers: { ...headers },
|
|
283
|
-
details: detailedProps,
|
|
293
|
+
details: detailedProps, // Also includes some properties from INodeUsageProps type
|
|
284
294
|
gcConfigs,
|
|
285
295
|
tombstoneFlags: {
|
|
286
296
|
DisableTombstone: this.mc.config.getBoolean(disableTombstoneKey),
|
|
@@ -372,7 +382,6 @@ export class GCTelemetryTracker {
|
|
|
372
382
|
// InactiveObject_Loaded, InactiveObject_Changed, InactiveObject_Revived
|
|
373
383
|
// SweepReadyObject_Loaded, SweepReadyObject_Changed, SweepReadyObject_Revived
|
|
374
384
|
for (const eventProps of this.pendingEventsQueue) {
|
|
375
|
-
// const { usageType, state, id, fromId, ...propsToLog } = eventProps;
|
|
376
385
|
const { usageType, state, id, fromId, headers, gcConfigs, ...detailedProps } =
|
|
377
386
|
eventProps;
|
|
378
387
|
/**
|
|
@@ -381,7 +390,7 @@ export class GCTelemetryTracker {
|
|
|
381
390
|
* Loaded and Changed events are logged only if the node is not active. If the node is active, it was
|
|
382
391
|
* revived and a Revived event will be logged for it.
|
|
383
392
|
*/
|
|
384
|
-
const nodeStateTracker = this.getNodeStateTracker(
|
|
393
|
+
const nodeStateTracker = this.getNodeStateTracker(detailedProps.trackedId); // Note: This is never SubDataStore path
|
|
385
394
|
const active =
|
|
386
395
|
nodeStateTracker === undefined ||
|
|
387
396
|
nodeStateTracker.state === UnreferencedState.Active;
|
package/src/gc/index.ts
CHANGED
|
@@ -23,6 +23,7 @@ export {
|
|
|
23
23
|
IGarbageCollectorCreateParams,
|
|
24
24
|
IGCMetadata,
|
|
25
25
|
IGCMetadata_Deprecated,
|
|
26
|
+
IGCNodeUpdatedProps,
|
|
26
27
|
IGCResult,
|
|
27
28
|
IGCRuntimeOptions,
|
|
28
29
|
IMarkPhaseStats,
|
|
@@ -46,8 +47,8 @@ export {
|
|
|
46
47
|
cloneGCData,
|
|
47
48
|
concatGarbageCollectionStates,
|
|
48
49
|
getGCVersionInEffect,
|
|
49
|
-
trimLeadingAndTrailingSlashes,
|
|
50
50
|
unpackChildNodesGCDetails,
|
|
51
|
+
urlToGCNodePath,
|
|
51
52
|
} from "./gcHelpers.js";
|
|
52
53
|
export { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
|
|
53
54
|
export {
|
package/src/index.ts
CHANGED
|
@@ -13,6 +13,7 @@ export {
|
|
|
13
13
|
isRuntimeMessage,
|
|
14
14
|
agentSchedulerId,
|
|
15
15
|
ContainerRuntime,
|
|
16
|
+
DeletedResponseHeaderKey,
|
|
16
17
|
TombstoneResponseHeaderKey,
|
|
17
18
|
InactiveResponseHeaderKey,
|
|
18
19
|
ISummaryConfiguration,
|
|
@@ -47,6 +48,7 @@ export {
|
|
|
47
48
|
IGCRuntimeOptions,
|
|
48
49
|
IMarkPhaseStats,
|
|
49
50
|
ISweepPhaseStats,
|
|
51
|
+
IGCNodeUpdatedProps,
|
|
50
52
|
IGCStats,
|
|
51
53
|
} from "./gc/index.js";
|
|
52
54
|
export {
|
package/src/packageVersion.ts
CHANGED
|
@@ -552,6 +552,7 @@ export class DocumentsSchemaController {
|
|
|
552
552
|
*/
|
|
553
553
|
public maybeSendSchemaMessage(): IDocumentSchemaChangeMessage | undefined {
|
|
554
554
|
if (this.sendOp && this.futureSchema !== undefined) {
|
|
555
|
+
this.sendOp = false;
|
|
555
556
|
assert(
|
|
556
557
|
this.explicitSchemaControl &&
|
|
557
558
|
this.futureSchema.runtime.explicitSchemaControl === true,
|
|
@@ -562,7 +563,6 @@ export class DocumentsSchemaController {
|
|
|
562
563
|
refSeq: this.documentSchema.refSeq,
|
|
563
564
|
};
|
|
564
565
|
}
|
|
565
|
-
this.sendOp = false;
|
|
566
566
|
}
|
|
567
567
|
|
|
568
568
|
/**
|