@fluidframework/container-runtime 2.0.0-rc.2.0.3 → 2.0.0-rc.2.0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/api-report/container-runtime.api.md +32 -4
- package/dist/channelCollection.d.ts +7 -3
- package/dist/channelCollection.d.ts.map +1 -1
- package/dist/channelCollection.js +80 -22
- package/dist/channelCollection.js.map +1 -1
- package/dist/container-runtime-alpha.d.ts +14 -4
- package/dist/container-runtime-beta.d.ts +6 -0
- package/dist/container-runtime-public.d.ts +6 -0
- package/dist/container-runtime-untrimmed.d.ts +43 -4
- package/dist/containerRuntime.d.ts +6 -0
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +16 -4
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +1 -1
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +12 -2
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStoreContexts.d.ts +2 -0
- package/dist/dataStoreContexts.d.ts.map +1 -1
- package/dist/dataStoreContexts.js +7 -0
- package/dist/dataStoreContexts.js.map +1 -1
- package/dist/gc/garbageCollection.d.ts +4 -11
- package/dist/gc/garbageCollection.d.ts.map +1 -1
- package/dist/gc/garbageCollection.js +45 -29
- package/dist/gc/garbageCollection.js.map +1 -1
- package/dist/gc/gcDefinitions.d.ts +26 -5
- package/dist/gc/gcDefinitions.d.ts.map +1 -1
- package/dist/gc/gcDefinitions.js.map +1 -1
- package/dist/gc/gcHelpers.d.ts +5 -4
- package/dist/gc/gcHelpers.d.ts.map +1 -1
- package/dist/gc/gcHelpers.js +14 -2
- package/dist/gc/gcHelpers.js.map +1 -1
- package/dist/gc/gcTelemetry.d.ts +13 -2
- package/dist/gc/gcTelemetry.d.ts.map +1 -1
- package/dist/gc/gcTelemetry.js +24 -21
- package/dist/gc/gcTelemetry.js.map +1 -1
- package/dist/gc/index.d.ts +2 -2
- package/dist/gc/index.d.ts.map +1 -1
- package/dist/gc/index.js +2 -2
- package/dist/gc/index.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/lib/channelCollection.d.ts +7 -3
- package/lib/channelCollection.d.ts.map +1 -1
- package/lib/channelCollection.js +82 -24
- package/lib/channelCollection.js.map +1 -1
- package/lib/container-runtime-alpha.d.ts +14 -4
- package/lib/container-runtime-beta.d.ts +6 -0
- package/lib/container-runtime-public.d.ts +6 -0
- package/lib/container-runtime-untrimmed.d.ts +43 -4
- package/lib/containerRuntime.d.ts +6 -0
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +15 -3
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +1 -1
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +12 -2
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStoreContexts.d.ts +2 -0
- package/lib/dataStoreContexts.d.ts.map +1 -1
- package/lib/dataStoreContexts.js +7 -0
- package/lib/dataStoreContexts.js.map +1 -1
- package/lib/gc/garbageCollection.d.ts +4 -11
- package/lib/gc/garbageCollection.d.ts.map +1 -1
- package/lib/gc/garbageCollection.js +47 -31
- package/lib/gc/garbageCollection.js.map +1 -1
- package/lib/gc/gcDefinitions.d.ts +26 -5
- package/lib/gc/gcDefinitions.d.ts.map +1 -1
- package/lib/gc/gcDefinitions.js.map +1 -1
- package/lib/gc/gcHelpers.d.ts +5 -4
- package/lib/gc/gcHelpers.d.ts.map +1 -1
- package/lib/gc/gcHelpers.js +12 -1
- package/lib/gc/gcHelpers.js.map +1 -1
- package/lib/gc/gcTelemetry.d.ts +13 -2
- package/lib/gc/gcTelemetry.d.ts.map +1 -1
- package/lib/gc/gcTelemetry.js +24 -21
- package/lib/gc/gcTelemetry.js.map +1 -1
- package/lib/gc/index.d.ts +2 -2
- package/lib/gc/index.d.ts.map +1 -1
- package/lib/gc/index.js +1 -1
- package/lib/gc/index.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/test/gc/garbageCollection.spec.js +23 -14
- package/lib/test/gc/garbageCollection.spec.js.map +1 -1
- package/lib/test/gc/gcHelpers.spec.js +69 -1
- package/lib/test/gc/gcHelpers.spec.js.map +1 -1
- package/lib/test/gc/gcTelemetry.spec.js +31 -3
- package/lib/test/gc/gcTelemetry.spec.js.map +1 -1
- package/package.json +16 -16
- package/src/channelCollection.ts +107 -43
- package/src/containerRuntime.ts +17 -23
- package/src/dataStoreContext.ts +14 -2
- package/src/dataStoreContexts.ts +12 -0
- package/src/gc/garbageCollection.ts +63 -41
- package/src/gc/gcDefinitions.ts +21 -9
- package/src/gc/gcHelpers.ts +14 -1
- package/src/gc/gcTelemetry.ts +56 -47
- package/src/gc/index.ts +2 -1
- package/src/index.ts +3 -0
- package/src/packageVersion.ts +1 -1
package/src/channelCollection.ts
CHANGED
|
@@ -61,7 +61,11 @@ import { AttachState } from "@fluidframework/container-definitions";
|
|
|
61
61
|
import { buildSnapshotTree } from "@fluidframework/driver-utils";
|
|
62
62
|
import { assert, Lazy, LazyPromise } from "@fluidframework/core-utils";
|
|
63
63
|
import { DataStoreContexts } from "./dataStoreContexts.js";
|
|
64
|
-
import {
|
|
64
|
+
import {
|
|
65
|
+
DeletedResponseHeaderKey,
|
|
66
|
+
RuntimeHeaderData,
|
|
67
|
+
defaultRuntimeHeaderData,
|
|
68
|
+
} from "./containerRuntime.js";
|
|
65
69
|
import {
|
|
66
70
|
FluidDataStoreContext,
|
|
67
71
|
RemoteFluidDataStoreContext,
|
|
@@ -78,7 +82,8 @@ import {
|
|
|
78
82
|
import {
|
|
79
83
|
GCNodeType,
|
|
80
84
|
detectOutboundRoutesViaDDSKey,
|
|
81
|
-
|
|
85
|
+
IGCNodeUpdatedProps,
|
|
86
|
+
urlToGCNodePath,
|
|
82
87
|
} from "./gc/index.js";
|
|
83
88
|
import {
|
|
84
89
|
IContainerRuntimeMetadata,
|
|
@@ -262,19 +267,13 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
262
267
|
>();
|
|
263
268
|
|
|
264
269
|
private readonly contexts: DataStoreContexts;
|
|
270
|
+
private readonly aliasedDataStores: Set<string>;
|
|
265
271
|
|
|
266
272
|
constructor(
|
|
267
273
|
private readonly baseSnapshot: ISnapshotTree | undefined,
|
|
268
274
|
public readonly parentContext: IFluidParentContext,
|
|
269
275
|
baseLogger: ITelemetryBaseLogger,
|
|
270
|
-
private readonly gcNodeUpdated: (
|
|
271
|
-
nodePath: string,
|
|
272
|
-
reason: "Loaded" | "Changed",
|
|
273
|
-
timestampMs?: number,
|
|
274
|
-
packagePath?: readonly string[],
|
|
275
|
-
request?: IRequest,
|
|
276
|
-
headerData?: RuntimeHeaderData,
|
|
277
|
-
) => void,
|
|
276
|
+
private readonly gcNodeUpdated: (props: IGCNodeUpdatedProps) => void,
|
|
278
277
|
private readonly isDataStoreDeleted: (nodePath: string) => boolean,
|
|
279
278
|
private readonly aliasMap: Map<string, string>,
|
|
280
279
|
provideEntryPoint: (runtime: ChannelCollection) => Promise<FluidObject>,
|
|
@@ -291,6 +290,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
291
290
|
"",
|
|
292
291
|
this.parentContext.IFluidHandleContext,
|
|
293
292
|
);
|
|
293
|
+
this.aliasedDataStores = new Set(aliasMap.values());
|
|
294
294
|
|
|
295
295
|
// Extract stores stored inside the snapshot
|
|
296
296
|
const fluidDataStores = new Map<string, ISnapshotTree>();
|
|
@@ -520,6 +520,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
520
520
|
this.parentContext.addedGCOutboundReference?.(this.containerRuntimeHandle, handle);
|
|
521
521
|
|
|
522
522
|
this.aliasMap.set(aliasMessage.alias, context.id);
|
|
523
|
+
this.aliasedDataStores.add(context.id);
|
|
523
524
|
context.setInMemoryRoot();
|
|
524
525
|
return true;
|
|
525
526
|
}
|
|
@@ -850,17 +851,18 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
850
851
|
|
|
851
852
|
// Notify that a GC node for the data store changed. This is used to detect if a deleted data store is
|
|
852
853
|
// being used.
|
|
853
|
-
this.gcNodeUpdated(
|
|
854
|
-
`/${envelope.address}
|
|
855
|
-
"Changed",
|
|
856
|
-
message.timestamp,
|
|
857
|
-
context.isLoaded ? context.packagePath : undefined,
|
|
858
|
-
);
|
|
854
|
+
this.gcNodeUpdated({
|
|
855
|
+
node: { type: "DataStore", path: `/${envelope.address}` },
|
|
856
|
+
reason: "Changed",
|
|
857
|
+
timestampMs: message.timestamp,
|
|
858
|
+
packagePath: context.isLoaded ? context.packagePath : undefined,
|
|
859
|
+
});
|
|
859
860
|
}
|
|
860
861
|
|
|
861
|
-
|
|
862
|
+
private async getDataStore(
|
|
862
863
|
id: string,
|
|
863
864
|
requestHeaderData: RuntimeHeaderData,
|
|
865
|
+
originalRequest: IRequest,
|
|
864
866
|
): Promise<FluidDataStoreContext> {
|
|
865
867
|
const headerData = { ...defaultRuntimeHeaderData, ...requestHeaderData };
|
|
866
868
|
if (
|
|
@@ -870,13 +872,15 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
870
872
|
"Requested",
|
|
871
873
|
"getDataStore",
|
|
872
874
|
requestHeaderData,
|
|
875
|
+
originalRequest,
|
|
873
876
|
)
|
|
874
877
|
) {
|
|
875
878
|
// The requested data store has been deleted by gc. Create a 404 response exception.
|
|
876
|
-
const request: IRequest = { url: id };
|
|
877
879
|
throw responseToException(
|
|
878
|
-
createResponseError(404, "DataStore was deleted",
|
|
879
|
-
|
|
880
|
+
createResponseError(404, "DataStore was deleted", originalRequest, {
|
|
881
|
+
[DeletedResponseHeaderKey]: true,
|
|
882
|
+
}),
|
|
883
|
+
originalRequest,
|
|
880
884
|
);
|
|
881
885
|
}
|
|
882
886
|
|
|
@@ -920,28 +924,69 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
920
924
|
* Checks if the data store has been deleted by GC. If so, log an error.
|
|
921
925
|
* @param id - The data store's id.
|
|
922
926
|
* @param context - The data store context.
|
|
927
|
+
* @param deletedLogSuffix - Whether it was Changed or Requested (will go into the eventName)
|
|
923
928
|
* @param callSite - The function name this is called from.
|
|
924
929
|
* @param requestHeaderData - The request header information to log if the data store is deleted.
|
|
930
|
+
* @param originalRequest - The original request (could be for a child of the DataStore)
|
|
925
931
|
* @returns true if the data store is deleted. Otherwise, returns false.
|
|
926
932
|
*/
|
|
927
933
|
private checkAndLogIfDeleted(
|
|
928
934
|
id: string,
|
|
929
935
|
context: FluidDataStoreContext | undefined,
|
|
930
|
-
deletedLogSuffix:
|
|
936
|
+
deletedLogSuffix: "Changed" | "Requested",
|
|
931
937
|
callSite: string,
|
|
932
938
|
requestHeaderData?: RuntimeHeaderData,
|
|
939
|
+
originalRequest?: IRequest,
|
|
933
940
|
) {
|
|
934
941
|
const dataStoreNodePath = `/${id}`;
|
|
935
942
|
if (!this.isDataStoreDeleted(dataStoreNodePath)) {
|
|
936
943
|
return false;
|
|
937
944
|
}
|
|
938
945
|
|
|
946
|
+
const idToLog =
|
|
947
|
+
originalRequest !== undefined
|
|
948
|
+
? urlToGCNodePath(originalRequest.url)
|
|
949
|
+
: dataStoreNodePath;
|
|
950
|
+
|
|
951
|
+
// Log the package details asynchronously since getInitialSnapshotDetails is async
|
|
952
|
+
const recentelyDeletedContext = this.contexts.getRecentlyDeletedContext(id);
|
|
953
|
+
if (recentelyDeletedContext !== undefined) {
|
|
954
|
+
recentelyDeletedContext
|
|
955
|
+
.getInitialSnapshotDetails()
|
|
956
|
+
.then((details) => {
|
|
957
|
+
return details.pkg.join("/");
|
|
958
|
+
})
|
|
959
|
+
.then(
|
|
960
|
+
(pkg) => ({ pkg, error: undefined }),
|
|
961
|
+
(error) => ({ pkg: undefined, error }),
|
|
962
|
+
)
|
|
963
|
+
.then(({ pkg, error }) => {
|
|
964
|
+
this.mc.logger.sendTelemetryEvent(
|
|
965
|
+
{
|
|
966
|
+
eventName: `GC_DeletedDataStore_PathInfo`,
|
|
967
|
+
...tagCodeArtifacts({
|
|
968
|
+
id: idToLog,
|
|
969
|
+
pkg,
|
|
970
|
+
}),
|
|
971
|
+
callSite,
|
|
972
|
+
},
|
|
973
|
+
error,
|
|
974
|
+
);
|
|
975
|
+
})
|
|
976
|
+
.catch(() => {});
|
|
977
|
+
}
|
|
978
|
+
|
|
939
979
|
this.mc.logger.sendErrorEvent({
|
|
940
980
|
eventName: `GC_Deleted_DataStore_${deletedLogSuffix}`,
|
|
941
|
-
...tagCodeArtifacts({ id }),
|
|
981
|
+
...tagCodeArtifacts({ id: idToLog }),
|
|
942
982
|
callSite,
|
|
943
983
|
headers: JSON.stringify(requestHeaderData),
|
|
944
984
|
exists: context !== undefined,
|
|
985
|
+
details: {
|
|
986
|
+
url: originalRequest?.url,
|
|
987
|
+
headers: JSON.stringify(originalRequest?.headers),
|
|
988
|
+
aliased: this.aliasedDataStores.has(id),
|
|
989
|
+
},
|
|
945
990
|
});
|
|
946
991
|
return true;
|
|
947
992
|
}
|
|
@@ -1198,7 +1243,10 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
1198
1243
|
continue;
|
|
1199
1244
|
}
|
|
1200
1245
|
const dataStoreId = pathParts[1];
|
|
1201
|
-
assert(
|
|
1246
|
+
assert(
|
|
1247
|
+
this.contexts.has(dataStoreId),
|
|
1248
|
+
"updateUnusedRoutes: No data store with specified id",
|
|
1249
|
+
);
|
|
1202
1250
|
// Delete the contexts of unused data stores.
|
|
1203
1251
|
this.contexts.delete(dataStoreId);
|
|
1204
1252
|
// Delete the summarizer node of the unused data stores.
|
|
@@ -1206,6 +1254,28 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
1206
1254
|
}
|
|
1207
1255
|
}
|
|
1208
1256
|
|
|
1257
|
+
private deleteChild(dataStoreId: string) {
|
|
1258
|
+
const dataStoreContext = this.contexts.get(dataStoreId);
|
|
1259
|
+
assert(dataStoreContext !== undefined, 0x2d7 /* No data store with specified id */);
|
|
1260
|
+
|
|
1261
|
+
if (dataStoreContext.isLoaded) {
|
|
1262
|
+
this.mc.logger.sendTelemetryEvent({
|
|
1263
|
+
eventName: "GC_DeletingLoadedDataStore",
|
|
1264
|
+
...tagCodeArtifacts({
|
|
1265
|
+
id: dataStoreId,
|
|
1266
|
+
pkg: dataStoreContext.packagePath.join("/"),
|
|
1267
|
+
}),
|
|
1268
|
+
});
|
|
1269
|
+
}
|
|
1270
|
+
|
|
1271
|
+
dataStoreContext.delete();
|
|
1272
|
+
|
|
1273
|
+
// Delete the contexts of sweep ready data stores.
|
|
1274
|
+
this.contexts.delete(dataStoreId);
|
|
1275
|
+
// Delete the summarizer node of the sweep ready data stores.
|
|
1276
|
+
this.parentContext.deleteChildSummarizerNode?.(dataStoreId);
|
|
1277
|
+
}
|
|
1278
|
+
|
|
1209
1279
|
/**
|
|
1210
1280
|
* Delete data stores and its objects that are sweep ready.
|
|
1211
1281
|
* @param sweepReadyDataStoreRoutes - The routes of data stores and its objects that are sweep ready and should
|
|
@@ -1239,12 +1309,7 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
1239
1309
|
continue;
|
|
1240
1310
|
}
|
|
1241
1311
|
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
// Delete the contexts of sweep ready data stores.
|
|
1245
|
-
this.contexts.delete(dataStoreId);
|
|
1246
|
-
// Delete the summarizer node of the sweep ready data stores.
|
|
1247
|
-
this.parentContext.deleteChildSummarizerNode?.(dataStoreId);
|
|
1312
|
+
this.deleteChild(dataStoreId);
|
|
1248
1313
|
}
|
|
1249
1314
|
return Array.from(sweepReadyDataStoreRoutes);
|
|
1250
1315
|
}
|
|
@@ -1282,8 +1347,9 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
1282
1347
|
*/
|
|
1283
1348
|
private async getOutboundRoutes(): Promise<string[]> {
|
|
1284
1349
|
const outboundRoutes: string[] = [];
|
|
1350
|
+
// Getting this information is a performance optimization that reduces network calls for virtualized datastores
|
|
1285
1351
|
for (const [contextId, context] of this.contexts) {
|
|
1286
|
-
const isRootDataStore = await context.isRoot();
|
|
1352
|
+
const isRootDataStore = await context.isRoot(this.aliasedDataStores);
|
|
1287
1353
|
if (isRootDataStore) {
|
|
1288
1354
|
outboundRoutes.push(`/${contextId}`);
|
|
1289
1355
|
}
|
|
@@ -1344,31 +1410,29 @@ export class ChannelCollection implements IFluidDataStoreChannel, IDisposable {
|
|
|
1344
1410
|
headerData.allowInactive = request.headers[AllowInactiveRequestHeaderKey];
|
|
1345
1411
|
}
|
|
1346
1412
|
|
|
1347
|
-
// We allow Tombstone requests for sub-DataStore objects
|
|
1413
|
+
// We allow Tombstone/Inactive requests for sub-DataStore objects
|
|
1348
1414
|
if (requestForChild) {
|
|
1349
1415
|
headerData.allowTombstone = true;
|
|
1416
|
+
headerData.allowInactive = true;
|
|
1350
1417
|
}
|
|
1351
1418
|
|
|
1352
1419
|
await this.waitIfPendingAlias(id);
|
|
1353
1420
|
const internalId = this.internalId(id);
|
|
1354
|
-
const dataStoreContext = await this.getDataStore(internalId, headerData);
|
|
1421
|
+
const dataStoreContext = await this.getDataStore(internalId, headerData, request);
|
|
1355
1422
|
|
|
1356
|
-
// Remove query params, leading and trailing slashes from the url. This is done to make sure the format is
|
|
1357
|
-
// the same as GC nodes id.
|
|
1358
|
-
const urlWithoutQuery = trimLeadingAndTrailingSlashes(request.url.split("?")[0]);
|
|
1359
1423
|
// Get the initial snapshot details which contain the data store package path.
|
|
1360
1424
|
const details = await dataStoreContext.getInitialSnapshotDetails();
|
|
1361
1425
|
|
|
1362
|
-
//
|
|
1363
|
-
// is
|
|
1364
|
-
this.gcNodeUpdated(
|
|
1365
|
-
`/${
|
|
1366
|
-
"Loaded",
|
|
1367
|
-
|
|
1368
|
-
details.pkg,
|
|
1426
|
+
// When notifying GC of this node being loaded, we only indicate the DataStore itself, not the full subDataStore url if applicable.
|
|
1427
|
+
// This is in case the url is to a route that Fluid doesn't understand or track for GC (e.g. if suited for a custom request handler)
|
|
1428
|
+
this.gcNodeUpdated({
|
|
1429
|
+
node: { type: "DataStore", path: `/${id}` },
|
|
1430
|
+
reason: "Loaded",
|
|
1431
|
+
packagePath: details.pkg,
|
|
1369
1432
|
request,
|
|
1370
1433
|
headerData,
|
|
1371
|
-
);
|
|
1434
|
+
});
|
|
1435
|
+
|
|
1372
1436
|
const dataStore = await dataStoreContext.realize();
|
|
1373
1437
|
|
|
1374
1438
|
const subRequest = requestParser.createSubRequest(1);
|
package/src/containerRuntime.ts
CHANGED
|
@@ -466,6 +466,11 @@ export interface IContainerRuntimeOptions {
|
|
|
466
466
|
readonly enableGroupedBatching?: boolean;
|
|
467
467
|
}
|
|
468
468
|
|
|
469
|
+
/**
|
|
470
|
+
* Error responses when requesting a deleted object will have this header set to true
|
|
471
|
+
* @alpha
|
|
472
|
+
*/
|
|
473
|
+
export const DeletedResponseHeaderKey = "wasDeleted";
|
|
469
474
|
/**
|
|
470
475
|
* Tombstone error responses will have this header set to true
|
|
471
476
|
* @alpha
|
|
@@ -479,6 +484,7 @@ export const InactiveResponseHeaderKey = "isInactive";
|
|
|
479
484
|
|
|
480
485
|
/**
|
|
481
486
|
* The full set of parsed header data that may be found on Runtime requests
|
|
487
|
+
* @internal
|
|
482
488
|
*/
|
|
483
489
|
export interface RuntimeHeaderData {
|
|
484
490
|
wait?: boolean;
|
|
@@ -1512,22 +1518,7 @@ export class ContainerRuntime
|
|
|
1512
1518
|
getSummaryForDatastores(baseSnapshot, metadata),
|
|
1513
1519
|
parentContext,
|
|
1514
1520
|
this.mc.logger,
|
|
1515
|
-
(
|
|
1516
|
-
path: string,
|
|
1517
|
-
reason: "Loaded" | "Changed",
|
|
1518
|
-
timestampMs?: number,
|
|
1519
|
-
packagePath?: readonly string[],
|
|
1520
|
-
request?: IRequest,
|
|
1521
|
-
headerData?: RuntimeHeaderData,
|
|
1522
|
-
) =>
|
|
1523
|
-
this.garbageCollector.nodeUpdated(
|
|
1524
|
-
path,
|
|
1525
|
-
reason,
|
|
1526
|
-
timestampMs,
|
|
1527
|
-
packagePath,
|
|
1528
|
-
request,
|
|
1529
|
-
headerData,
|
|
1530
|
-
),
|
|
1521
|
+
(props) => this.garbageCollector.nodeUpdated(props),
|
|
1531
1522
|
(path: string) => this.garbageCollector.isNodeDeleted(path),
|
|
1532
1523
|
new Map<string, string>(dataStoreAliasMap),
|
|
1533
1524
|
async (runtime: ChannelCollection) => provideEntryPoint,
|
|
@@ -1549,7 +1540,11 @@ export class ContainerRuntime
|
|
|
1549
1540
|
);
|
|
1550
1541
|
}
|
|
1551
1542
|
},
|
|
1552
|
-
(blobPath: string) =>
|
|
1543
|
+
(blobPath: string) =>
|
|
1544
|
+
this.garbageCollector.nodeUpdated({
|
|
1545
|
+
node: { type: "Blob", path: blobPath },
|
|
1546
|
+
reason: "Loaded",
|
|
1547
|
+
}),
|
|
1553
1548
|
(blobPath: string) => this.garbageCollector.isNodeDeleted(blobPath),
|
|
1554
1549
|
this,
|
|
1555
1550
|
pendingRuntimeState?.pendingAttachmentBlobs,
|
|
@@ -2707,12 +2702,11 @@ export class ContainerRuntime
|
|
|
2707
2702
|
"entryPoint must be defined on data store runtime for using getAliasedDataStoreEntryPoint",
|
|
2708
2703
|
);
|
|
2709
2704
|
}
|
|
2710
|
-
this.garbageCollector.nodeUpdated(
|
|
2711
|
-
`/${internalId}
|
|
2712
|
-
"Loaded",
|
|
2713
|
-
|
|
2714
|
-
|
|
2715
|
-
);
|
|
2705
|
+
this.garbageCollector.nodeUpdated({
|
|
2706
|
+
node: { type: "DataStore", path: `/${internalId}` },
|
|
2707
|
+
reason: "Loaded",
|
|
2708
|
+
packagePath: context.packagePath,
|
|
2709
|
+
});
|
|
2716
2710
|
return channel.entryPoint;
|
|
2717
2711
|
}
|
|
2718
2712
|
|
package/src/dataStoreContext.ts
CHANGED
|
@@ -230,8 +230,20 @@ export abstract class FluidDataStoreContext
|
|
|
230
230
|
* 2. is root as part of the base snapshot that the datastore loaded from
|
|
231
231
|
* @returns whether a datastore is root
|
|
232
232
|
*/
|
|
233
|
-
public async isRoot(): Promise<boolean> {
|
|
234
|
-
|
|
233
|
+
public async isRoot(aliasedDataStores?: Set<string>): Promise<boolean> {
|
|
234
|
+
if (this.isInMemoryRoot()) {
|
|
235
|
+
return true;
|
|
236
|
+
}
|
|
237
|
+
|
|
238
|
+
// This if is a performance optimization.
|
|
239
|
+
// We know that if the base snapshot is omitted, then the isRootDataStore flag is not set.
|
|
240
|
+
// That means we can skip the expensive call to getInitialSnapshotDetails for virtualized datastores,
|
|
241
|
+
// and get the information from the alias map directly.
|
|
242
|
+
if (aliasedDataStores !== undefined && this.baseSnapshot?.omitted === true) {
|
|
243
|
+
return aliasedDataStores.has(this.id);
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
return (await this.getInitialSnapshotDetails()).isRootDataStore;
|
|
235
247
|
}
|
|
236
248
|
|
|
237
249
|
/**
|
package/src/dataStoreContexts.ts
CHANGED
|
@@ -80,9 +80,21 @@ export class DataStoreContexts implements Iterable<[string, FluidDataStoreContex
|
|
|
80
80
|
public delete(id: string): boolean {
|
|
81
81
|
this.deferredContexts.delete(id);
|
|
82
82
|
this.notBoundContexts.delete(id);
|
|
83
|
+
|
|
84
|
+
// Stash the context here in case it's requested in this session, we can log some details about it
|
|
85
|
+
const context = this._contexts.get(id);
|
|
86
|
+
this._recentlyDeletedContexts.set(id, context);
|
|
87
|
+
|
|
83
88
|
return this._contexts.delete(id);
|
|
84
89
|
}
|
|
85
90
|
|
|
91
|
+
private readonly _recentlyDeletedContexts: Map<string, FluidDataStoreContext | undefined> =
|
|
92
|
+
new Map();
|
|
93
|
+
|
|
94
|
+
public getRecentlyDeletedContext(id: string) {
|
|
95
|
+
return this._recentlyDeletedContexts.get(id);
|
|
96
|
+
}
|
|
97
|
+
|
|
86
98
|
/**
|
|
87
99
|
* Return the unbound local context with the given id,
|
|
88
100
|
* or undefined if it's not found or not unbound.
|
|
@@ -23,11 +23,7 @@ import {
|
|
|
23
23
|
tagCodeArtifacts,
|
|
24
24
|
} from "@fluidframework/telemetry-utils";
|
|
25
25
|
import { BlobManager } from "../blobManager.js";
|
|
26
|
-
import {
|
|
27
|
-
InactiveResponseHeaderKey,
|
|
28
|
-
RuntimeHeaderData,
|
|
29
|
-
TombstoneResponseHeaderKey,
|
|
30
|
-
} from "../containerRuntime.js";
|
|
26
|
+
import { InactiveResponseHeaderKey, TombstoneResponseHeaderKey } from "../containerRuntime.js";
|
|
31
27
|
import { ClientSessionExpiredError } from "../error.js";
|
|
32
28
|
import { ContainerMessageType, ContainerRuntimeGCMessage } from "../messageTypes.js";
|
|
33
29
|
import { IRefreshSummaryResult } from "../summary/index.js";
|
|
@@ -47,12 +43,15 @@ import {
|
|
|
47
43
|
GarbageCollectionMessage,
|
|
48
44
|
GarbageCollectionMessageType,
|
|
49
45
|
disableAutoRecoveryKey,
|
|
46
|
+
type IGCNodeUpdatedProps,
|
|
50
47
|
} from "./gcDefinitions.js";
|
|
51
48
|
import {
|
|
52
49
|
cloneGCData,
|
|
53
50
|
compatBehaviorAllowsGCMessageType,
|
|
54
51
|
concatGarbageCollectionData,
|
|
52
|
+
dataStoreNodePathOnly,
|
|
55
53
|
getGCDataFromSnapshot,
|
|
54
|
+
urlToGCNodePath,
|
|
56
55
|
} from "./gcHelpers.js";
|
|
57
56
|
import { runGarbageCollection } from "./gcReferenceGraphAlgorithm.js";
|
|
58
57
|
import { IGarbageCollectionSnapshotData, IGarbageCollectionState } from "./gcSummaryDefinitions.js";
|
|
@@ -400,17 +399,27 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
400
399
|
return;
|
|
401
400
|
}
|
|
402
401
|
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
402
|
+
const initialized = this.gcDataFromLastRun !== undefined;
|
|
403
|
+
await PerformanceEvent.timedExecAsync(
|
|
404
|
+
this.mc.logger,
|
|
405
|
+
{
|
|
406
|
+
eventName: "InitializeOrUpdateGCState",
|
|
407
|
+
details: { initialized, unrefNodeCount: this.unreferencedNodesState.size },
|
|
408
|
+
},
|
|
409
|
+
async () => {
|
|
410
|
+
// If the GC state hasn't been initialized yet, initialize it and return.
|
|
411
|
+
if (!initialized) {
|
|
412
|
+
await this.initializeGCStateFromBaseSnapshotP;
|
|
413
|
+
return;
|
|
414
|
+
}
|
|
408
415
|
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
416
|
+
// If the GC state has been initialized, update the tracking of unreferenced nodes as per the current
|
|
417
|
+
// reference timestamp.
|
|
418
|
+
for (const [, nodeStateTracker] of this.unreferencedNodesState) {
|
|
419
|
+
nodeStateTracker.updateTracking(currentReferenceTimestampMs);
|
|
420
|
+
}
|
|
421
|
+
},
|
|
422
|
+
);
|
|
414
423
|
}
|
|
415
424
|
|
|
416
425
|
/**
|
|
@@ -977,30 +986,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
977
986
|
/**
|
|
978
987
|
* Called when a node with the given id is updated. If the node is inactive or tombstoned, this will log an error
|
|
979
988
|
* or throw an error if failing on incorrect usage is configured.
|
|
980
|
-
* @param
|
|
981
|
-
* @param reason - Whether the node was loaded or changed.
|
|
982
|
-
* @param timestampMs - The timestamp when the node changed.
|
|
983
|
-
* @param packagePath - The package path of the node. This may not be available if the node hasn't been loaded yet.
|
|
984
|
-
* @param request - The original request for loads to preserve it in telemetry.
|
|
985
|
-
* @param requestHeaders - If the node was loaded via request path, the headers in the request.
|
|
989
|
+
* @param IGCNodeUpdatedProps - Details about the node and how it was updated
|
|
986
990
|
*/
|
|
987
|
-
public nodeUpdated(
|
|
988
|
-
|
|
989
|
-
reason
|
|
990
|
-
timestampMs
|
|
991
|
-
packagePath
|
|
992
|
-
request
|
|
993
|
-
headerData
|
|
994
|
-
) {
|
|
991
|
+
public nodeUpdated({
|
|
992
|
+
node,
|
|
993
|
+
reason,
|
|
994
|
+
timestampMs,
|
|
995
|
+
packagePath,
|
|
996
|
+
request,
|
|
997
|
+
headerData,
|
|
998
|
+
}: IGCNodeUpdatedProps) {
|
|
995
999
|
if (!this.configs.shouldRunGC) {
|
|
996
1000
|
return;
|
|
997
1001
|
}
|
|
998
1002
|
|
|
999
|
-
|
|
1003
|
+
// trackedId will be either DataStore or Blob ID (not sub-DataStore ID, since some of those are unrecognized by GC)
|
|
1004
|
+
const trackedId = node.path;
|
|
1005
|
+
const isTombstoned = this.tombstones.includes(trackedId);
|
|
1006
|
+
const isInactive = this.unreferencedNodesState.get(trackedId)?.state === "Inactive";
|
|
1007
|
+
|
|
1008
|
+
const fullPath = request !== undefined ? urlToGCNodePath(request.url) : trackedId;
|
|
1000
1009
|
|
|
1001
1010
|
// This will log if appropriate
|
|
1002
|
-
this.telemetryTracker.nodeUsed({
|
|
1003
|
-
id:
|
|
1011
|
+
this.telemetryTracker.nodeUsed(trackedId, {
|
|
1012
|
+
id: fullPath,
|
|
1004
1013
|
usageType: reason,
|
|
1005
1014
|
currentReferenceTimestampMs:
|
|
1006
1015
|
timestampMs ?? this.runtime.getCurrentReferenceTimestampMs(),
|
|
@@ -1009,6 +1018,8 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1009
1018
|
isTombstoned,
|
|
1010
1019
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1011
1020
|
headers: headerData,
|
|
1021
|
+
requestUrl: request?.url,
|
|
1022
|
+
requestHeaders: JSON.stringify(request?.headers),
|
|
1012
1023
|
});
|
|
1013
1024
|
|
|
1014
1025
|
// Any time we log a Tombstone Loaded error (via Telemetry Tracker),
|
|
@@ -1017,17 +1028,20 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1017
1028
|
// to be loaded by the Summarizer, and auto-recovery will be triggered then.
|
|
1018
1029
|
if (isTombstoned && reason === "Loaded") {
|
|
1019
1030
|
// Note that when a DataStore and its DDS are all loaded, each will trigger AutoRecovery for itself.
|
|
1020
|
-
this.triggerAutoRecovery(
|
|
1031
|
+
this.triggerAutoRecovery(fullPath);
|
|
1021
1032
|
}
|
|
1022
1033
|
|
|
1023
|
-
const nodeType = this.runtime.getNodeType(
|
|
1034
|
+
const nodeType = this.runtime.getNodeType(fullPath);
|
|
1024
1035
|
|
|
1025
1036
|
// Unless this is a Loaded event for a Blob or DataStore, we're done after telemetry tracking
|
|
1026
|
-
|
|
1037
|
+
const loadedBlobOrDataStore =
|
|
1038
|
+
reason === "Loaded" &&
|
|
1039
|
+
(nodeType === GCNodeType.Blob || nodeType === GCNodeType.DataStore);
|
|
1040
|
+
if (!loadedBlobOrDataStore) {
|
|
1027
1041
|
return;
|
|
1028
1042
|
}
|
|
1029
1043
|
|
|
1030
|
-
const errorRequest: IRequest = request ?? { url:
|
|
1044
|
+
const errorRequest: IRequest = request ?? { url: fullPath };
|
|
1031
1045
|
if (isTombstoned && this.throwOnTombstoneLoad && headerData?.allowTombstone !== true) {
|
|
1032
1046
|
// The requested data store is removed by gc. Create a 404 gc response exception.
|
|
1033
1047
|
throw responseToException(
|
|
@@ -1039,7 +1053,7 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1039
1053
|
}
|
|
1040
1054
|
|
|
1041
1055
|
// If the object is inactive and inactive enforcement is configured, throw an error.
|
|
1042
|
-
if (
|
|
1056
|
+
if (isInactive) {
|
|
1043
1057
|
const shouldThrowOnInactiveLoad =
|
|
1044
1058
|
!this.isSummarizerClient &&
|
|
1045
1059
|
this.configs.throwOnInactiveLoad === true &&
|
|
@@ -1111,22 +1125,30 @@ export class GarbageCollector implements IGarbageCollector {
|
|
|
1111
1125
|
outboundRoutes.push(toNodePath);
|
|
1112
1126
|
this.newReferencesSinceLastRun.set(fromNodePath, outboundRoutes);
|
|
1113
1127
|
|
|
1114
|
-
|
|
1128
|
+
// GC won't recognize some subDataStore paths that we encounter (e.g. a path suited for a custom request handler)
|
|
1129
|
+
// So for subDataStore paths we need to check the parent dataStore for current tombstone/inactive status.
|
|
1130
|
+
const trackedId =
|
|
1131
|
+
this.runtime.getNodeType(toNodePath) === "SubDataStore"
|
|
1132
|
+
? dataStoreNodePathOnly(toNodePath)
|
|
1133
|
+
: toNodePath;
|
|
1134
|
+
this.telemetryTracker.nodeUsed(trackedId, {
|
|
1115
1135
|
id: toNodePath,
|
|
1136
|
+
fromId: fromNodePath,
|
|
1116
1137
|
usageType: "Revived",
|
|
1117
1138
|
currentReferenceTimestampMs: this.runtime.getCurrentReferenceTimestampMs(),
|
|
1118
1139
|
packagePath: undefined,
|
|
1119
1140
|
completedGCRuns: this.completedRuns,
|
|
1120
|
-
isTombstoned: this.tombstones.includes(
|
|
1141
|
+
isTombstoned: this.tombstones.includes(trackedId),
|
|
1121
1142
|
lastSummaryTime: this.getLastSummaryTimestampMs(),
|
|
1122
|
-
fromId: fromNodePath,
|
|
1123
1143
|
autorecovery,
|
|
1124
1144
|
});
|
|
1125
1145
|
|
|
1126
|
-
// This node is referenced - Clear its unreferenced state
|
|
1146
|
+
// This node is referenced - Clear its unreferenced state if present
|
|
1127
1147
|
// But don't delete the node id from the map yet.
|
|
1128
1148
|
// When generating GC stats, the set of nodes in here is used as the baseline for
|
|
1129
1149
|
// what was unreferenced in the last GC run.
|
|
1150
|
+
// NOTE: We use toNodePath not trackedId even though it may be an unrecognized subDataStore route (hence no-op),
|
|
1151
|
+
// because a reference to such a path is not sufficient to consider the DataStore referenced.
|
|
1130
1152
|
this.unreferencedNodesState.get(toNodePath)?.stopTracking();
|
|
1131
1153
|
}
|
|
1132
1154
|
|
package/src/gc/gcDefinitions.ts
CHANGED
|
@@ -245,7 +245,7 @@ export const GCNodeType = {
|
|
|
245
245
|
Blob: "Blob",
|
|
246
246
|
// Nodes that are neither of the above. For example, root node.
|
|
247
247
|
Other: "Other",
|
|
248
|
-
};
|
|
248
|
+
} as const;
|
|
249
249
|
|
|
250
250
|
/**
|
|
251
251
|
* @alpha
|
|
@@ -372,14 +372,7 @@ export interface IGarbageCollector {
|
|
|
372
372
|
* Called when a node with the given path is updated. If the node is inactive or tombstoned, this will log an error
|
|
373
373
|
* or throw an error if failing on incorrect usage is configured.
|
|
374
374
|
*/
|
|
375
|
-
nodeUpdated(
|
|
376
|
-
nodePath: string,
|
|
377
|
-
reason: "Loaded" | "Changed",
|
|
378
|
-
timestampMs?: number,
|
|
379
|
-
packagePath?: readonly string[],
|
|
380
|
-
request?: IRequest,
|
|
381
|
-
headerData?: RuntimeHeaderData,
|
|
382
|
-
): void;
|
|
375
|
+
nodeUpdated(props: IGCNodeUpdatedProps): void;
|
|
383
376
|
/** Called when a reference is added to a node. Used to identify nodes that were referenced between summaries. */
|
|
384
377
|
addedOutboundReference(fromNodePath: string, toNodePath: string, autorecovery?: true): void;
|
|
385
378
|
/** Called to process a garbage collection message. */
|
|
@@ -390,6 +383,25 @@ export interface IGarbageCollector {
|
|
|
390
383
|
dispose(): void;
|
|
391
384
|
}
|
|
392
385
|
|
|
386
|
+
/**
|
|
387
|
+
* Info needed by GC when notified that a node was updated (loaded or changed)
|
|
388
|
+
* @internal
|
|
389
|
+
*/
|
|
390
|
+
export interface IGCNodeUpdatedProps {
|
|
391
|
+
/** Type and path of the updated node */
|
|
392
|
+
node: { type: (typeof GCNodeType)["DataStore" | "Blob"]; path: string };
|
|
393
|
+
/** Whether the node (or a subpath) was loaded or changed. */
|
|
394
|
+
reason: "Loaded" | "Changed";
|
|
395
|
+
/** The op-based timestamp when the node changed, if applicable */
|
|
396
|
+
timestampMs?: number;
|
|
397
|
+
/** The package path of the node. This may not be available if the node hasn't been loaded yet */
|
|
398
|
+
packagePath?: readonly string[];
|
|
399
|
+
/** The original request for loads to preserve it in telemetry */
|
|
400
|
+
request?: IRequest;
|
|
401
|
+
/** If the node was loaded via request path, the header data. May be modified from the original request */
|
|
402
|
+
headerData?: RuntimeHeaderData;
|
|
403
|
+
}
|
|
404
|
+
|
|
393
405
|
/** Parameters necessary for creating a GarbageCollector. */
|
|
394
406
|
export interface IGarbageCollectorCreateParams {
|
|
395
407
|
readonly runtime: IGarbageCollectionRuntime;
|
package/src/gc/gcHelpers.ts
CHANGED
|
@@ -263,10 +263,23 @@ export function unpackChildNodesGCDetails(gcDetails: IGarbageCollectionDetailsBa
|
|
|
263
263
|
* @param str - A string that may contain leading and / or trailing slashes.
|
|
264
264
|
* @returns A new string without leading and trailing slashes.
|
|
265
265
|
*/
|
|
266
|
-
|
|
266
|
+
function trimLeadingAndTrailingSlashes(str: string) {
|
|
267
267
|
return str.replace(/^\/+|\/+$/g, "");
|
|
268
268
|
}
|
|
269
269
|
|
|
270
|
+
/** Reformats a request URL to match expected format for a GC node path */
|
|
271
|
+
export function urlToGCNodePath(url: string): string {
|
|
272
|
+
return `/${trimLeadingAndTrailingSlashes(url.split("?")[0])}`;
|
|
273
|
+
}
|
|
274
|
+
|
|
275
|
+
/**
|
|
276
|
+
* Pulls out the first path segment and formats it as a GC Node path
|
|
277
|
+
* e.g. "/dataStoreId/ddsId" yields "/dataStoreId"
|
|
278
|
+
*/
|
|
279
|
+
export function dataStoreNodePathOnly(subDataStorePath: string): string {
|
|
280
|
+
return `/${subDataStorePath.split("/")[1]}`;
|
|
281
|
+
}
|
|
282
|
+
|
|
270
283
|
/**
|
|
271
284
|
* Utility to implement compat behaviors given an unknown message type
|
|
272
285
|
* The parameters are typed to support compile-time enforcement of handling all known types/behaviors
|