@fluidframework/routerlicious-driver 2.0.0-internal.4.1.2 → 2.0.0-internal.4.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/contracts.d.ts +15 -0
- package/dist/contracts.d.ts.map +1 -0
- package/dist/contracts.js +7 -0
- package/dist/contracts.js.map +1 -0
- package/dist/documentService.d.ts +4 -2
- package/dist/documentService.d.ts.map +1 -1
- package/dist/documentService.js +5 -4
- package/dist/documentService.js.map +1 -1
- package/dist/documentServiceFactory.d.ts +2 -1
- package/dist/documentServiceFactory.d.ts.map +1 -1
- package/dist/documentServiceFactory.js +11 -4
- package/dist/documentServiceFactory.js.map +1 -1
- package/dist/documentStorageService.d.ts +3 -2
- package/dist/documentStorageService.d.ts.map +1 -1
- package/dist/documentStorageService.js +4 -4
- package/dist/documentStorageService.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/r11sSnapshotParser.d.ts +15 -0
- package/dist/r11sSnapshotParser.d.ts.map +1 -0
- package/dist/r11sSnapshotParser.js +75 -0
- package/dist/r11sSnapshotParser.js.map +1 -0
- package/dist/restWrapper.d.ts +3 -3
- package/dist/restWrapper.d.ts.map +1 -1
- package/dist/restWrapper.js +19 -15
- package/dist/restWrapper.js.map +1 -1
- package/dist/treeUtils.d.ts +1 -1
- package/dist/treeUtils.d.ts.map +1 -1
- package/dist/treeUtils.js.map +1 -1
- package/dist/wholeSummaryDocumentStorageService.d.ts +4 -3
- package/dist/wholeSummaryDocumentStorageService.d.ts.map +1 -1
- package/dist/wholeSummaryDocumentStorageService.js +57 -26
- package/dist/wholeSummaryDocumentStorageService.js.map +1 -1
- package/lib/contracts.d.ts +15 -0
- package/lib/contracts.d.ts.map +1 -0
- package/lib/contracts.js +6 -0
- package/lib/contracts.js.map +1 -0
- package/lib/documentService.d.ts +4 -2
- package/lib/documentService.d.ts.map +1 -1
- package/lib/documentService.js +5 -4
- package/lib/documentService.js.map +1 -1
- package/lib/documentServiceFactory.d.ts +2 -1
- package/lib/documentServiceFactory.d.ts.map +1 -1
- package/lib/documentServiceFactory.js +11 -4
- package/lib/documentServiceFactory.js.map +1 -1
- package/lib/documentStorageService.d.ts +3 -2
- package/lib/documentStorageService.d.ts.map +1 -1
- package/lib/documentStorageService.js +4 -4
- package/lib/documentStorageService.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/r11sSnapshotParser.d.ts +15 -0
- package/lib/r11sSnapshotParser.d.ts.map +1 -0
- package/lib/r11sSnapshotParser.js +71 -0
- package/lib/r11sSnapshotParser.js.map +1 -0
- package/lib/restWrapper.d.ts +3 -3
- package/lib/restWrapper.d.ts.map +1 -1
- package/lib/restWrapper.js +20 -16
- package/lib/restWrapper.js.map +1 -1
- package/lib/treeUtils.d.ts +1 -1
- package/lib/treeUtils.d.ts.map +1 -1
- package/lib/treeUtils.js.map +1 -1
- package/lib/wholeSummaryDocumentStorageService.d.ts +4 -3
- package/lib/wholeSummaryDocumentStorageService.d.ts.map +1 -1
- package/lib/wholeSummaryDocumentStorageService.js +56 -25
- package/lib/wholeSummaryDocumentStorageService.js.map +1 -1
- package/package.json +8 -8
- package/src/contracts.ts +16 -0
- package/src/documentService.ts +6 -3
- package/src/documentServiceFactory.ts +16 -5
- package/src/documentStorageService.ts +8 -4
- package/src/packageVersion.ts +1 -1
- package/src/r11sSnapshotParser.ts +83 -0
- package/src/restWrapper.ts +15 -18
- package/src/treeUtils.ts +1 -1
- package/src/wholeSummaryDocumentStorageService.ts +79 -38
package/src/documentService.ts
CHANGED
|
@@ -24,6 +24,7 @@ import { pkgVersion as driverVersion } from "./packageVersion";
|
|
|
24
24
|
import { GitManager } from "./gitManager";
|
|
25
25
|
import { Historian } from "./historian";
|
|
26
26
|
import { RestWrapper } from "./restWrapperBase";
|
|
27
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
27
28
|
|
|
28
29
|
/**
|
|
29
30
|
* Amount of time between discoveries within which we don't need to rediscover on re-connect.
|
|
@@ -63,7 +64,8 @@ export class DocumentService implements api.IDocumentService {
|
|
|
63
64
|
private readonly documentStorageServicePolicies: api.IDocumentStorageServicePolicies,
|
|
64
65
|
private readonly driverPolicies: IRouterliciousDriverPolicies,
|
|
65
66
|
private readonly blobCache: ICache<ArrayBufferLike>,
|
|
66
|
-
private readonly
|
|
67
|
+
private readonly wholeSnapshotTreeCache: ICache<INormalizedWholeSummary>,
|
|
68
|
+
private readonly shreddedSummaryTreeCache: ICache<ISnapshotTreeVersion>,
|
|
67
69
|
private readonly discoverFluidResolvedUrl: () => Promise<api.IFluidResolvedUrl>,
|
|
68
70
|
) {}
|
|
69
71
|
|
|
@@ -125,7 +127,8 @@ export class DocumentService implements api.IDocumentService {
|
|
|
125
127
|
this.documentStorageServicePolicies,
|
|
126
128
|
this.driverPolicies,
|
|
127
129
|
this.blobCache,
|
|
128
|
-
this.
|
|
130
|
+
this.wholeSnapshotTreeCache,
|
|
131
|
+
this.shreddedSummaryTreeCache,
|
|
129
132
|
noCacheStorageManager,
|
|
130
133
|
getStorageManager,
|
|
131
134
|
);
|
|
@@ -184,7 +187,7 @@ export class DocumentService implements api.IDocumentService {
|
|
|
184
187
|
*/
|
|
185
188
|
public async connectToDeltaStream(client: IClient): Promise<api.IDocumentDeltaConnection> {
|
|
186
189
|
const connect = async (refreshToken?: boolean) => {
|
|
187
|
-
let ordererToken = this.ordererRestWrapper.getToken();
|
|
190
|
+
let ordererToken = await this.ordererRestWrapper.getToken();
|
|
188
191
|
if (this.shouldUpdateDiscoveredSessionInfo()) {
|
|
189
192
|
await this.refreshDiscovery();
|
|
190
193
|
}
|
|
@@ -34,6 +34,7 @@ import { parseFluidUrl, replaceDocumentIdInPath, getDiscoveredFluidResolvedUrl }
|
|
|
34
34
|
import { ICache, InMemoryCache, NullCache } from "./cache";
|
|
35
35
|
import { pkgVersion as driverVersion } from "./packageVersion";
|
|
36
36
|
import { ISnapshotTreeVersion } from "./definitions";
|
|
37
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
37
38
|
|
|
38
39
|
const maximumSnapshotCacheDurationMs: FiveDaysMs = 432_000_000; // 5 days in ms
|
|
39
40
|
|
|
@@ -55,7 +56,8 @@ const defaultRouterliciousDriverPolicies: IRouterliciousDriverPolicies = {
|
|
|
55
56
|
export class RouterliciousDocumentServiceFactory implements IDocumentServiceFactory {
|
|
56
57
|
private readonly driverPolicies: IRouterliciousDriverPolicies;
|
|
57
58
|
private readonly blobCache: ICache<ArrayBufferLike>;
|
|
58
|
-
private readonly
|
|
59
|
+
private readonly wholeSnapshotTreeCache: ICache<INormalizedWholeSummary> = new NullCache();
|
|
60
|
+
private readonly shreddedSummaryTreeCache: ICache<ISnapshotTreeVersion> = new NullCache();
|
|
59
61
|
|
|
60
62
|
constructor(
|
|
61
63
|
private readonly tokenProvider: ITokenProvider,
|
|
@@ -69,9 +71,17 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
|
|
|
69
71
|
...driverPolicies,
|
|
70
72
|
};
|
|
71
73
|
this.blobCache = new InMemoryCache<ArrayBufferLike>();
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
74
|
+
if (this.driverPolicies.enableInternalSummaryCaching) {
|
|
75
|
+
if (this.driverPolicies.enableWholeSummaryUpload) {
|
|
76
|
+
this.wholeSnapshotTreeCache = new InMemoryCache<INormalizedWholeSummary>(
|
|
77
|
+
snapshotCacheExpiryMs,
|
|
78
|
+
);
|
|
79
|
+
} else {
|
|
80
|
+
this.shreddedSummaryTreeCache = new InMemoryCache<ISnapshotTreeVersion>(
|
|
81
|
+
snapshotCacheExpiryMs,
|
|
82
|
+
);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
75
85
|
}
|
|
76
86
|
|
|
77
87
|
/**
|
|
@@ -307,7 +317,8 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
|
|
|
307
317
|
documentStorageServicePolicies,
|
|
308
318
|
this.driverPolicies,
|
|
309
319
|
this.blobCache,
|
|
310
|
-
this.
|
|
320
|
+
this.wholeSnapshotTreeCache,
|
|
321
|
+
this.shreddedSummaryTreeCache,
|
|
311
322
|
discoverFluidResolvedUrl,
|
|
312
323
|
);
|
|
313
324
|
}
|
|
@@ -18,8 +18,9 @@ import { IRouterliciousDriverPolicies } from "./policies";
|
|
|
18
18
|
import { ICache } from "./cache";
|
|
19
19
|
import { WholeSummaryDocumentStorageService } from "./wholeSummaryDocumentStorageService";
|
|
20
20
|
import { ShreddedSummaryDocumentStorageService } from "./shreddedSummaryDocumentStorageService";
|
|
21
|
-
import { ISnapshotTreeVersion } from "./definitions";
|
|
22
21
|
import { GitManager } from "./gitManager";
|
|
22
|
+
import { ISnapshotTreeVersion } from "./definitions";
|
|
23
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
23
24
|
|
|
24
25
|
export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
25
26
|
private _logTailSha: string | undefined = undefined;
|
|
@@ -35,7 +36,8 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
35
36
|
policies: IDocumentStorageServicePolicies,
|
|
36
37
|
driverPolicies?: IRouterliciousDriverPolicies,
|
|
37
38
|
blobCache?: ICache<ArrayBufferLike>,
|
|
38
|
-
snapshotTreeCache?: ICache<
|
|
39
|
+
snapshotTreeCache?: ICache<INormalizedWholeSummary>,
|
|
40
|
+
shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
|
|
39
41
|
noCacheGitManager?: GitManager,
|
|
40
42
|
getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
|
|
41
43
|
): IDocumentStorageService {
|
|
@@ -58,7 +60,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
58
60
|
policies,
|
|
59
61
|
driverPolicies,
|
|
60
62
|
blobCache,
|
|
61
|
-
|
|
63
|
+
shreddedSummaryTreeCache,
|
|
62
64
|
getStorageManager,
|
|
63
65
|
);
|
|
64
66
|
// TODO: worth prefetching latest summary making version + snapshot call with WholeSummary storage?
|
|
@@ -78,7 +80,8 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
78
80
|
policies: IDocumentStorageServicePolicies,
|
|
79
81
|
driverPolicies?: IRouterliciousDriverPolicies,
|
|
80
82
|
blobCache?: ICache<ArrayBufferLike>,
|
|
81
|
-
snapshotTreeCache?: ICache<
|
|
83
|
+
snapshotTreeCache?: ICache<INormalizedWholeSummary>,
|
|
84
|
+
shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
|
|
82
85
|
public noCacheGitManager?: GitManager,
|
|
83
86
|
getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
|
|
84
87
|
) {
|
|
@@ -91,6 +94,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
91
94
|
driverPolicies,
|
|
92
95
|
blobCache,
|
|
93
96
|
snapshotTreeCache,
|
|
97
|
+
shreddedSummaryTreeCache,
|
|
94
98
|
noCacheGitManager,
|
|
95
99
|
getStorageManager,
|
|
96
100
|
),
|
package/src/packageVersion.ts
CHANGED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { ISnapshotTree } from "@fluidframework/protocol-definitions";
|
|
7
|
+
import { IWholeFlatSummary, IWholeFlatSummaryTree } from "@fluidframework/server-services-client";
|
|
8
|
+
import { stringToBuffer } from "@fluidframework/common-utils";
|
|
9
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
10
|
+
|
|
11
|
+
/**
|
|
12
|
+
* Build a tree heirarchy from a flat tree.
|
|
13
|
+
*
|
|
14
|
+
* @param flatTree - a flat tree
|
|
15
|
+
* @param treePrefixToRemove - tree prefix to strip
|
|
16
|
+
* @returns the heirarchical tree
|
|
17
|
+
*/
|
|
18
|
+
function buildHierarchy(
|
|
19
|
+
flatTree: IWholeFlatSummaryTree,
|
|
20
|
+
treePrefixToRemove: string,
|
|
21
|
+
): ISnapshotTree {
|
|
22
|
+
const lookup: { [path: string]: ISnapshotTree } = {};
|
|
23
|
+
// Root tree id will be used to determine which version was downloaded.
|
|
24
|
+
const root: ISnapshotTree = { id: flatTree.id, blobs: {}, trees: {} };
|
|
25
|
+
lookup[""] = root;
|
|
26
|
+
|
|
27
|
+
for (const entry of flatTree.entries) {
|
|
28
|
+
// Strip the `treePrefixToRemove` path from tree entries such that they are stored under root.
|
|
29
|
+
const entryPath = entry.path.replace(new RegExp(`^${treePrefixToRemove}/`), "");
|
|
30
|
+
const lastIndex = entryPath.lastIndexOf("/");
|
|
31
|
+
const entryPathDir = entryPath.slice(0, Math.max(0, lastIndex));
|
|
32
|
+
const entryPathBase = entryPath.slice(lastIndex + 1);
|
|
33
|
+
|
|
34
|
+
// The flat output is breadth-first so we can assume we see tree nodes prior to their contents
|
|
35
|
+
const node = lookup[entryPathDir];
|
|
36
|
+
|
|
37
|
+
// Add in either the blob or tree
|
|
38
|
+
if (entry.type === "tree") {
|
|
39
|
+
const newTree: ISnapshotTree = {
|
|
40
|
+
blobs: {},
|
|
41
|
+
trees: {},
|
|
42
|
+
unreferenced: entry.unreferenced,
|
|
43
|
+
};
|
|
44
|
+
node.trees[decodeURIComponent(entryPathBase)] = newTree;
|
|
45
|
+
lookup[entryPath] = newTree;
|
|
46
|
+
} else if (entry.type === "blob") {
|
|
47
|
+
node.blobs[decodeURIComponent(entryPathBase)] = entry.id;
|
|
48
|
+
} else {
|
|
49
|
+
throw new Error(`Unknown entry type!!`);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
return root;
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
/**
|
|
57
|
+
* Converts existing IWholeFlatSummary to snapshot tree, blob array, and sequence number.
|
|
58
|
+
*
|
|
59
|
+
* @param flatSummary - flat summary
|
|
60
|
+
* @param treePrefixToRemove - tree prefix to strip. By default we are stripping ".app" prefix
|
|
61
|
+
* @returns snapshot tree, blob array, and sequence number
|
|
62
|
+
*/
|
|
63
|
+
export function convertWholeFlatSummaryToSnapshotTreeAndBlobs(
|
|
64
|
+
flatSummary: IWholeFlatSummary,
|
|
65
|
+
treePrefixToRemove: string = ".app",
|
|
66
|
+
): INormalizedWholeSummary {
|
|
67
|
+
const blobs = new Map<string, ArrayBuffer>();
|
|
68
|
+
if (flatSummary.blobs) {
|
|
69
|
+
flatSummary.blobs.forEach((blob) => {
|
|
70
|
+
blobs.set(blob.id, stringToBuffer(blob.content, blob.encoding ?? "utf-8"));
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
const flatSummaryTree = flatSummary.trees?.[0];
|
|
74
|
+
const sequenceNumber = flatSummaryTree?.sequenceNumber;
|
|
75
|
+
const snapshotTree = buildHierarchy(flatSummaryTree, treePrefixToRemove);
|
|
76
|
+
|
|
77
|
+
return {
|
|
78
|
+
blobs,
|
|
79
|
+
snapshotTree,
|
|
80
|
+
sequenceNumber,
|
|
81
|
+
id: flatSummary.id,
|
|
82
|
+
};
|
|
83
|
+
}
|
package/src/restWrapper.ts
CHANGED
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { ITelemetryLogger, ITelemetryProperties } from "@fluidframework/common-definitions";
|
|
7
|
-
import { fromUtf8ToBase64, performance } from "@fluidframework/common-utils";
|
|
7
|
+
import { assert, fromUtf8ToBase64, performance } from "@fluidframework/common-utils";
|
|
8
8
|
import { RateLimiter } from "@fluidframework/driver-utils";
|
|
9
9
|
import {
|
|
10
10
|
getAuthorizationTokenFromCredentials,
|
|
@@ -99,11 +99,11 @@ export function getPropsToLogFromResponse(headers: {
|
|
|
99
99
|
|
|
100
100
|
export class RouterliciousRestWrapper extends RestWrapper {
|
|
101
101
|
private readonly restLess = new RestLessClient();
|
|
102
|
+
private token: ITokenResponse | undefined;
|
|
102
103
|
|
|
103
104
|
constructor(
|
|
104
105
|
logger: ITelemetryLogger,
|
|
105
106
|
private readonly rateLimiter: RateLimiter,
|
|
106
|
-
private token: ITokenResponse,
|
|
107
107
|
private readonly fetchRefreshedToken: TokenFetcher,
|
|
108
108
|
private readonly getAuthorizationHeader: AuthorizationHeaderGetter,
|
|
109
109
|
private readonly useRestLess: boolean,
|
|
@@ -120,7 +120,7 @@ export class RouterliciousRestWrapper extends RestWrapper {
|
|
|
120
120
|
): Promise<IR11sResponse<T>> {
|
|
121
121
|
const config = {
|
|
122
122
|
...requestConfig,
|
|
123
|
-
headers: this.generateHeaders(requestConfig.headers),
|
|
123
|
+
headers: await this.generateHeaders(requestConfig.headers),
|
|
124
124
|
};
|
|
125
125
|
|
|
126
126
|
const translatedConfig = this.useRestLess ? this.restLess.translate(config) : config;
|
|
@@ -199,9 +199,11 @@ export class RouterliciousRestWrapper extends RestWrapper {
|
|
|
199
199
|
);
|
|
200
200
|
}
|
|
201
201
|
|
|
202
|
-
private generateHeaders(
|
|
202
|
+
private async generateHeaders(
|
|
203
203
|
requestHeaders?: AxiosRequestHeaders | undefined,
|
|
204
|
-
): Record<string, string
|
|
204
|
+
): Promise<Record<string, string>> {
|
|
205
|
+
const token = await this.getToken();
|
|
206
|
+
assert(token !== undefined, 0x679 /* token should be present */);
|
|
205
207
|
const correlationId = requestHeaders?.["x-correlation-id"] ?? uuid();
|
|
206
208
|
|
|
207
209
|
return {
|
|
@@ -211,12 +213,17 @@ export class RouterliciousRestWrapper extends RestWrapper {
|
|
|
211
213
|
"x-correlation-id": correlationId as string,
|
|
212
214
|
"x-driver-version": driverVersion,
|
|
213
215
|
// NOTE: If this.authorizationHeader is undefined, should "Authorization" be removed entirely?
|
|
214
|
-
"Authorization": this.getAuthorizationHeader(
|
|
216
|
+
"Authorization": this.getAuthorizationHeader(token),
|
|
215
217
|
};
|
|
216
218
|
}
|
|
217
219
|
|
|
218
|
-
public getToken(): ITokenResponse {
|
|
219
|
-
|
|
220
|
+
public async getToken(): Promise<ITokenResponse> {
|
|
221
|
+
if (this.token !== undefined) {
|
|
222
|
+
return this.token;
|
|
223
|
+
}
|
|
224
|
+
const token = await this.fetchRefreshedToken();
|
|
225
|
+
this.setToken(token);
|
|
226
|
+
return token;
|
|
220
227
|
}
|
|
221
228
|
|
|
222
229
|
public setToken(token: ITokenResponse) {
|
|
@@ -228,7 +235,6 @@ export class RouterliciousStorageRestWrapper extends RouterliciousRestWrapper {
|
|
|
228
235
|
private constructor(
|
|
229
236
|
logger: ITelemetryLogger,
|
|
230
237
|
rateLimiter: RateLimiter,
|
|
231
|
-
token: ITokenResponse,
|
|
232
238
|
fetchToken: TokenFetcher,
|
|
233
239
|
getAuthorizationHeader: AuthorizationHeaderGetter,
|
|
234
240
|
useRestLess: boolean,
|
|
@@ -238,7 +244,6 @@ export class RouterliciousStorageRestWrapper extends RouterliciousRestWrapper {
|
|
|
238
244
|
super(
|
|
239
245
|
logger,
|
|
240
246
|
rateLimiter,
|
|
241
|
-
token,
|
|
242
247
|
fetchToken,
|
|
243
248
|
getAuthorizationHeader,
|
|
244
249
|
useRestLess,
|
|
@@ -290,12 +295,9 @@ export class RouterliciousStorageRestWrapper extends RouterliciousRestWrapper {
|
|
|
290
295
|
return getAuthorizationTokenFromCredentials(credentials);
|
|
291
296
|
};
|
|
292
297
|
|
|
293
|
-
const storagetoken = await fetchStorageToken();
|
|
294
|
-
|
|
295
298
|
const restWrapper = new RouterliciousStorageRestWrapper(
|
|
296
299
|
logger,
|
|
297
300
|
rateLimiter,
|
|
298
|
-
storagetoken,
|
|
299
301
|
fetchStorageToken,
|
|
300
302
|
getAuthorizationHeader,
|
|
301
303
|
useRestLess,
|
|
@@ -311,7 +313,6 @@ export class RouterliciousOrdererRestWrapper extends RouterliciousRestWrapper {
|
|
|
311
313
|
private constructor(
|
|
312
314
|
logger: ITelemetryLogger,
|
|
313
315
|
rateLimiter: RateLimiter,
|
|
314
|
-
token: ITokenResponse,
|
|
315
316
|
fetchToken: TokenFetcher,
|
|
316
317
|
getAuthorizationHeader: AuthorizationHeaderGetter,
|
|
317
318
|
useRestLess: boolean,
|
|
@@ -321,7 +322,6 @@ export class RouterliciousOrdererRestWrapper extends RouterliciousRestWrapper {
|
|
|
321
322
|
super(
|
|
322
323
|
logger,
|
|
323
324
|
rateLimiter,
|
|
324
|
-
token,
|
|
325
325
|
fetchToken,
|
|
326
326
|
getAuthorizationHeader,
|
|
327
327
|
useRestLess,
|
|
@@ -364,12 +364,9 @@ export class RouterliciousOrdererRestWrapper extends RouterliciousRestWrapper {
|
|
|
364
364
|
);
|
|
365
365
|
};
|
|
366
366
|
|
|
367
|
-
const newtoken = await fetchOrdererToken();
|
|
368
|
-
|
|
369
367
|
const restWrapper = new RouterliciousOrdererRestWrapper(
|
|
370
368
|
logger,
|
|
371
369
|
rateLimiter,
|
|
372
|
-
newtoken,
|
|
373
370
|
fetchOrdererToken,
|
|
374
371
|
getAuthorizationHeader,
|
|
375
372
|
useRestLess,
|
package/src/treeUtils.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
ISummaryTree,
|
|
11
11
|
SummaryObject,
|
|
12
12
|
} from "@fluidframework/protocol-definitions";
|
|
13
|
-
import { INormalizedWholeSummary } from "
|
|
13
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Summary tree assembler props
|
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
stringToBuffer,
|
|
11
11
|
Uint8ArrayToString,
|
|
12
12
|
} from "@fluidframework/common-utils";
|
|
13
|
-
import { getW3CData } from "@fluidframework/driver-base";
|
|
13
|
+
import { getW3CData, promiseRaceWithWinner } from "@fluidframework/driver-base";
|
|
14
14
|
import {
|
|
15
15
|
IDocumentStorageService,
|
|
16
16
|
ISummaryContext,
|
|
@@ -23,14 +23,9 @@ import {
|
|
|
23
23
|
ISummaryTree,
|
|
24
24
|
IVersion,
|
|
25
25
|
} from "@fluidframework/protocol-definitions";
|
|
26
|
-
import {
|
|
27
|
-
convertWholeFlatSummaryToSnapshotTreeAndBlobs,
|
|
28
|
-
INormalizedWholeSummary,
|
|
29
|
-
IWholeFlatSummary,
|
|
30
|
-
} from "@fluidframework/server-services-client";
|
|
26
|
+
import { IWholeFlatSummary } from "@fluidframework/server-services-client";
|
|
31
27
|
import { PerformanceEvent } from "@fluidframework/telemetry-utils";
|
|
32
28
|
import { ICache, InMemoryCache } from "./cache";
|
|
33
|
-
import { ISnapshotTreeVersion } from "./definitions";
|
|
34
29
|
import { IRouterliciousDriverPolicies } from "./policies";
|
|
35
30
|
import {
|
|
36
31
|
convertSnapshotAndBlobsToSummaryTree,
|
|
@@ -41,6 +36,8 @@ import { GitManager } from "./gitManager";
|
|
|
41
36
|
import { WholeSummaryUploadManager } from "./wholeSummaryUploadManager";
|
|
42
37
|
import { ISummaryUploadManager } from "./storageContracts";
|
|
43
38
|
import { IR11sResponse } from "./restWrapper";
|
|
39
|
+
import { INormalizedWholeSummary } from "./contracts";
|
|
40
|
+
import { convertWholeFlatSummaryToSnapshotTreeAndBlobs } from "./r11sSnapshotParser";
|
|
44
41
|
|
|
45
42
|
const latestSnapshotId: string = "latest";
|
|
46
43
|
|
|
@@ -63,7 +60,7 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
|
|
|
63
60
|
public readonly policies: IDocumentStorageServicePolicies,
|
|
64
61
|
private readonly driverPolicies?: IRouterliciousDriverPolicies,
|
|
65
62
|
private readonly blobCache: ICache<ArrayBufferLike> = new InMemoryCache(),
|
|
66
|
-
private readonly snapshotTreeCache: ICache<
|
|
63
|
+
private readonly snapshotTreeCache: ICache<INormalizedWholeSummary> = new InMemoryCache(),
|
|
67
64
|
private readonly noCacheGitManager?: GitManager,
|
|
68
65
|
private readonly getStorageManager: (
|
|
69
66
|
disableCache?: boolean,
|
|
@@ -86,14 +83,60 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
|
|
|
86
83
|
// If this is the first versions call for the document, we know we will want the latest summary.
|
|
87
84
|
// Fetch latest summary, cache it, and return its id.
|
|
88
85
|
if (this.firstVersionsCall && count === 1) {
|
|
86
|
+
const normalizedSnapshotContents = await PerformanceEvent.timedExecAsync(
|
|
87
|
+
this.logger,
|
|
88
|
+
{
|
|
89
|
+
eventName: "ObtainSnapshot",
|
|
90
|
+
versionId: versionId ?? undefined,
|
|
91
|
+
count,
|
|
92
|
+
enableDiscovery: this.driverPolicies?.enableDiscovery,
|
|
93
|
+
},
|
|
94
|
+
async (event) => {
|
|
95
|
+
let method: string;
|
|
96
|
+
const cachedSnapshotP = this.snapshotTreeCache.get(
|
|
97
|
+
this.getCacheKey(latestSnapshotId),
|
|
98
|
+
);
|
|
99
|
+
|
|
100
|
+
const networkSnapshotP = !this.driverPolicies?.enableDiscovery
|
|
101
|
+
? this.fetchSnapshotTree(latestSnapshotId, false, "getVersions")
|
|
102
|
+
: this.fetchSnapshotTree(latestSnapshotId, true, "getVersions");
|
|
103
|
+
|
|
104
|
+
const promiseRaceWinner = await promiseRaceWithWinner([
|
|
105
|
+
cachedSnapshotP.catch(() => undefined),
|
|
106
|
+
networkSnapshotP.catch(() => undefined),
|
|
107
|
+
]);
|
|
108
|
+
|
|
109
|
+
let retrievedSnapshot = promiseRaceWinner.value;
|
|
110
|
+
method = promiseRaceWinner.index === 0 ? "cache" : "network";
|
|
111
|
+
|
|
112
|
+
if (retrievedSnapshot === undefined) {
|
|
113
|
+
// if network failed -> wait for cache ( then return network failure)
|
|
114
|
+
// If cache returned empty or failed -> wait for network (success of failure)
|
|
115
|
+
if (promiseRaceWinner.index === 1) {
|
|
116
|
+
retrievedSnapshot = await cachedSnapshotP;
|
|
117
|
+
method = "cache";
|
|
118
|
+
}
|
|
119
|
+
if (retrievedSnapshot === undefined) {
|
|
120
|
+
retrievedSnapshot = await networkSnapshotP;
|
|
121
|
+
method = "network";
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
event.end({
|
|
125
|
+
method,
|
|
126
|
+
});
|
|
127
|
+
return retrievedSnapshot;
|
|
128
|
+
},
|
|
129
|
+
);
|
|
130
|
+
|
|
131
|
+
const _id = await this.initializeFromSnapshot(
|
|
132
|
+
normalizedSnapshotContents,
|
|
133
|
+
latestSnapshotId,
|
|
134
|
+
);
|
|
89
135
|
this.firstVersionsCall = false;
|
|
90
|
-
const { id: _id, snapshotTree } = !this.driverPolicies?.enableDiscovery
|
|
91
|
-
? await this.fetchAndCacheSnapshotTree(latestSnapshotId, false)
|
|
92
|
-
: await this.fetchAndCacheSnapshotTree(latestSnapshotId, true);
|
|
93
136
|
return [
|
|
94
137
|
{
|
|
95
138
|
id: _id,
|
|
96
|
-
treeId: snapshotTree.id!,
|
|
139
|
+
treeId: normalizedSnapshotContents.snapshotTree.id!,
|
|
97
140
|
},
|
|
98
141
|
];
|
|
99
142
|
}
|
|
@@ -130,7 +173,14 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
|
|
|
130
173
|
requestVersion = versions[0];
|
|
131
174
|
}
|
|
132
175
|
|
|
133
|
-
|
|
176
|
+
const normalizedWholeSnapshot = await this.snapshotTreeCache.get(
|
|
177
|
+
this.getCacheKey(requestVersion.id),
|
|
178
|
+
);
|
|
179
|
+
if (normalizedWholeSnapshot !== undefined) {
|
|
180
|
+
return normalizedWholeSnapshot.snapshotTree;
|
|
181
|
+
}
|
|
182
|
+
return (await this.fetchSnapshotTree(requestVersion.id, undefined, "getSnapshotTree"))
|
|
183
|
+
.snapshotTree;
|
|
134
184
|
}
|
|
135
185
|
|
|
136
186
|
public async readBlob(blobId: string): Promise<ArrayBufferLike> {
|
|
@@ -230,25 +280,17 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
|
|
|
230
280
|
);
|
|
231
281
|
}
|
|
232
282
|
|
|
233
|
-
private async
|
|
283
|
+
private async fetchSnapshotTree(
|
|
234
284
|
versionId: string,
|
|
235
285
|
disableCache?: boolean,
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
this.getCacheKey(versionId),
|
|
239
|
-
);
|
|
240
|
-
if (cachedSnapshotTreeVersion !== undefined) {
|
|
241
|
-
return {
|
|
242
|
-
id: cachedSnapshotTreeVersion.id,
|
|
243
|
-
snapshotTree: cachedSnapshotTreeVersion.snapshotTree,
|
|
244
|
-
};
|
|
245
|
-
}
|
|
246
|
-
|
|
286
|
+
scenarioName?: string,
|
|
287
|
+
): Promise<INormalizedWholeSummary> {
|
|
247
288
|
const normalizedWholeSummary = await PerformanceEvent.timedExecAsync(
|
|
248
289
|
this.logger,
|
|
249
290
|
{
|
|
250
291
|
eventName: "getWholeFlatSummary",
|
|
251
292
|
treeId: versionId,
|
|
293
|
+
scenarioName,
|
|
252
294
|
},
|
|
253
295
|
async (event) => {
|
|
254
296
|
const manager = await this.getStorageManager(disableCache);
|
|
@@ -275,33 +317,32 @@ export class WholeSummaryDocumentStorageService implements IDocumentStorageServi
|
|
|
275
317
|
},
|
|
276
318
|
);
|
|
277
319
|
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
0x275 /* "Root tree should contain the id" */,
|
|
281
|
-
);
|
|
282
|
-
const wholeFlatSummaryId: string = normalizedWholeSummary.snapshotTree.id;
|
|
283
|
-
const snapshotTreeVersion = {
|
|
284
|
-
id: wholeFlatSummaryId,
|
|
285
|
-
snapshotTree: normalizedWholeSummary.snapshotTree,
|
|
286
|
-
};
|
|
320
|
+
return normalizedWholeSummary;
|
|
321
|
+
}
|
|
287
322
|
|
|
323
|
+
private async initializeFromSnapshot(
|
|
324
|
+
normalizedWholeSummary: INormalizedWholeSummary,
|
|
325
|
+
versionId: string | null,
|
|
326
|
+
): Promise<string> {
|
|
327
|
+
const snapshotId = normalizedWholeSummary.id;
|
|
328
|
+
assert(snapshotId !== undefined, 0x275 /* "Root tree should contain the id" */);
|
|
288
329
|
const cachePs: Promise<any>[] = [
|
|
289
|
-
this.snapshotTreeCache.put(this.getCacheKey(
|
|
330
|
+
this.snapshotTreeCache.put(this.getCacheKey(snapshotId), normalizedWholeSummary),
|
|
290
331
|
this.initBlobCache(normalizedWholeSummary.blobs),
|
|
291
332
|
];
|
|
292
|
-
if (
|
|
333
|
+
if (snapshotId !== versionId && versionId !== null) {
|
|
293
334
|
// versionId could be "latest". When summarizer checks cache for "latest", we want it to be available.
|
|
294
335
|
// TODO: For in-memory cache, <latest,snapshotTree> will be a shared pointer with <snapshotId,snapshotTree>,
|
|
295
336
|
// However, for something like Redis, this will cache the same value twice. Alternatively, could we simply
|
|
296
337
|
// cache with versionId?
|
|
297
338
|
cachePs.push(
|
|
298
|
-
this.snapshotTreeCache.put(this.getCacheKey(versionId),
|
|
339
|
+
this.snapshotTreeCache.put(this.getCacheKey(versionId), normalizedWholeSummary),
|
|
299
340
|
);
|
|
300
341
|
}
|
|
301
342
|
|
|
302
343
|
await Promise.all(cachePs);
|
|
303
344
|
|
|
304
|
-
return
|
|
345
|
+
return snapshotId;
|
|
305
346
|
}
|
|
306
347
|
|
|
307
348
|
private async initBlobCache(blobs: Map<string, ArrayBuffer>): Promise<void> {
|