@fluidframework/routerlicious-driver 2.0.0-internal.5.3.2 → 2.0.0-internal.5.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/dist/contracts.d.ts +28 -1
- package/dist/contracts.d.ts.map +1 -1
- package/dist/contracts.js.map +1 -1
- package/dist/deltaStorageService.d.ts.map +1 -1
- package/dist/deltaStorageService.js +8 -1
- package/dist/deltaStorageService.js.map +1 -1
- package/dist/documentService.d.ts +2 -2
- package/dist/documentService.d.ts.map +1 -1
- package/dist/documentService.js.map +1 -1
- package/dist/documentServiceFactory.d.ts.map +1 -1
- package/dist/documentServiceFactory.js +7 -3
- package/dist/documentServiceFactory.js.map +1 -1
- package/dist/documentStorageService.d.ts +2 -2
- package/dist/documentStorageService.d.ts.map +1 -1
- package/dist/documentStorageService.js.map +1 -1
- package/dist/errorUtils.d.ts +5 -6
- package/dist/errorUtils.d.ts.map +1 -1
- package/dist/errorUtils.js +13 -15
- package/dist/errorUtils.js.map +1 -1
- package/dist/gitManager.d.ts +3 -2
- package/dist/gitManager.d.ts.map +1 -1
- package/dist/gitManager.js +2 -2
- package/dist/gitManager.js.map +1 -1
- package/dist/historian.d.ts +3 -2
- package/dist/historian.d.ts.map +1 -1
- package/dist/historian.js +1 -1
- package/dist/historian.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/r11sSnapshotParser.d.ts +4 -5
- package/dist/r11sSnapshotParser.d.ts.map +1 -1
- package/dist/r11sSnapshotParser.js +11 -11
- package/dist/r11sSnapshotParser.js.map +1 -1
- package/dist/restWrapper.d.ts.map +1 -1
- package/dist/restWrapper.js +20 -11
- package/dist/restWrapper.js.map +1 -1
- package/dist/retriableGitManager.d.ts +3 -2
- package/dist/retriableGitManager.d.ts.map +1 -1
- package/dist/retriableGitManager.js +2 -2
- package/dist/retriableGitManager.js.map +1 -1
- package/dist/storageContracts.d.ts +4 -3
- package/dist/storageContracts.d.ts.map +1 -1
- package/dist/storageContracts.js.map +1 -1
- package/dist/treeUtils.d.ts +2 -2
- package/dist/treeUtils.d.ts.map +1 -1
- package/dist/treeUtils.js.map +1 -1
- package/dist/wholeSummaryDocumentStorageService.d.ts +2 -2
- package/dist/wholeSummaryDocumentStorageService.d.ts.map +1 -1
- package/dist/wholeSummaryDocumentStorageService.js +5 -5
- package/dist/wholeSummaryDocumentStorageService.js.map +1 -1
- package/lib/contracts.d.ts +28 -1
- package/lib/contracts.d.ts.map +1 -1
- package/lib/contracts.js.map +1 -1
- package/lib/deltaStorageService.d.ts.map +1 -1
- package/lib/deltaStorageService.js +8 -1
- package/lib/deltaStorageService.js.map +1 -1
- package/lib/documentService.d.ts +2 -2
- package/lib/documentService.d.ts.map +1 -1
- package/lib/documentService.js.map +1 -1
- package/lib/documentServiceFactory.d.ts.map +1 -1
- package/lib/documentServiceFactory.js +8 -4
- package/lib/documentServiceFactory.js.map +1 -1
- package/lib/documentStorageService.d.ts +2 -2
- package/lib/documentStorageService.d.ts.map +1 -1
- package/lib/documentStorageService.js.map +1 -1
- package/lib/errorUtils.d.ts +5 -6
- package/lib/errorUtils.d.ts.map +1 -1
- package/lib/errorUtils.js +13 -15
- package/lib/errorUtils.js.map +1 -1
- package/lib/gitManager.d.ts +3 -2
- package/lib/gitManager.d.ts.map +1 -1
- package/lib/gitManager.js +2 -2
- package/lib/gitManager.js.map +1 -1
- package/lib/historian.d.ts +3 -2
- package/lib/historian.d.ts.map +1 -1
- package/lib/historian.js +1 -1
- package/lib/historian.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/r11sSnapshotParser.d.ts +4 -5
- package/lib/r11sSnapshotParser.d.ts.map +1 -1
- package/lib/r11sSnapshotParser.js +9 -9
- package/lib/r11sSnapshotParser.js.map +1 -1
- package/lib/restWrapper.d.ts.map +1 -1
- package/lib/restWrapper.js +23 -14
- package/lib/restWrapper.js.map +1 -1
- package/lib/retriableGitManager.d.ts +3 -2
- package/lib/retriableGitManager.d.ts.map +1 -1
- package/lib/retriableGitManager.js +2 -2
- package/lib/retriableGitManager.js.map +1 -1
- package/lib/storageContracts.d.ts +4 -3
- package/lib/storageContracts.d.ts.map +1 -1
- package/lib/storageContracts.js.map +1 -1
- package/lib/treeUtils.d.ts +2 -2
- package/lib/treeUtils.d.ts.map +1 -1
- package/lib/treeUtils.js.map +1 -1
- package/lib/wholeSummaryDocumentStorageService.d.ts +2 -2
- package/lib/wholeSummaryDocumentStorageService.d.ts.map +1 -1
- package/lib/wholeSummaryDocumentStorageService.js +6 -6
- package/lib/wholeSummaryDocumentStorageService.js.map +1 -1
- package/package.json +9 -9
- package/src/contracts.ts +45 -1
- package/src/deltaStorageService.ts +8 -1
- package/src/documentService.ts +2 -2
- package/src/documentServiceFactory.ts +11 -7
- package/src/documentStorageService.ts +3 -3
- package/src/errorUtils.ts +23 -30
- package/src/gitManager.ts +3 -3
- package/src/historian.ts +3 -3
- package/src/packageVersion.ts +1 -1
- package/src/r11sSnapshotParser.ts +13 -14
- package/src/restWrapper.ts +29 -17
- package/src/retriableGitManager.ts +3 -3
- package/src/storageContracts.ts +3 -3
- package/src/treeUtils.ts +2 -2
- package/src/wholeSummaryDocumentStorageService.ts +12 -13
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fluidframework/routerlicious-driver",
|
|
3
|
-
"version": "2.0.0-internal.5.
|
|
3
|
+
"version": "2.0.0-internal.5.4.0",
|
|
4
4
|
"description": "Socket.IO + Git implementation of Fluid service API",
|
|
5
5
|
"homepage": "https://fluidframework.com",
|
|
6
6
|
"repository": {
|
|
@@ -37,15 +37,15 @@
|
|
|
37
37
|
"dependencies": {
|
|
38
38
|
"@fluidframework/common-definitions": "^0.20.1",
|
|
39
39
|
"@fluidframework/common-utils": "^1.1.1",
|
|
40
|
-
"@fluidframework/core-interfaces": ">=2.0.0-internal.5.
|
|
41
|
-
"@fluidframework/driver-base": ">=2.0.0-internal.5.
|
|
42
|
-
"@fluidframework/driver-definitions": ">=2.0.0-internal.5.
|
|
43
|
-
"@fluidframework/driver-utils": ">=2.0.0-internal.5.
|
|
40
|
+
"@fluidframework/core-interfaces": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
41
|
+
"@fluidframework/driver-base": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
42
|
+
"@fluidframework/driver-definitions": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
43
|
+
"@fluidframework/driver-utils": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
44
44
|
"@fluidframework/gitresources": "^0.1039.1000",
|
|
45
45
|
"@fluidframework/protocol-base": "^0.1039.1000",
|
|
46
46
|
"@fluidframework/protocol-definitions": "^1.1.0",
|
|
47
47
|
"@fluidframework/server-services-client": "^0.1039.1000",
|
|
48
|
-
"@fluidframework/telemetry-utils": ">=2.0.0-internal.5.
|
|
48
|
+
"@fluidframework/telemetry-utils": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
49
49
|
"cross-fetch": "^3.1.5",
|
|
50
50
|
"json-stringify-safe": "5.0.1",
|
|
51
51
|
"socket.io-client": "^4.6.1",
|
|
@@ -57,12 +57,12 @@
|
|
|
57
57
|
"@fluidframework/build-common": "^1.2.0",
|
|
58
58
|
"@fluidframework/build-tools": "^0.21.0",
|
|
59
59
|
"@fluidframework/eslint-config-fluid": "^2.0.0",
|
|
60
|
-
"@fluidframework/mocha-test-setup": ">=2.0.0-internal.5.
|
|
60
|
+
"@fluidframework/mocha-test-setup": ">=2.0.0-internal.5.4.0 <2.0.0-internal.5.5.0",
|
|
61
61
|
"@fluidframework/routerlicious-driver-previous": "npm:@fluidframework/routerlicious-driver@2.0.0-internal.5.2.0",
|
|
62
62
|
"@microsoft/api-extractor": "^7.34.4",
|
|
63
63
|
"@types/mocha": "^9.1.1",
|
|
64
64
|
"@types/nock": "^9.3.0",
|
|
65
|
-
"@types/node": "^
|
|
65
|
+
"@types/node": "^16.18.38",
|
|
66
66
|
"@types/sinon": "^7.0.13",
|
|
67
67
|
"@types/url-parse": "1.4.4",
|
|
68
68
|
"@types/uuid": "^8.3.0",
|
|
@@ -94,7 +94,7 @@
|
|
|
94
94
|
"build:genver": "gen-version",
|
|
95
95
|
"build:test": "tsc --project ./src/test/tsconfig.json",
|
|
96
96
|
"ci:build:docs": "api-extractor run --typescript-compiler-folder ../../../node_modules/typescript && copyfiles -u 1 ./_api-extractor-temp/* ../../../_api-extractor-temp/",
|
|
97
|
-
"clean": "rimraf dist lib *.tsbuildinfo *.build.log",
|
|
97
|
+
"clean": "rimraf --glob \"dist\" \"lib\" \"*.tsbuildinfo\" \"*.build.log\"",
|
|
98
98
|
"eslint": "eslint --format stylish src",
|
|
99
99
|
"eslint:fix": "eslint --format stylish src --fix --fix-type problem,suggestion,layout",
|
|
100
100
|
"format": "npm run prettier:fix",
|
package/src/contracts.ts
CHANGED
|
@@ -5,10 +5,54 @@
|
|
|
5
5
|
|
|
6
6
|
import { ISnapshotTree } from "@fluidframework/protocol-definitions";
|
|
7
7
|
|
|
8
|
+
/*
|
|
9
|
+
*
|
|
10
|
+
* Whole Snapshot Download Data Structures
|
|
11
|
+
*
|
|
12
|
+
*/
|
|
13
|
+
|
|
14
|
+
export interface IWholeFlatSnapshotTreeEntryTree {
|
|
15
|
+
path: string;
|
|
16
|
+
type: "tree";
|
|
17
|
+
// Indicates that this tree entry is unreferenced. If this is not present, the tree entry is considered referenced.
|
|
18
|
+
unreferenced?: true;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export interface IWholeFlatSnapshotTreeEntryBlob {
|
|
22
|
+
id: string;
|
|
23
|
+
path: string;
|
|
24
|
+
type: "blob";
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
export type IWholeFlatSnapshotTreeEntry =
|
|
28
|
+
| IWholeFlatSnapshotTreeEntryTree
|
|
29
|
+
| IWholeFlatSnapshotTreeEntryBlob;
|
|
30
|
+
|
|
31
|
+
export interface IWholeFlatSnapshotTree {
|
|
32
|
+
entries: IWholeFlatSnapshotTreeEntry[];
|
|
33
|
+
id: string;
|
|
34
|
+
sequenceNumber: number;
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
export interface IWholeFlatSnapshotBlob {
|
|
38
|
+
content: string;
|
|
39
|
+
encoding: "base64" | "utf-8";
|
|
40
|
+
id: string;
|
|
41
|
+
size: number;
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export interface IWholeFlatSnapshot {
|
|
45
|
+
// The same as the id of the first snapshot tree in the trees array.
|
|
46
|
+
id: string;
|
|
47
|
+
// Receive an array of snapshot trees for future-proofing, however, always length 1 for now.
|
|
48
|
+
trees: IWholeFlatSnapshotTree[];
|
|
49
|
+
blobs?: IWholeFlatSnapshotBlob[];
|
|
50
|
+
}
|
|
51
|
+
|
|
8
52
|
/**
|
|
9
53
|
* Normalized Whole Summary with decoded blobs and unflattened snapshot tree.
|
|
10
54
|
*/
|
|
11
|
-
export interface
|
|
55
|
+
export interface INormalizedWholeSnapshot {
|
|
12
56
|
blobs: Map<string, ArrayBuffer>;
|
|
13
57
|
snapshotTree: ISnapshotTree;
|
|
14
58
|
sequenceNumber: number | undefined;
|
|
@@ -22,7 +22,14 @@ import { ITelemetryLoggerExt, PerformanceEvent } from "@fluidframework/telemetry
|
|
|
22
22
|
import { DocumentStorageService } from "./documentStorageService";
|
|
23
23
|
import { RestWrapper } from "./restWrapperBase";
|
|
24
24
|
|
|
25
|
-
|
|
25
|
+
/**
|
|
26
|
+
* Maximum number of ops we can fetch at a time. This should be kept at 2k, as
|
|
27
|
+
* server determines whether to try to fallback to long-term storage if the ops range requested is larger than
|
|
28
|
+
* what they have locally available in short-term storage. So if we request 2k ops, they know it is not a
|
|
29
|
+
* specific request and they don't fall to long term storage which takes time.
|
|
30
|
+
* Please coordinate to AFR team if this value need to be changed.
|
|
31
|
+
*/
|
|
32
|
+
const MaxBatchDeltas = 2000;
|
|
26
33
|
|
|
27
34
|
/**
|
|
28
35
|
* Storage service limited to only being able to fetch documents for a specific document
|
package/src/documentService.ts
CHANGED
|
@@ -27,7 +27,7 @@ import { pkgVersion as driverVersion } from "./packageVersion";
|
|
|
27
27
|
import { GitManager } from "./gitManager";
|
|
28
28
|
import { Historian } from "./historian";
|
|
29
29
|
import { RestWrapper } from "./restWrapperBase";
|
|
30
|
-
import {
|
|
30
|
+
import { INormalizedWholeSnapshot } from "./contracts";
|
|
31
31
|
|
|
32
32
|
/**
|
|
33
33
|
* Amount of time between discoveries within which we don't need to rediscover on re-connect.
|
|
@@ -67,7 +67,7 @@ export class DocumentService implements api.IDocumentService {
|
|
|
67
67
|
private readonly documentStorageServicePolicies: api.IDocumentStorageServicePolicies,
|
|
68
68
|
private readonly driverPolicies: IRouterliciousDriverPolicies,
|
|
69
69
|
private readonly blobCache: ICache<ArrayBufferLike>,
|
|
70
|
-
private readonly wholeSnapshotTreeCache: ICache<
|
|
70
|
+
private readonly wholeSnapshotTreeCache: ICache<INormalizedWholeSnapshot>,
|
|
71
71
|
private readonly shreddedSummaryTreeCache: ICache<ISnapshotTreeVersion>,
|
|
72
72
|
private readonly discoverFluidResolvedUrl: () => Promise<api.IResolvedUrl>,
|
|
73
73
|
private storageRestWrapper: RouterliciousStorageRestWrapper,
|
|
@@ -21,7 +21,7 @@ import {
|
|
|
21
21
|
isCombinedAppAndProtocolSummary,
|
|
22
22
|
RateLimiter,
|
|
23
23
|
} from "@fluidframework/driver-utils";
|
|
24
|
-
import {
|
|
24
|
+
import { createChildLogger, PerformanceEvent } from "@fluidframework/telemetry-utils";
|
|
25
25
|
import { ISession } from "@fluidframework/server-services-client";
|
|
26
26
|
import { DocumentService } from "./documentService";
|
|
27
27
|
import { IRouterliciousDriverPolicies } from "./policies";
|
|
@@ -37,7 +37,7 @@ import { parseFluidUrl, replaceDocumentIdInPath, getDiscoveredFluidResolvedUrl }
|
|
|
37
37
|
import { ICache, InMemoryCache, NullCache } from "./cache";
|
|
38
38
|
import { pkgVersion as driverVersion } from "./packageVersion";
|
|
39
39
|
import { ISnapshotTreeVersion } from "./definitions";
|
|
40
|
-
import {
|
|
40
|
+
import { INormalizedWholeSnapshot } from "./contracts";
|
|
41
41
|
|
|
42
42
|
const maximumSnapshotCacheDurationMs: FiveDaysMs = 432_000_000; // 5 days in ms
|
|
43
43
|
|
|
@@ -60,7 +60,7 @@ const defaultRouterliciousDriverPolicies: IRouterliciousDriverPolicies = {
|
|
|
60
60
|
export class RouterliciousDocumentServiceFactory implements IDocumentServiceFactory {
|
|
61
61
|
private readonly driverPolicies: IRouterliciousDriverPolicies;
|
|
62
62
|
private readonly blobCache: ICache<ArrayBufferLike>;
|
|
63
|
-
private readonly wholeSnapshotTreeCache: ICache<
|
|
63
|
+
private readonly wholeSnapshotTreeCache: ICache<INormalizedWholeSnapshot> = new NullCache();
|
|
64
64
|
private readonly shreddedSummaryTreeCache: ICache<ISnapshotTreeVersion> = new NullCache();
|
|
65
65
|
|
|
66
66
|
constructor(
|
|
@@ -77,7 +77,7 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
|
|
|
77
77
|
this.blobCache = new InMemoryCache<ArrayBufferLike>();
|
|
78
78
|
if (this.driverPolicies.enableInternalSummaryCaching) {
|
|
79
79
|
if (this.driverPolicies.enableWholeSummaryUpload) {
|
|
80
|
-
this.wholeSnapshotTreeCache = new InMemoryCache<
|
|
80
|
+
this.wholeSnapshotTreeCache = new InMemoryCache<INormalizedWholeSnapshot>(
|
|
81
81
|
snapshotCacheExpiryMs,
|
|
82
82
|
);
|
|
83
83
|
} else {
|
|
@@ -119,7 +119,7 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
|
|
|
119
119
|
const documentAttributes = getDocAttributesFromProtocolSummary(protocolSummary);
|
|
120
120
|
const quorumValues = getQuorumValuesFromProtocolSummary(protocolSummary);
|
|
121
121
|
|
|
122
|
-
const logger2 =
|
|
122
|
+
const logger2 = createChildLogger({ logger, namespace: "RouterliciousDriver" });
|
|
123
123
|
const ordererTokenFetcher = toInstrumentedR11sOrdererTokenFetcher(
|
|
124
124
|
tenantId,
|
|
125
125
|
undefined /* documentId */,
|
|
@@ -247,8 +247,12 @@ export class RouterliciousDocumentServiceFactory implements IDocumentServiceFact
|
|
|
247
247
|
`Couldn't parse documentId and/or tenantId. [documentId:${documentId}][tenantId:${tenantId}]`,
|
|
248
248
|
);
|
|
249
249
|
}
|
|
250
|
-
const logger2 =
|
|
251
|
-
|
|
250
|
+
const logger2 = createChildLogger({
|
|
251
|
+
logger,
|
|
252
|
+
namespace: "RouterliciousDriver",
|
|
253
|
+
properties: {
|
|
254
|
+
all: { driverVersion },
|
|
255
|
+
},
|
|
252
256
|
});
|
|
253
257
|
|
|
254
258
|
const ordererTokenFetcher = toInstrumentedR11sOrdererTokenFetcher(
|
|
@@ -20,7 +20,7 @@ import { WholeSummaryDocumentStorageService } from "./wholeSummaryDocumentStorag
|
|
|
20
20
|
import { ShreddedSummaryDocumentStorageService } from "./shreddedSummaryDocumentStorageService";
|
|
21
21
|
import { GitManager } from "./gitManager";
|
|
22
22
|
import { ISnapshotTreeVersion } from "./definitions";
|
|
23
|
-
import {
|
|
23
|
+
import { INormalizedWholeSnapshot } from "./contracts";
|
|
24
24
|
|
|
25
25
|
export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
26
26
|
private _logTailSha: string | undefined = undefined;
|
|
@@ -36,7 +36,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
36
36
|
policies: IDocumentStorageServicePolicies,
|
|
37
37
|
driverPolicies?: IRouterliciousDriverPolicies,
|
|
38
38
|
blobCache?: ICache<ArrayBufferLike>,
|
|
39
|
-
snapshotTreeCache?: ICache<
|
|
39
|
+
snapshotTreeCache?: ICache<INormalizedWholeSnapshot>,
|
|
40
40
|
shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
|
|
41
41
|
noCacheGitManager?: GitManager,
|
|
42
42
|
getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
|
|
@@ -80,7 +80,7 @@ export class DocumentStorageService extends DocumentStorageServiceProxy {
|
|
|
80
80
|
policies: IDocumentStorageServicePolicies,
|
|
81
81
|
driverPolicies?: IRouterliciousDriverPolicies,
|
|
82
82
|
blobCache?: ICache<ArrayBufferLike>,
|
|
83
|
-
snapshotTreeCache?: ICache<
|
|
83
|
+
snapshotTreeCache?: ICache<INormalizedWholeSnapshot>,
|
|
84
84
|
shreddedSummaryTreeCache?: ICache<ISnapshotTreeVersion>,
|
|
85
85
|
public noCacheGitManager?: GitManager,
|
|
86
86
|
getStorageManager?: (disableCache?: boolean) => Promise<GitManager>,
|
package/src/errorUtils.ts
CHANGED
|
@@ -3,13 +3,14 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { DriverError } from "@fluidframework/driver-definitions";
|
|
6
|
+
import { DriverError, IDriverErrorBase } from "@fluidframework/driver-definitions";
|
|
7
7
|
import {
|
|
8
8
|
NonRetryableError,
|
|
9
9
|
GenericNetworkError,
|
|
10
10
|
createGenericNetworkError,
|
|
11
11
|
AuthorizationError,
|
|
12
12
|
} from "@fluidframework/driver-utils";
|
|
13
|
+
import { IFluidErrorBase } from "@fluidframework/telemetry-utils";
|
|
13
14
|
import { pkgVersion as driverVersion } from "./packageVersion";
|
|
14
15
|
|
|
15
16
|
/**
|
|
@@ -55,61 +56,53 @@ export interface IR11sSocketError {
|
|
|
55
56
|
retryAfterMs?: number;
|
|
56
57
|
}
|
|
57
58
|
|
|
58
|
-
export interface IR11sError {
|
|
59
|
+
export interface IR11sError extends Omit<IDriverErrorBase, "errorType"> {
|
|
59
60
|
readonly errorType: RouterliciousErrorType;
|
|
60
|
-
readonly message: string;
|
|
61
|
-
canRetry: boolean;
|
|
62
61
|
}
|
|
63
62
|
|
|
64
63
|
export type R11sError = DriverError | IR11sError;
|
|
65
64
|
|
|
66
65
|
export function createR11sNetworkError(
|
|
67
66
|
errorMessage: string,
|
|
68
|
-
statusCode
|
|
67
|
+
statusCode: number,
|
|
69
68
|
retryAfterMs?: number,
|
|
70
|
-
): R11sError {
|
|
69
|
+
): IFluidErrorBase & R11sError {
|
|
70
|
+
let error: IFluidErrorBase & R11sError;
|
|
71
71
|
const props = { statusCode, driverVersion };
|
|
72
72
|
switch (statusCode) {
|
|
73
|
-
case undefined:
|
|
74
|
-
// If a service is temporarily down or a browser resource limit is reached, RestWrapper will throw
|
|
75
|
-
// a network error with no status code (e.g. err:ERR_CONN_REFUSED or err:ERR_FAILED) and
|
|
76
|
-
// the error message will start with NetworkError as defined in restWrapper.ts
|
|
77
|
-
// If there exists a self-signed SSL certificates error, throw a NonRetryableError
|
|
78
|
-
// TODO: instead of relying on string matching, filter error based on the error code like we do for websocket connections
|
|
79
|
-
if (errorMessage.includes("failed, reason: self signed certificate")) {
|
|
80
|
-
return new NonRetryableError(
|
|
81
|
-
errorMessage,
|
|
82
|
-
RouterliciousErrorType.sslCertError,
|
|
83
|
-
props,
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
return new GenericNetworkError(
|
|
87
|
-
errorMessage,
|
|
88
|
-
errorMessage.startsWith("NetworkError"),
|
|
89
|
-
props,
|
|
90
|
-
);
|
|
91
73
|
case 401:
|
|
92
74
|
// The first 401 is manually retried in RouterliciousRestWrapper with a refreshed token,
|
|
93
75
|
// so we treat repeat 401s the same as 403.
|
|
94
76
|
case 403:
|
|
95
|
-
|
|
77
|
+
error = new AuthorizationError(errorMessage, undefined, undefined, props);
|
|
78
|
+
break;
|
|
96
79
|
case 404:
|
|
97
80
|
const errorType = RouterliciousErrorType.fileNotFoundOrAccessDeniedError;
|
|
98
|
-
|
|
81
|
+
error = new NonRetryableError(errorMessage, errorType, props);
|
|
82
|
+
break;
|
|
99
83
|
case 429:
|
|
100
|
-
|
|
84
|
+
error = createGenericNetworkError(
|
|
85
|
+
errorMessage,
|
|
86
|
+
{ canRetry: true, retryAfterMs },
|
|
87
|
+
props,
|
|
88
|
+
);
|
|
89
|
+
break;
|
|
101
90
|
case 500:
|
|
102
91
|
case 502:
|
|
103
|
-
|
|
92
|
+
error = new GenericNetworkError(errorMessage, true, props);
|
|
93
|
+
break;
|
|
104
94
|
default:
|
|
105
95
|
const retryInfo = { canRetry: retryAfterMs !== undefined, retryAfterMs };
|
|
106
|
-
|
|
96
|
+
error = createGenericNetworkError(errorMessage, retryInfo, props);
|
|
97
|
+
break;
|
|
107
98
|
}
|
|
99
|
+
error.addTelemetryProperties({ endpointReached: true });
|
|
100
|
+
return error;
|
|
108
101
|
}
|
|
109
102
|
|
|
110
103
|
export function throwR11sNetworkError(
|
|
111
104
|
errorMessage: string,
|
|
112
|
-
statusCode
|
|
105
|
+
statusCode: number,
|
|
113
106
|
retryAfterMs?: number,
|
|
114
107
|
): never {
|
|
115
108
|
const networkError = createR11sNetworkError(errorMessage, statusCode, retryAfterMs);
|
package/src/gitManager.ts
CHANGED
|
@@ -5,12 +5,12 @@
|
|
|
5
5
|
|
|
6
6
|
import * as resources from "@fluidframework/gitresources";
|
|
7
7
|
import {
|
|
8
|
-
IWholeFlatSummary,
|
|
9
8
|
IWholeSummaryPayload,
|
|
10
9
|
IWriteSummaryResponse,
|
|
11
10
|
} from "@fluidframework/server-services-client";
|
|
12
11
|
import { IGitManager, IHistorian } from "./storageContracts";
|
|
13
12
|
import { IR11sResponse, createR11sResponseFromContent } from "./restWrapper";
|
|
13
|
+
import { IWholeFlatSnapshot } from "./contracts";
|
|
14
14
|
|
|
15
15
|
export class GitManager implements IGitManager {
|
|
16
16
|
private readonly blobCache = new Map<string, resources.IBlob>();
|
|
@@ -110,7 +110,7 @@ export class GitManager implements IGitManager {
|
|
|
110
110
|
return this.historian.createSummary(summary, initial);
|
|
111
111
|
}
|
|
112
112
|
|
|
113
|
-
public async
|
|
114
|
-
return this.historian.
|
|
113
|
+
public async getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>> {
|
|
114
|
+
return this.historian.getSnapshot(sha);
|
|
115
115
|
}
|
|
116
116
|
}
|
package/src/historian.ts
CHANGED
|
@@ -6,13 +6,13 @@
|
|
|
6
6
|
import { fromUtf8ToBase64 } from "@fluidframework/common-utils";
|
|
7
7
|
import * as git from "@fluidframework/gitresources";
|
|
8
8
|
import {
|
|
9
|
-
IWholeFlatSummary,
|
|
10
9
|
IWholeSummaryPayload,
|
|
11
10
|
IWriteSummaryResponse,
|
|
12
11
|
} from "@fluidframework/server-services-client";
|
|
13
12
|
import { QueryStringType, RestWrapper } from "./restWrapperBase";
|
|
14
13
|
import { IR11sResponse } from "./restWrapper";
|
|
15
14
|
import { IHistorian } from "./storageContracts";
|
|
15
|
+
import { IWholeFlatSnapshot } from "./contracts";
|
|
16
16
|
|
|
17
17
|
export interface ICredentials {
|
|
18
18
|
user: string;
|
|
@@ -98,8 +98,8 @@ export class Historian implements IHistorian {
|
|
|
98
98
|
);
|
|
99
99
|
}
|
|
100
100
|
|
|
101
|
-
public async
|
|
102
|
-
return this.restWrapper.get<
|
|
101
|
+
public async getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>> {
|
|
102
|
+
return this.restWrapper.get<IWholeFlatSnapshot>(
|
|
103
103
|
`/git/summaries/${sha}`,
|
|
104
104
|
this.getQueryString(),
|
|
105
105
|
);
|
package/src/packageVersion.ts
CHANGED
|
@@ -4,9 +4,8 @@
|
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
6
|
import { ISnapshotTree } from "@fluidframework/protocol-definitions";
|
|
7
|
-
import { IWholeFlatSummary, IWholeFlatSummaryTree } from "@fluidframework/server-services-client";
|
|
8
7
|
import { stringToBuffer } from "@fluidframework/common-utils";
|
|
9
|
-
import {
|
|
8
|
+
import { INormalizedWholeSnapshot, IWholeFlatSnapshot, IWholeFlatSnapshotTree } from "./contracts";
|
|
10
9
|
|
|
11
10
|
/**
|
|
12
11
|
* Build a tree hierarchy from a flat tree.
|
|
@@ -16,7 +15,7 @@ import { INormalizedWholeSummary } from "./contracts";
|
|
|
16
15
|
* @returns the heirarchical tree
|
|
17
16
|
*/
|
|
18
17
|
function buildHierarchy(
|
|
19
|
-
flatTree:
|
|
18
|
+
flatTree: IWholeFlatSnapshotTree,
|
|
20
19
|
treePrefixToRemove: string,
|
|
21
20
|
): ISnapshotTree {
|
|
22
21
|
const lookup: { [path: string]: ISnapshotTree } = {};
|
|
@@ -54,30 +53,30 @@ function buildHierarchy(
|
|
|
54
53
|
}
|
|
55
54
|
|
|
56
55
|
/**
|
|
57
|
-
* Converts existing
|
|
56
|
+
* Converts existing IWholeFlatSnapshot to snapshot tree, blob array, and sequence number.
|
|
58
57
|
*
|
|
59
|
-
* @param
|
|
58
|
+
* @param flatSnapshot - flat snapshot
|
|
60
59
|
* @param treePrefixToRemove - tree prefix to strip. By default we are stripping ".app" prefix
|
|
61
60
|
* @returns snapshot tree, blob array, and sequence number
|
|
62
61
|
*/
|
|
63
|
-
export function
|
|
64
|
-
|
|
62
|
+
export function convertWholeFlatSnapshotToSnapshotTreeAndBlobs(
|
|
63
|
+
flatSnapshot: IWholeFlatSnapshot,
|
|
65
64
|
treePrefixToRemove: string = ".app",
|
|
66
|
-
):
|
|
65
|
+
): INormalizedWholeSnapshot {
|
|
67
66
|
const blobs = new Map<string, ArrayBuffer>();
|
|
68
|
-
if (
|
|
69
|
-
|
|
67
|
+
if (flatSnapshot.blobs) {
|
|
68
|
+
flatSnapshot.blobs.forEach((blob) => {
|
|
70
69
|
blobs.set(blob.id, stringToBuffer(blob.content, blob.encoding ?? "utf-8"));
|
|
71
70
|
});
|
|
72
71
|
}
|
|
73
|
-
const
|
|
74
|
-
const sequenceNumber =
|
|
75
|
-
const snapshotTree = buildHierarchy(
|
|
72
|
+
const flatSnapshotTree = flatSnapshot.trees?.[0];
|
|
73
|
+
const sequenceNumber = flatSnapshotTree?.sequenceNumber;
|
|
74
|
+
const snapshotTree = buildHierarchy(flatSnapshotTree, treePrefixToRemove);
|
|
76
75
|
|
|
77
76
|
return {
|
|
78
77
|
blobs,
|
|
79
78
|
snapshotTree,
|
|
80
79
|
sequenceNumber,
|
|
81
|
-
id:
|
|
80
|
+
id: flatSnapshot.id,
|
|
82
81
|
};
|
|
83
82
|
}
|
package/src/restWrapper.ts
CHANGED
|
@@ -7,19 +7,20 @@ import { ITelemetryProperties } from "@fluidframework/core-interfaces";
|
|
|
7
7
|
import {
|
|
8
8
|
ITelemetryLoggerExt,
|
|
9
9
|
PerformanceEvent,
|
|
10
|
-
|
|
10
|
+
numberFromString,
|
|
11
11
|
} from "@fluidframework/telemetry-utils";
|
|
12
12
|
import { assert, fromUtf8ToBase64, performance } from "@fluidframework/common-utils";
|
|
13
|
-
import { RateLimiter } from "@fluidframework/driver-utils";
|
|
13
|
+
import { GenericNetworkError, NonRetryableError, RateLimiter } from "@fluidframework/driver-utils";
|
|
14
14
|
import {
|
|
15
|
+
CorrelationIdHeaderName,
|
|
16
|
+
DriverVersionHeaderName,
|
|
15
17
|
getAuthorizationTokenFromCredentials,
|
|
16
18
|
RestLessClient,
|
|
17
19
|
} from "@fluidframework/server-services-client";
|
|
18
20
|
import fetch from "cross-fetch";
|
|
19
21
|
import type { AxiosRequestConfig, AxiosRequestHeaders } from "axios";
|
|
20
22
|
import safeStringify from "json-stringify-safe";
|
|
21
|
-
import {
|
|
22
|
-
import { throwR11sNetworkError } from "./errorUtils";
|
|
23
|
+
import { RouterliciousErrorType, throwR11sNetworkError } from "./errorUtils";
|
|
23
24
|
import { ITokenProvider, ITokenResponse } from "./tokens";
|
|
24
25
|
import { pkgVersion as driverVersion } from "./packageVersion";
|
|
25
26
|
import { QueryStringType, RestWrapper } from "./restWrapperBase";
|
|
@@ -84,12 +85,12 @@ export function getPropsToLogFromResponse(headers: {
|
|
|
84
85
|
// We rename headers so that otel doesn't scrub them away. Otel doesn't allow
|
|
85
86
|
// certain characters in headers including '-'
|
|
86
87
|
const headersToLog: LoggingHeader[] = [
|
|
87
|
-
{ headerName:
|
|
88
|
+
{ headerName: CorrelationIdHeaderName, logName: "requestCorrelationId" },
|
|
88
89
|
{ headerName: "content-encoding", logName: "contentEncoding" },
|
|
89
90
|
{ headerName: "content-type", logName: "contentType" },
|
|
90
91
|
];
|
|
91
92
|
const additionalProps: ITelemetryProperties = {
|
|
92
|
-
contentsize:
|
|
93
|
+
contentsize: numberFromString(headers.get("content-length")),
|
|
93
94
|
};
|
|
94
95
|
headersToLog.forEach((header) => {
|
|
95
96
|
const headerValue = headers.get(header.headerName);
|
|
@@ -136,9 +137,24 @@ export class RouterliciousRestWrapper extends RestWrapper {
|
|
|
136
137
|
const result = await fetch(...fetchRequestConfig).catch(async (error) => {
|
|
137
138
|
// Browser Fetch throws a TypeError on network error, `node-fetch` throws a FetchError
|
|
138
139
|
const isNetworkError = ["TypeError", "FetchError"].includes(error?.name);
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
140
|
+
const errorMessage = isNetworkError
|
|
141
|
+
? `NetworkError: ${error.message}`
|
|
142
|
+
: safeStringify(error);
|
|
143
|
+
// If a service is temporarily down or a browser resource limit is reached, RestWrapper will throw
|
|
144
|
+
// a network error with no status code (e.g. err:ERR_CONN_REFUSED or err:ERR_FAILED) and
|
|
145
|
+
// the error message will start with NetworkError as defined in restWrapper.ts
|
|
146
|
+
// If there exists a self-signed SSL certificates error, throw a NonRetryableError
|
|
147
|
+
// TODO: instead of relying on string matching, filter error based on the error code like we do for websocket connections
|
|
148
|
+
const err = errorMessage.includes("failed, reason: self signed certificate")
|
|
149
|
+
? new NonRetryableError(errorMessage, RouterliciousErrorType.sslCertError, {
|
|
150
|
+
driverVersion,
|
|
151
|
+
})
|
|
152
|
+
: new GenericNetworkError(
|
|
153
|
+
errorMessage,
|
|
154
|
+
errorMessage.startsWith("NetworkError"),
|
|
155
|
+
{ driverVersion },
|
|
156
|
+
);
|
|
157
|
+
throw err;
|
|
142
158
|
});
|
|
143
159
|
return {
|
|
144
160
|
response: result,
|
|
@@ -209,17 +225,13 @@ export class RouterliciousRestWrapper extends RestWrapper {
|
|
|
209
225
|
): Promise<Record<string, string>> {
|
|
210
226
|
const token = await this.getToken();
|
|
211
227
|
assert(token !== undefined, 0x679 /* token should be present */);
|
|
212
|
-
const
|
|
213
|
-
|
|
214
|
-
return {
|
|
228
|
+
const headers: Record<string, string> = {
|
|
215
229
|
...requestHeaders,
|
|
216
|
-
|
|
217
|
-
// NOTE: Can correlationId actually be number | true?
|
|
218
|
-
"x-correlation-id": correlationId as string,
|
|
219
|
-
"x-driver-version": driverVersion,
|
|
230
|
+
[DriverVersionHeaderName]: driverVersion,
|
|
220
231
|
// NOTE: If this.authorizationHeader is undefined, should "Authorization" be removed entirely?
|
|
221
|
-
|
|
232
|
+
Authorization: this.getAuthorizationHeader(token),
|
|
222
233
|
};
|
|
234
|
+
return headers;
|
|
223
235
|
}
|
|
224
236
|
|
|
225
237
|
public async getToken(): Promise<ITokenResponse> {
|
|
@@ -5,7 +5,6 @@
|
|
|
5
5
|
|
|
6
6
|
import type * as git from "@fluidframework/gitresources";
|
|
7
7
|
import {
|
|
8
|
-
IWholeFlatSummary,
|
|
9
8
|
IWholeSummaryPayload,
|
|
10
9
|
IWriteSummaryResponse,
|
|
11
10
|
} from "@fluidframework/server-services-client";
|
|
@@ -13,6 +12,7 @@ import { runWithRetry } from "@fluidframework/driver-utils";
|
|
|
13
12
|
import { ITelemetryLoggerExt } from "@fluidframework/telemetry-utils";
|
|
14
13
|
import { IGitManager } from "./storageContracts";
|
|
15
14
|
import { IR11sResponse } from "./restWrapper";
|
|
15
|
+
import { IWholeFlatSnapshot } from "./contracts";
|
|
16
16
|
|
|
17
17
|
export class RetriableGitManager implements IGitManager {
|
|
18
18
|
constructor(
|
|
@@ -70,9 +70,9 @@ export class RetriableGitManager implements IGitManager {
|
|
|
70
70
|
);
|
|
71
71
|
}
|
|
72
72
|
|
|
73
|
-
public async
|
|
73
|
+
public async getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>> {
|
|
74
74
|
return this.runWithRetry(
|
|
75
|
-
async () => this.internalGitManager.
|
|
75
|
+
async () => this.internalGitManager.getSnapshot(sha),
|
|
76
76
|
"gitManager_getSummary",
|
|
77
77
|
);
|
|
78
78
|
}
|
package/src/storageContracts.ts
CHANGED
|
@@ -6,12 +6,12 @@
|
|
|
6
6
|
import * as git from "@fluidframework/gitresources";
|
|
7
7
|
import * as api from "@fluidframework/protocol-definitions";
|
|
8
8
|
import {
|
|
9
|
-
IWholeFlatSummary,
|
|
10
9
|
IWholeSummaryPayload,
|
|
11
10
|
IWholeSummaryPayloadType,
|
|
12
11
|
IWriteSummaryResponse,
|
|
13
12
|
} from "@fluidframework/server-services-client";
|
|
14
13
|
import { IR11sResponse } from "./restWrapper";
|
|
14
|
+
import { IWholeFlatSnapshot } from "./contracts";
|
|
15
15
|
|
|
16
16
|
/**
|
|
17
17
|
* Interface to a generic Git provider
|
|
@@ -26,7 +26,7 @@ export interface IHistorian {
|
|
|
26
26
|
summary: IWholeSummaryPayload,
|
|
27
27
|
initial?: boolean,
|
|
28
28
|
): Promise<IR11sResponse<IWriteSummaryResponse>>;
|
|
29
|
-
|
|
29
|
+
getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>>;
|
|
30
30
|
}
|
|
31
31
|
|
|
32
32
|
export interface IGitManager {
|
|
@@ -39,7 +39,7 @@ export interface IGitManager {
|
|
|
39
39
|
summary: IWholeSummaryPayload,
|
|
40
40
|
initial?: boolean,
|
|
41
41
|
): Promise<IR11sResponse<IWriteSummaryResponse>>;
|
|
42
|
-
|
|
42
|
+
getSnapshot(sha: string): Promise<IR11sResponse<IWholeFlatSnapshot>>;
|
|
43
43
|
}
|
|
44
44
|
|
|
45
45
|
/**
|
package/src/treeUtils.ts
CHANGED
|
@@ -10,7 +10,7 @@ import {
|
|
|
10
10
|
ISummaryTree,
|
|
11
11
|
SummaryObject,
|
|
12
12
|
} from "@fluidframework/protocol-definitions";
|
|
13
|
-
import {
|
|
13
|
+
import { INormalizedWholeSnapshot } from "./contracts";
|
|
14
14
|
|
|
15
15
|
/**
|
|
16
16
|
* Summary tree assembler props
|
|
@@ -107,7 +107,7 @@ export function convertSnapshotAndBlobsToSummaryTree(
|
|
|
107
107
|
return assembler.summary;
|
|
108
108
|
}
|
|
109
109
|
|
|
110
|
-
export function evalBlobsAndTrees(snapshot:
|
|
110
|
+
export function evalBlobsAndTrees(snapshot: INormalizedWholeSnapshot) {
|
|
111
111
|
const trees = countTreesInSnapshotTree(snapshot.snapshotTree);
|
|
112
112
|
const numBlobs = snapshot.blobs.size;
|
|
113
113
|
let encodedBlobsSize = 0;
|