@fluidframework/container-loader 2.0.0-dev.5.2.0.169897 → 2.0.0-dev.6.4.0.191258
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +162 -0
- package/README.md +10 -6
- package/dist/audience.d.ts +1 -0
- package/dist/audience.d.ts.map +1 -1
- package/dist/audience.js +5 -3
- package/dist/audience.js.map +1 -1
- package/dist/catchUpMonitor.d.ts +1 -1
- package/dist/catchUpMonitor.d.ts.map +1 -1
- package/dist/catchUpMonitor.js +2 -2
- package/dist/catchUpMonitor.js.map +1 -1
- package/dist/connectionManager.d.ts +6 -6
- package/dist/connectionManager.d.ts.map +1 -1
- package/dist/connectionManager.js +97 -93
- package/dist/connectionManager.js.map +1 -1
- package/dist/connectionStateHandler.d.ts +19 -15
- package/dist/connectionStateHandler.d.ts.map +1 -1
- package/dist/connectionStateHandler.js +59 -59
- package/dist/connectionStateHandler.js.map +1 -1
- package/dist/container.d.ts +48 -38
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +447 -325
- package/dist/container.js.map +1 -1
- package/dist/containerContext.d.ts +22 -70
- package/dist/containerContext.d.ts.map +1 -1
- package/dist/containerContext.js +24 -221
- package/dist/containerContext.js.map +1 -1
- package/dist/containerStorageAdapter.d.ts +1 -1
- package/dist/containerStorageAdapter.d.ts.map +1 -1
- package/dist/containerStorageAdapter.js +47 -16
- package/dist/containerStorageAdapter.js.map +1 -1
- package/dist/contracts.d.ts +21 -10
- package/dist/contracts.d.ts.map +1 -1
- package/dist/contracts.js +3 -3
- package/dist/contracts.js.map +1 -1
- package/dist/debugLogger.d.ts +30 -0
- package/dist/debugLogger.d.ts.map +1 -0
- package/dist/debugLogger.js +95 -0
- package/dist/debugLogger.js.map +1 -0
- package/dist/deltaManager.d.ts +21 -9
- package/dist/deltaManager.d.ts.map +1 -1
- package/dist/deltaManager.js +114 -66
- package/dist/deltaManager.js.map +1 -1
- package/dist/deltaQueue.d.ts +1 -1
- package/dist/deltaQueue.d.ts.map +1 -1
- package/dist/deltaQueue.js +10 -10
- package/dist/deltaQueue.js.map +1 -1
- package/dist/disposal.d.ts +13 -0
- package/dist/disposal.d.ts.map +1 -0
- package/dist/disposal.js +25 -0
- package/dist/disposal.js.map +1 -0
- package/dist/error.d.ts +23 -0
- package/dist/error.d.ts.map +1 -0
- package/dist/error.js +32 -0
- package/dist/error.js.map +1 -0
- package/dist/loader.d.ts +23 -5
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +82 -51
- package/dist/loader.js.map +1 -1
- package/dist/noopHeuristic.d.ts +23 -0
- package/dist/noopHeuristic.d.ts.map +1 -0
- package/dist/noopHeuristic.js +90 -0
- package/dist/noopHeuristic.js.map +1 -0
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/protocol.d.ts +9 -12
- package/dist/protocol.d.ts.map +1 -1
- package/dist/protocol.js +26 -7
- package/dist/protocol.js.map +1 -1
- package/dist/protocolTreeDocumentStorageService.d.ts +1 -1
- package/dist/protocolTreeDocumentStorageService.d.ts.map +1 -1
- package/dist/protocolTreeDocumentStorageService.js.map +1 -1
- package/dist/quorum.d.ts +1 -14
- package/dist/quorum.d.ts.map +1 -1
- package/dist/quorum.js +1 -29
- package/dist/quorum.js.map +1 -1
- package/dist/retriableDocumentStorageService.d.ts +1 -1
- package/dist/retriableDocumentStorageService.d.ts.map +1 -1
- package/dist/retriableDocumentStorageService.js +4 -4
- package/dist/retriableDocumentStorageService.js.map +1 -1
- package/dist/utils.d.ts +8 -1
- package/dist/utils.d.ts.map +1 -1
- package/dist/utils.js +30 -11
- package/dist/utils.js.map +1 -1
- package/lib/audience.d.ts +1 -0
- package/lib/audience.d.ts.map +1 -1
- package/lib/audience.js +4 -2
- package/lib/audience.js.map +1 -1
- package/lib/catchUpMonitor.d.ts +1 -1
- package/lib/catchUpMonitor.d.ts.map +1 -1
- package/lib/catchUpMonitor.js +1 -1
- package/lib/catchUpMonitor.js.map +1 -1
- package/lib/connectionManager.d.ts +6 -6
- package/lib/connectionManager.d.ts.map +1 -1
- package/lib/connectionManager.js +74 -67
- package/lib/connectionManager.js.map +1 -1
- package/lib/connectionStateHandler.d.ts +19 -15
- package/lib/connectionStateHandler.d.ts.map +1 -1
- package/lib/connectionStateHandler.js +36 -36
- package/lib/connectionStateHandler.js.map +1 -1
- package/lib/container.d.ts +48 -38
- package/lib/container.d.ts.map +1 -1
- package/lib/container.js +414 -292
- package/lib/container.js.map +1 -1
- package/lib/containerContext.d.ts +22 -70
- package/lib/containerContext.d.ts.map +1 -1
- package/lib/containerContext.js +24 -221
- package/lib/containerContext.js.map +1 -1
- package/lib/containerStorageAdapter.d.ts +1 -1
- package/lib/containerStorageAdapter.d.ts.map +1 -1
- package/lib/containerStorageAdapter.js +43 -12
- package/lib/containerStorageAdapter.js.map +1 -1
- package/lib/contracts.d.ts +21 -10
- package/lib/contracts.d.ts.map +1 -1
- package/lib/contracts.js +3 -3
- package/lib/contracts.js.map +1 -1
- package/lib/debugLogger.d.ts +30 -0
- package/lib/debugLogger.d.ts.map +1 -0
- package/lib/debugLogger.js +91 -0
- package/lib/debugLogger.js.map +1 -0
- package/lib/deltaManager.d.ts +21 -9
- package/lib/deltaManager.d.ts.map +1 -1
- package/lib/deltaManager.js +88 -37
- package/lib/deltaManager.js.map +1 -1
- package/lib/deltaQueue.d.ts +1 -1
- package/lib/deltaQueue.d.ts.map +1 -1
- package/lib/deltaQueue.js +3 -3
- package/lib/deltaQueue.js.map +1 -1
- package/lib/disposal.d.ts +13 -0
- package/lib/disposal.d.ts.map +1 -0
- package/lib/disposal.js +21 -0
- package/lib/disposal.js.map +1 -0
- package/lib/error.d.ts +23 -0
- package/lib/error.d.ts.map +1 -0
- package/lib/error.js +28 -0
- package/lib/error.js.map +1 -0
- package/lib/loader.d.ts +23 -5
- package/lib/loader.d.ts.map +1 -1
- package/lib/loader.js +82 -51
- package/lib/loader.js.map +1 -1
- package/lib/noopHeuristic.d.ts +23 -0
- package/lib/noopHeuristic.d.ts.map +1 -0
- package/lib/{collabWindowTracker.js → noopHeuristic.js} +31 -42
- package/lib/noopHeuristic.js.map +1 -0
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/protocol.d.ts +9 -12
- package/lib/protocol.d.ts.map +1 -1
- package/lib/protocol.js +24 -6
- package/lib/protocol.js.map +1 -1
- package/lib/protocolTreeDocumentStorageService.d.ts +1 -1
- package/lib/protocolTreeDocumentStorageService.d.ts.map +1 -1
- package/lib/protocolTreeDocumentStorageService.js.map +1 -1
- package/lib/quorum.d.ts +1 -14
- package/lib/quorum.d.ts.map +1 -1
- package/lib/quorum.js +0 -26
- package/lib/quorum.js.map +1 -1
- package/lib/retriableDocumentStorageService.d.ts +1 -1
- package/lib/retriableDocumentStorageService.d.ts.map +1 -1
- package/lib/retriableDocumentStorageService.js +2 -2
- package/lib/retriableDocumentStorageService.js.map +1 -1
- package/lib/utils.d.ts +8 -1
- package/lib/utils.d.ts.map +1 -1
- package/lib/utils.js +25 -7
- package/lib/utils.js.map +1 -1
- package/package.json +26 -28
- package/src/audience.ts +7 -1
- package/src/catchUpMonitor.ts +2 -2
- package/src/connectionManager.ts +76 -52
- package/src/connectionStateHandler.ts +46 -48
- package/src/container.ts +561 -326
- package/src/containerContext.ts +31 -349
- package/src/containerStorageAdapter.ts +49 -6
- package/src/contracts.ts +27 -13
- package/src/debugLogger.ts +113 -0
- package/src/deltaManager.ts +93 -36
- package/src/deltaQueue.ts +2 -1
- package/src/disposal.ts +25 -0
- package/src/error.ts +44 -0
- package/src/loader.ts +84 -36
- package/src/{collabWindowTracker.ts → noopHeuristic.ts} +38 -47
- package/src/packageVersion.ts +1 -1
- package/src/protocol.ts +26 -16
- package/src/protocolTreeDocumentStorageService.ts +1 -1
- package/src/quorum.ts +1 -40
- package/src/retriableDocumentStorageService.ts +3 -4
- package/src/utils.ts +33 -8
- package/dist/collabWindowTracker.d.ts +0 -19
- package/dist/collabWindowTracker.d.ts.map +0 -1
- package/dist/collabWindowTracker.js +0 -101
- package/dist/collabWindowTracker.js.map +0 -1
- package/dist/deltaManagerProxy.d.ts +0 -42
- package/dist/deltaManagerProxy.d.ts.map +0 -1
- package/dist/deltaManagerProxy.js +0 -79
- package/dist/deltaManagerProxy.js.map +0 -1
- package/lib/collabWindowTracker.d.ts +0 -19
- package/lib/collabWindowTracker.d.ts.map +0 -1
- package/lib/collabWindowTracker.js.map +0 -1
- package/lib/deltaManagerProxy.d.ts +0 -42
- package/lib/deltaManagerProxy.d.ts.map +0 -1
- package/lib/deltaManagerProxy.js +0 -74
- package/lib/deltaManagerProxy.js.map +0 -1
- package/src/deltaManagerProxy.ts +0 -109
package/src/audience.ts
CHANGED
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
import { EventEmitter } from "events";
|
|
6
|
-
import { assert } from "@fluidframework/
|
|
6
|
+
import { assert } from "@fluidframework/core-utils";
|
|
7
7
|
import { IAudienceOwner } from "@fluidframework/container-definitions";
|
|
8
8
|
import { IClient } from "@fluidframework/protocol-definitions";
|
|
9
9
|
|
|
@@ -13,6 +13,12 @@ import { IClient } from "@fluidframework/protocol-definitions";
|
|
|
13
13
|
export class Audience extends EventEmitter implements IAudienceOwner {
|
|
14
14
|
private readonly members = new Map<string, IClient>();
|
|
15
15
|
|
|
16
|
+
constructor() {
|
|
17
|
+
super();
|
|
18
|
+
// We are expecting this class to have many listeners, so we suppress noisy "MaxListenersExceededWarning" logging.
|
|
19
|
+
super.setMaxListeners(0);
|
|
20
|
+
}
|
|
21
|
+
|
|
16
22
|
public on(
|
|
17
23
|
event: "addMember" | "removeMember",
|
|
18
24
|
listener: (clientId: string, client: IClient) => void,
|
package/src/catchUpMonitor.ts
CHANGED
|
@@ -3,8 +3,8 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { IDisposable } from "@fluidframework/
|
|
7
|
-
import { assert } from "@fluidframework/
|
|
6
|
+
import { IDisposable } from "@fluidframework/core-interfaces";
|
|
7
|
+
import { assert } from "@fluidframework/core-utils";
|
|
8
8
|
import { IDeltaManager } from "@fluidframework/container-definitions";
|
|
9
9
|
import { ISequencedDocumentMessage } from "@fluidframework/protocol-definitions";
|
|
10
10
|
|
package/src/connectionManager.ts
CHANGED
|
@@ -3,16 +3,14 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import {
|
|
7
|
-
import {
|
|
8
|
-
import {
|
|
6
|
+
import { IDisposable, ITelemetryProperties } from "@fluidframework/core-interfaces";
|
|
7
|
+
import { assert } from "@fluidframework/core-utils";
|
|
8
|
+
import { performance, TypedEventEmitter } from "@fluid-internal/client-utils";
|
|
9
9
|
import {
|
|
10
|
+
ICriticalContainerError,
|
|
10
11
|
IDeltaQueue,
|
|
11
12
|
ReadOnlyInfo,
|
|
12
|
-
IConnectionDetailsInternal,
|
|
13
|
-
ICriticalContainerError,
|
|
14
13
|
} from "@fluidframework/container-definitions";
|
|
15
|
-
import { GenericError, UsageError } from "@fluidframework/container-utils";
|
|
16
14
|
import {
|
|
17
15
|
IAnyDriverError,
|
|
18
16
|
IDocumentService,
|
|
@@ -26,6 +24,7 @@ import {
|
|
|
26
24
|
getRetryDelayFromError,
|
|
27
25
|
logNetworkFailure,
|
|
28
26
|
isRuntimeMessage,
|
|
27
|
+
calculateMaxWaitTime,
|
|
29
28
|
} from "@fluidframework/driver-utils";
|
|
30
29
|
import {
|
|
31
30
|
ConnectionMode,
|
|
@@ -44,16 +43,23 @@ import {
|
|
|
44
43
|
ISequencedDocumentSystemMessage,
|
|
45
44
|
} from "@fluidframework/protocol-definitions";
|
|
46
45
|
import {
|
|
46
|
+
formatTick,
|
|
47
|
+
GenericError,
|
|
47
48
|
ITelemetryLoggerExt,
|
|
48
|
-
TelemetryLogger,
|
|
49
49
|
normalizeError,
|
|
50
|
+
UsageError,
|
|
50
51
|
} from "@fluidframework/telemetry-utils";
|
|
51
|
-
import {
|
|
52
|
+
import {
|
|
53
|
+
ReconnectMode,
|
|
54
|
+
IConnectionManager,
|
|
55
|
+
IConnectionManagerFactoryArgs,
|
|
56
|
+
IConnectionDetailsInternal,
|
|
57
|
+
IConnectionStateChangeReason,
|
|
58
|
+
} from "./contracts";
|
|
52
59
|
import { DeltaQueue } from "./deltaQueue";
|
|
53
60
|
import { SignalType } from "./protocol";
|
|
54
61
|
import { isDeltaStreamConnectionForbiddenError } from "./utils";
|
|
55
62
|
|
|
56
|
-
const MaxReconnectDelayInMs = 8000;
|
|
57
63
|
const InitialReconnectDelayInMs = 1000;
|
|
58
64
|
const DefaultChunkSize = 16 * 1024;
|
|
59
65
|
|
|
@@ -337,7 +343,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
337
343
|
|
|
338
344
|
private static detailsFromConnection(
|
|
339
345
|
connection: IDocumentDeltaConnection,
|
|
340
|
-
reason:
|
|
346
|
+
reason: IConnectionStateChangeReason,
|
|
341
347
|
): IConnectionDetailsInternal {
|
|
342
348
|
return {
|
|
343
349
|
claims: connection.claims,
|
|
@@ -356,7 +362,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
356
362
|
constructor(
|
|
357
363
|
private readonly serviceProvider: () => IDocumentService | undefined,
|
|
358
364
|
public readonly containerDirty: () => boolean,
|
|
359
|
-
private client: IClient,
|
|
365
|
+
private readonly client: IClient,
|
|
360
366
|
reconnectAllowed: boolean,
|
|
361
367
|
private readonly logger: ITelemetryLoggerExt,
|
|
362
368
|
private readonly props: IConnectionManagerFactoryArgs,
|
|
@@ -390,8 +396,12 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
390
396
|
|
|
391
397
|
this._outbound.clear();
|
|
392
398
|
|
|
393
|
-
const disconnectReason =
|
|
399
|
+
const disconnectReason: IConnectionStateChangeReason = {
|
|
400
|
+
text: "Closing DeltaManager",
|
|
401
|
+
error,
|
|
402
|
+
};
|
|
394
403
|
|
|
404
|
+
const oldReadonlyValue = this.readonly;
|
|
395
405
|
// This raises "disconnect" event if we have active connection.
|
|
396
406
|
this.disconnectFromDeltaStream(disconnectReason);
|
|
397
407
|
|
|
@@ -399,7 +409,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
399
409
|
// Notify everyone we are in read-only state.
|
|
400
410
|
// Useful for data stores in case we hit some critical error,
|
|
401
411
|
// to switch to a mode where user edits are not accepted
|
|
402
|
-
this.set_readonlyPermissions(true);
|
|
412
|
+
this.set_readonlyPermissions(true, oldReadonlyValue);
|
|
403
413
|
}
|
|
404
414
|
}
|
|
405
415
|
|
|
@@ -407,7 +417,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
407
417
|
* Enables or disables automatic reconnecting.
|
|
408
418
|
* Will throw an error if reconnectMode set to Never.
|
|
409
419
|
*/
|
|
410
|
-
public setAutoReconnect(mode: ReconnectMode): void {
|
|
420
|
+
public setAutoReconnect(mode: ReconnectMode, reason: IConnectionStateChangeReason): void {
|
|
411
421
|
assert(
|
|
412
422
|
mode !== ReconnectMode.Never && this._reconnectMode !== ReconnectMode.Never,
|
|
413
423
|
0x278 /* "API is not supported for non-connecting or closed container" */,
|
|
@@ -417,7 +427,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
417
427
|
|
|
418
428
|
if (mode !== ReconnectMode.Enabled) {
|
|
419
429
|
// immediately disconnect - do not rely on service eventually dropping connection.
|
|
420
|
-
this.disconnectFromDeltaStream(
|
|
430
|
+
this.disconnectFromDeltaStream(reason);
|
|
421
431
|
}
|
|
422
432
|
}
|
|
423
433
|
|
|
@@ -464,32 +474,37 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
464
474
|
this.logger.sendErrorEvent({ eventName: "ForceReadonlyPendingChanged" });
|
|
465
475
|
}
|
|
466
476
|
|
|
467
|
-
reconnect = this.disconnectFromDeltaStream("Force readonly");
|
|
477
|
+
reconnect = this.disconnectFromDeltaStream({ text: "Force readonly" });
|
|
468
478
|
}
|
|
469
479
|
this.props.readonlyChangeHandler(this.readonly);
|
|
470
480
|
if (reconnect) {
|
|
471
481
|
// reconnect if we disconnected from before.
|
|
472
|
-
this.triggerConnect("Force Readonly", "read");
|
|
482
|
+
this.triggerConnect({ text: "Force Readonly" }, "read");
|
|
473
483
|
}
|
|
474
484
|
}
|
|
475
485
|
}
|
|
476
486
|
|
|
477
|
-
private set_readonlyPermissions(
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
487
|
+
private set_readonlyPermissions(
|
|
488
|
+
newReadonlyValue: boolean,
|
|
489
|
+
oldReadonlyValue: boolean | undefined,
|
|
490
|
+
) {
|
|
491
|
+
this._readonlyPermissions = newReadonlyValue;
|
|
492
|
+
if (oldReadonlyValue !== this.readonly) {
|
|
481
493
|
this.props.readonlyChangeHandler(this.readonly);
|
|
482
494
|
}
|
|
483
495
|
}
|
|
484
496
|
|
|
485
|
-
public connect(reason:
|
|
486
|
-
this.connectCore(reason, connectionMode).catch((
|
|
487
|
-
const normalizedError = normalizeError(
|
|
497
|
+
public connect(reason: IConnectionStateChangeReason, connectionMode?: ConnectionMode) {
|
|
498
|
+
this.connectCore(reason, connectionMode).catch((e) => {
|
|
499
|
+
const normalizedError = normalizeError(e, { props: fatalConnectErrorProp });
|
|
488
500
|
this.props.closeHandler(normalizedError);
|
|
489
501
|
});
|
|
490
502
|
}
|
|
491
503
|
|
|
492
|
-
private async connectCore(
|
|
504
|
+
private async connectCore(
|
|
505
|
+
reason: IConnectionStateChangeReason,
|
|
506
|
+
connectionMode?: ConnectionMode,
|
|
507
|
+
): Promise<void> {
|
|
493
508
|
assert(!this._disposed, 0x26a /* "not closed" */);
|
|
494
509
|
|
|
495
510
|
if (this.connection !== undefined) {
|
|
@@ -553,7 +568,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
553
568
|
this.logger.sendTelemetryEvent({
|
|
554
569
|
eventName: "ConnectionAttemptCancelled",
|
|
555
570
|
attempts: connectRepeatCount,
|
|
556
|
-
duration:
|
|
571
|
+
duration: formatTick(performance.now() - connectStartTime),
|
|
557
572
|
connectionEstablished: false,
|
|
558
573
|
});
|
|
559
574
|
return;
|
|
@@ -593,7 +608,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
593
608
|
attempts: connectRepeatCount,
|
|
594
609
|
delay: delayMs, // milliseconds
|
|
595
610
|
eventName: "DeltaConnectionFailureToConnect",
|
|
596
|
-
duration:
|
|
611
|
+
duration: formatTick(performance.now() - connectStartTime),
|
|
597
612
|
},
|
|
598
613
|
origError,
|
|
599
614
|
);
|
|
@@ -613,7 +628,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
613
628
|
// We skip this delay if we're confident we're offline, because we probably just need to wait to come back online.
|
|
614
629
|
await new Promise<void>((resolve) => {
|
|
615
630
|
setTimeout(resolve, delayMs);
|
|
616
|
-
delayMs = Math.min(delayMs * 2,
|
|
631
|
+
delayMs = Math.min(delayMs * 2, calculateMaxWaitTime(origError));
|
|
617
632
|
});
|
|
618
633
|
}
|
|
619
634
|
|
|
@@ -639,7 +654,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
639
654
|
{
|
|
640
655
|
eventName: "MultipleDeltaConnectionFailures",
|
|
641
656
|
attempts: connectRepeatCount,
|
|
642
|
-
duration:
|
|
657
|
+
duration: formatTick(performance.now() - connectStartTime),
|
|
643
658
|
},
|
|
644
659
|
lastError,
|
|
645
660
|
);
|
|
@@ -651,7 +666,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
651
666
|
this.logger.sendTelemetryEvent({
|
|
652
667
|
eventName: "ConnectionAttemptCancelled",
|
|
653
668
|
attempts: connectRepeatCount,
|
|
654
|
-
duration:
|
|
669
|
+
duration: formatTick(performance.now() - connectStartTime),
|
|
655
670
|
connectionEstablished: true,
|
|
656
671
|
});
|
|
657
672
|
return;
|
|
@@ -665,7 +680,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
665
680
|
* And report the error if it escapes for any reason.
|
|
666
681
|
* @param args - The connection arguments
|
|
667
682
|
*/
|
|
668
|
-
private triggerConnect(reason:
|
|
683
|
+
private triggerConnect(reason: IConnectionStateChangeReason, connectionMode: ConnectionMode) {
|
|
669
684
|
// reconnect() includes async awaits, and that causes potential race conditions
|
|
670
685
|
// where we might already have a connection. If it were to happen, it's possible that we will connect
|
|
671
686
|
// with different mode to `connectionMode`. Glancing through the caller chains, it looks like code should be
|
|
@@ -685,7 +700,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
685
700
|
* @param error - Error causing the disconnect if any.
|
|
686
701
|
* @returns A boolean that indicates if there was an existing connection (or pending connection) to disconnect
|
|
687
702
|
*/
|
|
688
|
-
private disconnectFromDeltaStream(reason:
|
|
703
|
+
private disconnectFromDeltaStream(reason: IConnectionStateChangeReason): boolean {
|
|
689
704
|
this.pendingReconnect = false;
|
|
690
705
|
|
|
691
706
|
if (this.connection === undefined) {
|
|
@@ -718,7 +733,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
718
733
|
this._outbound.clear();
|
|
719
734
|
connection.dispose();
|
|
720
735
|
|
|
721
|
-
this.props.disconnectHandler(reason
|
|
736
|
+
this.props.disconnectHandler(reason);
|
|
722
737
|
|
|
723
738
|
this._connectionVerboseProps = {};
|
|
724
739
|
|
|
@@ -728,7 +743,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
728
743
|
/**
|
|
729
744
|
* Cancel in-progress connection attempt.
|
|
730
745
|
*/
|
|
731
|
-
private cancelConnection(reason:
|
|
746
|
+
private cancelConnection(reason: IConnectionStateChangeReason) {
|
|
732
747
|
assert(
|
|
733
748
|
this.pendingConnection !== undefined,
|
|
734
749
|
0x345 /* this.pendingConnection is undefined when trying to cancel */,
|
|
@@ -736,7 +751,10 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
736
751
|
this.pendingConnection.abort();
|
|
737
752
|
this.pendingConnection = undefined;
|
|
738
753
|
this.logger.sendTelemetryEvent({ eventName: "ConnectionCancelReceived" });
|
|
739
|
-
this.props.cancelConnectionHandler(
|
|
754
|
+
this.props.cancelConnectionHandler({
|
|
755
|
+
text: `Cancel Pending Connection due to ${reason.text}`,
|
|
756
|
+
error: reason.error,
|
|
757
|
+
});
|
|
740
758
|
}
|
|
741
759
|
|
|
742
760
|
/**
|
|
@@ -747,7 +765,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
747
765
|
private setupNewSuccessfulConnection(
|
|
748
766
|
connection: IDocumentDeltaConnection,
|
|
749
767
|
requestedMode: ConnectionMode,
|
|
750
|
-
reason:
|
|
768
|
+
reason: IConnectionStateChangeReason,
|
|
751
769
|
) {
|
|
752
770
|
// Old connection should have been cleaned up before establishing a new one
|
|
753
771
|
assert(
|
|
@@ -761,6 +779,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
761
779
|
|
|
762
780
|
this.pendingConnection = undefined;
|
|
763
781
|
|
|
782
|
+
const oldReadonlyValue = this.readonly;
|
|
764
783
|
this.connection = connection;
|
|
765
784
|
|
|
766
785
|
// Does information in scopes & mode matches?
|
|
@@ -786,11 +805,11 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
786
805
|
0x0e8 /* "readonly perf with write connection" */,
|
|
787
806
|
);
|
|
788
807
|
|
|
789
|
-
this.set_readonlyPermissions(readonly);
|
|
808
|
+
this.set_readonlyPermissions(readonly, oldReadonlyValue);
|
|
790
809
|
|
|
791
810
|
if (this._disposed) {
|
|
792
811
|
// Raise proper events, Log telemetry event and close connection.
|
|
793
|
-
this.disconnectFromDeltaStream("ConnectionManager already closed");
|
|
812
|
+
this.disconnectFromDeltaStream({ text: "ConnectionManager already closed" });
|
|
794
813
|
return;
|
|
795
814
|
}
|
|
796
815
|
|
|
@@ -895,7 +914,9 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
895
914
|
* @returns A promise that resolves when the connection is reestablished or we stop trying
|
|
896
915
|
*/
|
|
897
916
|
private reconnectOnError(requestedMode: ConnectionMode, error: IAnyDriverError) {
|
|
898
|
-
this.reconnect(requestedMode, error.message, error).catch(
|
|
917
|
+
this.reconnect(requestedMode, { text: error.message, error }).catch(
|
|
918
|
+
this.props.closeHandler,
|
|
919
|
+
);
|
|
899
920
|
}
|
|
900
921
|
|
|
901
922
|
/**
|
|
@@ -907,26 +928,25 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
907
928
|
*/
|
|
908
929
|
private async reconnect(
|
|
909
930
|
requestedMode: ConnectionMode,
|
|
910
|
-
|
|
911
|
-
error?: IAnyDriverError,
|
|
931
|
+
reason: IConnectionStateChangeReason<IAnyDriverError>,
|
|
912
932
|
) {
|
|
913
933
|
// We quite often get protocol errors before / after observing nack/disconnect
|
|
914
934
|
// we do not want to run through same sequence twice.
|
|
915
935
|
// If we're already disconnected/disconnecting it's not appropriate to call this again.
|
|
916
936
|
assert(this.connection !== undefined, 0x0eb /* "Missing connection for reconnect" */);
|
|
917
937
|
|
|
918
|
-
this.disconnectFromDeltaStream(
|
|
938
|
+
this.disconnectFromDeltaStream(reason);
|
|
919
939
|
|
|
920
940
|
// We will always trigger reconnect, even if canRetry is false.
|
|
921
941
|
// Any truly fatal error state will result in container close upon attempted reconnect,
|
|
922
942
|
// which is a preferable to closing abruptly when a live connection fails.
|
|
923
|
-
if (error
|
|
943
|
+
if (reason.error?.canRetry === false) {
|
|
924
944
|
this.logger.sendTelemetryEvent(
|
|
925
945
|
{
|
|
926
946
|
eventName: "reconnectingDespiteFatalError",
|
|
927
947
|
reconnectMode: this.reconnectMode,
|
|
928
948
|
},
|
|
929
|
-
error,
|
|
949
|
+
reason.error,
|
|
930
950
|
);
|
|
931
951
|
}
|
|
932
952
|
|
|
@@ -943,9 +963,9 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
943
963
|
}
|
|
944
964
|
|
|
945
965
|
// If the error tells us to wait before retrying, then do so.
|
|
946
|
-
const delayMs = getRetryDelayFromError(error);
|
|
947
|
-
if (error !== undefined && delayMs !== undefined) {
|
|
948
|
-
this.props.reconnectionDelayHandler(delayMs, error);
|
|
966
|
+
const delayMs = getRetryDelayFromError(reason.error);
|
|
967
|
+
if (reason.error !== undefined && delayMs !== undefined) {
|
|
968
|
+
this.props.reconnectionDelayHandler(delayMs, reason.error);
|
|
949
969
|
await new Promise<void>((resolve) => {
|
|
950
970
|
setTimeout(resolve, delayMs);
|
|
951
971
|
});
|
|
@@ -957,9 +977,13 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
957
977
|
await waitForOnline();
|
|
958
978
|
|
|
959
979
|
this.triggerConnect(
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
980
|
+
{
|
|
981
|
+
text:
|
|
982
|
+
reason.error !== undefined
|
|
983
|
+
? "Reconnecting due to Error"
|
|
984
|
+
: `Reconnecting due to: ${reason.text}`,
|
|
985
|
+
error: reason.error,
|
|
986
|
+
},
|
|
963
987
|
requestedMode,
|
|
964
988
|
);
|
|
965
989
|
}
|
|
@@ -1030,7 +1054,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
1030
1054
|
// still valid?
|
|
1031
1055
|
await this.reconnect(
|
|
1032
1056
|
"write", // connectionMode
|
|
1033
|
-
"Switch to write", // message
|
|
1057
|
+
{ text: "Switch to write" }, // message
|
|
1034
1058
|
);
|
|
1035
1059
|
}
|
|
1036
1060
|
})
|
|
@@ -1084,7 +1108,7 @@ export class ConnectionManager implements IConnectionManager {
|
|
|
1084
1108
|
// Note - this may close container!
|
|
1085
1109
|
this.reconnect(
|
|
1086
1110
|
"read", // connectionMode
|
|
1087
|
-
"Switch to read", // message
|
|
1111
|
+
{ text: "Switch to read" }, // message
|
|
1088
1112
|
).catch((error) => {
|
|
1089
1113
|
this.logger.sendErrorEvent({ eventName: "SwitchToReadConnection" }, error);
|
|
1090
1114
|
});
|
|
@@ -3,19 +3,20 @@
|
|
|
3
3
|
* Licensed under the MIT License.
|
|
4
4
|
*/
|
|
5
5
|
|
|
6
|
-
import { ITelemetryProperties, TelemetryEventCategory } from "@fluidframework/
|
|
7
|
-
import { assert, Timer } from "@fluidframework/
|
|
8
|
-
import {
|
|
9
|
-
import { IAnyDriverError } from "@fluidframework/driver-definitions";
|
|
6
|
+
import { ITelemetryProperties, TelemetryEventCategory } from "@fluidframework/core-interfaces";
|
|
7
|
+
import { assert, Timer } from "@fluidframework/core-utils";
|
|
8
|
+
import { IDeltaManager } from "@fluidframework/container-definitions";
|
|
10
9
|
import { ISequencedClient, IClient } from "@fluidframework/protocol-definitions";
|
|
11
10
|
import {
|
|
12
11
|
ITelemetryLoggerExt,
|
|
13
12
|
PerformanceEvent,
|
|
14
13
|
loggerToMonitoringContext,
|
|
15
14
|
} from "@fluidframework/telemetry-utils";
|
|
16
|
-
import {
|
|
15
|
+
import { IAnyDriverError } from "@fluidframework/driver-definitions";
|
|
17
16
|
import { CatchUpMonitor, ICatchUpMonitor } from "./catchUpMonitor";
|
|
18
|
-
import {
|
|
17
|
+
import { ConnectionState } from "./connectionState";
|
|
18
|
+
import { IConnectionDetailsInternal, IConnectionStateChangeReason } from "./contracts";
|
|
19
|
+
import { IProtocolHandler } from "./protocol";
|
|
19
20
|
|
|
20
21
|
// Based on recent data, it looks like majority of cases where we get stuck are due to really slow or
|
|
21
22
|
// timing out ops fetches. So attempt recovery infrequently. Also fetch uses 30 second timeout, so
|
|
@@ -32,8 +33,7 @@ export interface IConnectionStateHandlerInputs {
|
|
|
32
33
|
connectionStateChanged: (
|
|
33
34
|
value: ConnectionState,
|
|
34
35
|
oldState: ConnectionState,
|
|
35
|
-
reason?:
|
|
36
|
-
error?: IAnyDriverError,
|
|
36
|
+
reason?: IConnectionStateChangeReason,
|
|
37
37
|
) => void;
|
|
38
38
|
/** Whether to expect the client to join in write mode on next connection */
|
|
39
39
|
shouldClientJoinWrite: () => boolean;
|
|
@@ -45,6 +45,8 @@ export interface IConnectionStateHandlerInputs {
|
|
|
45
45
|
category: TelemetryEventCategory,
|
|
46
46
|
details?: ITelemetryProperties,
|
|
47
47
|
) => void;
|
|
48
|
+
/** Callback to note that an old local client ID is still present in the Quorum that should have left and should now be considered invalid */
|
|
49
|
+
clientShouldHaveLeft: (clientId: string) => void;
|
|
48
50
|
}
|
|
49
51
|
|
|
50
52
|
/**
|
|
@@ -58,14 +60,14 @@ export interface IConnectionStateHandler {
|
|
|
58
60
|
dispose(): void;
|
|
59
61
|
initProtocol(protocol: IProtocolHandler): void;
|
|
60
62
|
receivedConnectEvent(details: IConnectionDetailsInternal): void;
|
|
61
|
-
receivedDisconnectEvent(reason:
|
|
62
|
-
establishingConnection(reason:
|
|
63
|
+
receivedDisconnectEvent(reason: IConnectionStateChangeReason): void;
|
|
64
|
+
establishingConnection(reason: IConnectionStateChangeReason): void;
|
|
63
65
|
/**
|
|
64
66
|
* Switches state to disconnected when we are still establishing connection during container.load(),
|
|
65
67
|
* container connect() or reconnect and the container gets closed or disposed or disconnect happens.
|
|
66
68
|
* @param reason - reason for cancelling the connection.
|
|
67
69
|
*/
|
|
68
|
-
cancelEstablishingConnection(reason:
|
|
70
|
+
cancelEstablishingConnection(reason: IConnectionStateChangeReason): void;
|
|
69
71
|
}
|
|
70
72
|
|
|
71
73
|
export function createConnectionStateHandler(
|
|
@@ -147,15 +149,15 @@ class ConnectionStateHandlerPassThrough
|
|
|
147
149
|
public initProtocol(protocol: IProtocolHandler) {
|
|
148
150
|
return this.pimpl.initProtocol(protocol);
|
|
149
151
|
}
|
|
150
|
-
public receivedDisconnectEvent(reason:
|
|
151
|
-
return this.pimpl.receivedDisconnectEvent(reason
|
|
152
|
+
public receivedDisconnectEvent(reason: IConnectionStateChangeReason<IAnyDriverError>) {
|
|
153
|
+
return this.pimpl.receivedDisconnectEvent(reason);
|
|
152
154
|
}
|
|
153
155
|
|
|
154
|
-
public establishingConnection(reason:
|
|
156
|
+
public establishingConnection(reason: IConnectionStateChangeReason) {
|
|
155
157
|
return this.pimpl.establishingConnection(reason);
|
|
156
158
|
}
|
|
157
159
|
|
|
158
|
-
public cancelEstablishingConnection(reason:
|
|
160
|
+
public cancelEstablishingConnection(reason: IConnectionStateChangeReason) {
|
|
159
161
|
return this.pimpl.cancelEstablishingConnection(reason);
|
|
160
162
|
}
|
|
161
163
|
|
|
@@ -173,10 +175,9 @@ class ConnectionStateHandlerPassThrough
|
|
|
173
175
|
public connectionStateChanged(
|
|
174
176
|
value: ConnectionState,
|
|
175
177
|
oldState: ConnectionState,
|
|
176
|
-
reason?:
|
|
177
|
-
error?: IAnyDriverError,
|
|
178
|
+
reason?: IConnectionStateChangeReason,
|
|
178
179
|
) {
|
|
179
|
-
return this.inputs.connectionStateChanged(value, oldState, reason
|
|
180
|
+
return this.inputs.connectionStateChanged(value, oldState, reason);
|
|
180
181
|
}
|
|
181
182
|
public shouldClientJoinWrite() {
|
|
182
183
|
return this.inputs.shouldClientJoinWrite();
|
|
@@ -191,6 +192,9 @@ class ConnectionStateHandlerPassThrough
|
|
|
191
192
|
) {
|
|
192
193
|
return this.inputs.logConnectionIssue(eventName, category, details);
|
|
193
194
|
}
|
|
195
|
+
public clientShouldHaveLeft(clientId: string) {
|
|
196
|
+
return this.inputs.clientShouldHaveLeft(clientId);
|
|
197
|
+
}
|
|
194
198
|
}
|
|
195
199
|
|
|
196
200
|
/**
|
|
@@ -217,8 +221,7 @@ class ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {
|
|
|
217
221
|
public connectionStateChanged(
|
|
218
222
|
value: ConnectionState,
|
|
219
223
|
oldState: ConnectionState,
|
|
220
|
-
reason?:
|
|
221
|
-
error?: IAnyDriverError,
|
|
224
|
+
reason?: IConnectionStateChangeReason<IAnyDriverError>,
|
|
222
225
|
) {
|
|
223
226
|
switch (value) {
|
|
224
227
|
case ConnectionState.Connected:
|
|
@@ -262,7 +265,7 @@ class ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {
|
|
|
262
265
|
default:
|
|
263
266
|
}
|
|
264
267
|
this._connectionState = value;
|
|
265
|
-
this.inputs.connectionStateChanged(value, oldState, reason
|
|
268
|
+
this.inputs.connectionStateChanged(value, oldState, reason);
|
|
266
269
|
}
|
|
267
270
|
|
|
268
271
|
private readonly transitionToConnectedState = () => {
|
|
@@ -271,11 +274,9 @@ class ConnectionStateCatchup extends ConnectionStateHandlerPassThrough {
|
|
|
271
274
|
assert(state === ConnectionState.Connected, 0x3e5 /* invariant broken */);
|
|
272
275
|
assert(this._connectionState === ConnectionState.CatchingUp, 0x3e6 /* invariant broken */);
|
|
273
276
|
this._connectionState = ConnectionState.Connected;
|
|
274
|
-
this.inputs.connectionStateChanged(
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
"caught up",
|
|
278
|
-
);
|
|
277
|
+
this.inputs.connectionStateChanged(ConnectionState.Connected, ConnectionState.CatchingUp, {
|
|
278
|
+
text: "caught up",
|
|
279
|
+
});
|
|
279
280
|
};
|
|
280
281
|
}
|
|
281
282
|
|
|
@@ -489,12 +490,12 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
489
490
|
}
|
|
490
491
|
}
|
|
491
492
|
|
|
492
|
-
public receivedDisconnectEvent(reason:
|
|
493
|
+
public receivedDisconnectEvent(reason: IConnectionStateChangeReason<IAnyDriverError>) {
|
|
493
494
|
this.connection = undefined;
|
|
494
|
-
this.setConnectionState(ConnectionState.Disconnected, reason
|
|
495
|
+
this.setConnectionState(ConnectionState.Disconnected, reason);
|
|
495
496
|
}
|
|
496
497
|
|
|
497
|
-
public cancelEstablishingConnection(reason:
|
|
498
|
+
public cancelEstablishingConnection(reason: IConnectionStateChangeReason) {
|
|
498
499
|
assert(
|
|
499
500
|
this._connectionState === ConnectionState.EstablishingConnection,
|
|
500
501
|
0x6d3 /* Connection state should be EstablishingConnection */,
|
|
@@ -505,14 +506,13 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
505
506
|
this.handler.connectionStateChanged(ConnectionState.Disconnected, oldState, reason);
|
|
506
507
|
}
|
|
507
508
|
|
|
508
|
-
public establishingConnection(reason:
|
|
509
|
+
public establishingConnection(reason: IConnectionStateChangeReason) {
|
|
509
510
|
const oldState = this._connectionState;
|
|
510
511
|
this._connectionState = ConnectionState.EstablishingConnection;
|
|
511
|
-
this.handler.connectionStateChanged(
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
);
|
|
512
|
+
this.handler.connectionStateChanged(ConnectionState.EstablishingConnection, oldState, {
|
|
513
|
+
text: `Establishing Connection due to ${reason.text}`,
|
|
514
|
+
error: reason.error,
|
|
515
|
+
});
|
|
516
516
|
}
|
|
517
517
|
|
|
518
518
|
private shouldWaitForJoinSignal() {
|
|
@@ -577,14 +577,12 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
577
577
|
|
|
578
578
|
private setConnectionState(
|
|
579
579
|
value: ConnectionState.Disconnected,
|
|
580
|
-
reason:
|
|
581
|
-
error?: IAnyDriverError,
|
|
580
|
+
reason: IConnectionStateChangeReason,
|
|
582
581
|
): void;
|
|
583
582
|
private setConnectionState(value: ConnectionState.Connected): void;
|
|
584
583
|
private setConnectionState(
|
|
585
584
|
value: ConnectionState.Disconnected | ConnectionState.Connected,
|
|
586
|
-
reason?:
|
|
587
|
-
error?: IAnyDriverError,
|
|
585
|
+
reason?: IConnectionStateChangeReason,
|
|
588
586
|
): void {
|
|
589
587
|
if (this.connectionState === value) {
|
|
590
588
|
// Already in the desired state - exit early
|
|
@@ -598,18 +596,18 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
598
596
|
// This is the only place in code that deals with quorum. The rest works with audience
|
|
599
597
|
// The code below ensures that we do not send ops until we know that old "write" client's disconnect
|
|
600
598
|
// produced (and sequenced) leave op
|
|
601
|
-
|
|
602
|
-
|
|
603
|
-
|
|
604
|
-
}
|
|
599
|
+
const currentClientInQuorum =
|
|
600
|
+
this._clientId !== undefined &&
|
|
601
|
+
this.protocol?.quorum?.getMember(this._clientId) !== undefined;
|
|
605
602
|
if (value === ConnectionState.Connected) {
|
|
606
603
|
assert(
|
|
607
604
|
oldState === ConnectionState.CatchingUp,
|
|
608
605
|
0x1d8 /* "Should only transition from Connecting state" */,
|
|
609
606
|
);
|
|
610
607
|
// Mark our old client should have left in the quorum if it's still there
|
|
611
|
-
if (
|
|
612
|
-
|
|
608
|
+
if (currentClientInQuorum) {
|
|
609
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
610
|
+
this.handler.clientShouldHaveLeft(this._clientId!);
|
|
613
611
|
}
|
|
614
612
|
this._clientId = this.pendingClientId;
|
|
615
613
|
} else if (value === ConnectionState.Disconnected) {
|
|
@@ -625,7 +623,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
625
623
|
// we could receive "Disconnected" event multiple times without getting connected and in that case we
|
|
626
624
|
// don't want to reset the timer as we still want to wait on original client which started this timer.
|
|
627
625
|
if (
|
|
628
|
-
|
|
626
|
+
currentClientInQuorum &&
|
|
629
627
|
this.handler.shouldClientJoinWrite() &&
|
|
630
628
|
!this.waitingForLeaveOp // same as !this.prevClientLeftTimer.hasTimer
|
|
631
629
|
) {
|
|
@@ -636,7 +634,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
636
634
|
eventName: "noWaitOnDisconnected",
|
|
637
635
|
details: JSON.stringify({
|
|
638
636
|
clientId: this._clientId,
|
|
639
|
-
inQuorum:
|
|
637
|
+
inQuorum: currentClientInQuorum,
|
|
640
638
|
waitingForLeaveOp: this.waitingForLeaveOp,
|
|
641
639
|
hadOutstandingOps: this.handler.shouldClientJoinWrite(),
|
|
642
640
|
}),
|
|
@@ -645,7 +643,7 @@ class ConnectionStateHandler implements IConnectionStateHandler {
|
|
|
645
643
|
}
|
|
646
644
|
|
|
647
645
|
// Report transition before we propagate event across layers
|
|
648
|
-
this.handler.connectionStateChanged(this._connectionState, oldState, reason
|
|
646
|
+
this.handler.connectionStateChanged(this._connectionState, oldState, reason);
|
|
649
647
|
}
|
|
650
648
|
|
|
651
649
|
// Helper method to switch between quorum and audience.
|