@fluidframework/driver-base 2.0.0-internal.3.0.1 → 2.0.0-internal.3.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -5,31 +5,31 @@
5
5
 
6
6
  import { assert } from "@fluidframework/common-utils";
7
7
  import {
8
- IAnyDriverError,
9
- IDocumentDeltaConnection,
10
- IDocumentDeltaConnectionEvents,
8
+ IAnyDriverError,
9
+ IDocumentDeltaConnection,
10
+ IDocumentDeltaConnectionEvents,
11
11
  } from "@fluidframework/driver-definitions";
12
12
  import { createGenericNetworkError } from "@fluidframework/driver-utils";
13
13
  import {
14
- ConnectionMode,
15
- IClientConfiguration,
16
- IConnect,
17
- IConnected,
18
- IDocumentMessage,
19
- ISequencedDocumentMessage,
20
- ISignalClient,
21
- ISignalMessage,
22
- ITokenClaims,
23
- ScopeType,
14
+ ConnectionMode,
15
+ IClientConfiguration,
16
+ IConnect,
17
+ IConnected,
18
+ IDocumentMessage,
19
+ ISequencedDocumentMessage,
20
+ ISignalClient,
21
+ ISignalMessage,
22
+ ITokenClaims,
23
+ ScopeType,
24
24
  } from "@fluidframework/protocol-definitions";
25
25
  import { IDisposable, ITelemetryLogger } from "@fluidframework/common-definitions";
26
26
  import {
27
- ChildLogger,
28
- extractLogSafeErrorProperties,
29
- getCircularReplacer,
30
- loggerToMonitoringContext,
31
- MonitoringContext,
32
- EventEmitterWithErrorHandling,
27
+ ChildLogger,
28
+ extractLogSafeErrorProperties,
29
+ getCircularReplacer,
30
+ loggerToMonitoringContext,
31
+ MonitoringContext,
32
+ EventEmitterWithErrorHandling,
33
33
  } from "@fluidframework/telemetry-utils";
34
34
  import type { Socket } from "socket.io-client";
35
35
  // For now, this package is versioned and released in unison with the specific drivers
@@ -39,596 +39,644 @@ import { pkgVersion as driverVersion } from "./packageVersion";
39
39
  * Represents a connection to a stream of delta updates
40
40
  */
41
41
  export class DocumentDeltaConnection
42
- extends EventEmitterWithErrorHandling<IDocumentDeltaConnectionEvents>
43
- implements IDocumentDeltaConnection, IDisposable {
44
- static readonly eventsToForward = ["nack", "op", "signal", "pong"];
45
-
46
- // WARNING: These are critical events that we can't miss, so registration for them has to be in place at all times!
47
- // Including before handshake is over, and after that (but before DeltaManager had a chance to put its own handlers)
48
- static readonly eventsAlwaysForwarded = ["disconnect", "error"];
49
-
50
- /**
51
- * Last known sequence number to ordering service at the time of connection
52
- * It may lap actual last sequence number (quite a bit, if container is very active).
53
- * But it's best information for client to figure out how far it is behind, at least
54
- * for "read" connections. "write" connections may use own "join" op to similar information,
55
- * that is likely to be more up-to-date.
56
- */
57
- public checkpointSequenceNumber: number | undefined;
58
-
59
- // Listen for ops sent before we receive a response to connect_document
60
- protected readonly queuedMessages: ISequencedDocumentMessage[] = [];
61
- protected readonly queuedSignals: ISignalMessage[] = [];
62
-
63
- /**
64
- * A flag to indicate whether we have our handler attached. If it's attached, we're queueing incoming ops
65
- * to later be retrieved via initialMessages.
66
- */
67
- private earlyOpHandlerAttached: boolean = false;
68
-
69
- private socketConnectionTimeout: ReturnType<typeof setTimeout> | undefined;
70
-
71
- private _details: IConnected | undefined;
72
-
73
- // Listeners only needed while the connection is in progress
74
- private readonly connectionListeners: Map<string, (...args: any[]) => void> = new Map();
75
- // Listeners used throughout the lifetime of the DocumentDeltaConnection
76
- private readonly trackedListeners: Map<string, (...args: any[]) => void> = new Map();
77
-
78
- protected get hasDetails(): boolean {
79
- return !!this._details;
80
- }
81
-
82
- public get disposed() {
83
- assert(this._disposed || this.socket.connected, 0x244 /* "Socket is closed, but connection is not!" */);
84
- return this._disposed;
85
- }
86
-
87
- /**
88
- * Flag to indicate whether the DocumentDeltaConnection is expected to still be capable of sending messages.
89
- * After disconnection, we flip this to prevent any stale messages from being emitted.
90
- */
91
- protected _disposed: boolean = false;
92
- private readonly mc: MonitoringContext;
93
-
94
- /**
95
- * @deprecated Implementors should manage their own logger or monitoring context
96
- */
97
- protected get logger(): ITelemetryLogger {
98
- return this.mc.logger;
99
- }
100
-
101
- public get details(): IConnected {
102
- if (!this._details) {
103
- throw new Error("Internal error: calling method before _details is initialized!");
104
- }
105
- return this._details;
106
- }
107
-
108
- /**
109
- * @param socket - websocket to be used
110
- * @param documentId - ID of the document
111
- * @param logger - for reporting telemetry events
112
- * @param enableLongPollingDowngrades - allow connection to be downgraded to long-polling on websocket failure
113
- */
114
- protected constructor(
115
- protected readonly socket: Socket,
116
- public documentId: string,
117
- logger: ITelemetryLogger,
118
- private readonly enableLongPollingDowngrades: boolean = false,
119
- ) {
120
- super((name, error) => {
121
- logger.sendErrorEvent(
122
- {
123
- eventName: "DeltaConnection:EventException",
124
- name,
125
- },
126
- error);
127
- });
128
-
129
- this.mc = loggerToMonitoringContext(
130
- ChildLogger.create(logger, "DeltaConnection"));
131
-
132
- this.on("newListener", (event, listener) => {
133
- assert(!this.disposed, 0x20a /* "register for event on disposed object" */);
134
-
135
- // Some events are already forwarded - see this.addTrackedListener() calls in initialize().
136
- if (DocumentDeltaConnection.eventsAlwaysForwarded.includes(event)) {
137
- assert(this.trackedListeners.has(event), 0x245 /* "tracked listener" */);
138
- return;
139
- }
140
-
141
- if (!DocumentDeltaConnection.eventsToForward.includes(event)) {
142
- throw new Error(`DocumentDeltaConnection: Registering for unknown event: ${event}`);
143
- }
144
-
145
- // Whenever listener is added, we should subscribe on same event on socket, so these two things
146
- // should be in sync. This currently assumes that nobody unregisters and registers back listeners,
147
- // and that there are no "internal" listeners installed (like "error" case we skip above)
148
- // Better flow might be to always unconditionally register all handlers on successful connection,
149
- // though some logic (naming assert in initialMessages getter) might need to be adjusted (it becomes noop)
150
- assert((this.listeners(event).length !== 0) === this.trackedListeners.has(event), 0x20b /* "mismatch" */);
151
- if (!this.trackedListeners.has(event)) {
152
- this.addTrackedListener(
153
- event,
154
- (...args: any[]) => {
155
- this.emit(event, ...args);
156
- });
157
- }
158
- });
159
- }
160
-
161
- /**
162
- * Get the ID of the client who is sending the message
163
- *
164
- * @returns the client ID
165
- */
166
- public get clientId(): string {
167
- return this.details.clientId;
168
- }
169
-
170
- /**
171
- * Get the mode of the client
172
- *
173
- * @returns the client mode
174
- */
175
- public get mode(): ConnectionMode {
176
- return this.details.mode;
177
- }
178
-
179
- /**
180
- * Get the claims of the client who is sending the message
181
- *
182
- * @returns client claims
183
- */
184
- public get claims(): ITokenClaims {
185
- return this.details.claims;
186
- }
187
-
188
- /**
189
- * Get whether or not this is an existing document
190
- *
191
- * @returns true if the document exists
192
- */
193
- public get existing(): boolean {
194
- return this.details.existing;
195
- }
196
-
197
- /**
198
- * Get the maximum size of a message before chunking is required
199
- *
200
- * @returns the maximum size of a message before chunking is required
201
- */
202
- public get maxMessageSize(): number {
203
- return this.details.serviceConfiguration.maxMessageSize;
204
- }
205
-
206
- /**
207
- * Semver of protocol being used with the service
208
- */
209
- public get version(): string {
210
- return this.details.version;
211
- }
212
-
213
- /**
214
- * Configuration details provided by the service
215
- */
216
- public get serviceConfiguration(): IClientConfiguration {
217
- return this.details.serviceConfiguration;
218
- }
219
-
220
- private checkNotClosed() {
221
- assert(!this.disposed, 0x20c /* "connection disposed" */);
222
- }
223
-
224
- /**
225
- * Get messages sent during the connection
226
- *
227
- * @returns messages sent during the connection
228
- */
229
- public get initialMessages(): ISequencedDocumentMessage[] {
230
- this.checkNotClosed();
231
-
232
- // If we call this when the earlyOpHandler is not attached, then the queuedMessages may not include the
233
- // latest ops. This could possibly indicate that initialMessages was called twice.
234
- assert(this.earlyOpHandlerAttached, 0x08e /* "Potentially missed initial messages" */);
235
- // We will lose ops and perf will tank as we need to go to storage to become current!
236
- assert(this.listeners("op").length !== 0, 0x08f /* "No op handler is setup!" */);
237
-
238
- this.removeEarlyOpHandler();
239
-
240
- if (this.queuedMessages.length > 0) {
241
- // Some messages were queued.
242
- // add them to the list of initialMessages to be processed
243
- this.details.initialMessages.push(...this.queuedMessages);
244
- this.details.initialMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
245
- this.queuedMessages.length = 0;
246
- }
247
- return this.details.initialMessages;
248
- }
249
-
250
- /**
251
- * Get signals sent during the connection
252
- *
253
- * @returns signals sent during the connection
254
- */
255
- public get initialSignals(): ISignalMessage[] {
256
- this.checkNotClosed();
257
- assert(this.listeners("signal").length !== 0, 0x090 /* "No signal handler is setup!" */);
258
-
259
- this.removeEarlySignalHandler();
260
-
261
- if (this.queuedSignals.length > 0) {
262
- // Some signals were queued.
263
- // add them to the list of initialSignals to be processed
264
- this.details.initialSignals.push(...this.queuedSignals);
265
- this.queuedSignals.length = 0;
266
- }
267
- return this.details.initialSignals;
268
- }
269
-
270
- /**
271
- * Get initial client list
272
- *
273
- * @returns initial client list sent during the connection
274
- */
275
- public get initialClients(): ISignalClient[] {
276
- this.checkNotClosed();
277
- return this.details.initialClients;
278
- }
279
-
280
- protected emitMessages(type: string, messages: IDocumentMessage[][]) {
281
- // Although the implementation here disconnects the socket and does not reuse it, other subclasses
282
- // (e.g. OdspDocumentDeltaConnection) may reuse the socket. In these cases, we need to avoid emitting
283
- // on the still-live socket.
284
- if (!this.disposed) {
285
- this.socket.emit(type, this.clientId, messages);
286
- }
287
- }
288
-
289
- protected submitCore(type: string, messages: IDocumentMessage[]) {
290
- this.emitMessages(type, [messages]);
291
- }
292
-
293
- /**
294
- * Submits a new delta operation to the server
295
- *
296
- * @param message - delta operation to submit
297
- */
298
- public submit(messages: IDocumentMessage[]): void {
299
- this.checkNotClosed();
300
- this.submitCore("submitOp", messages);
301
- }
302
-
303
- /**
304
- * Submits a new signal to the server
305
- *
306
- * @param message - signal to submit
307
- */
308
- public submitSignal(message: IDocumentMessage): void {
309
- this.checkNotClosed();
310
- this.submitCore("submitSignal", [message]);
311
- }
312
-
313
- /**
314
- * Disconnect from the websocket and close the websocket too.
315
- */
316
- protected closeSocket(error: IAnyDriverError) {
317
- this.disconnect(error);
318
- }
319
-
320
- /**
321
- * Disconnect from the websocket, and permanently disable this DocumentDeltaConnection and close the socket.
322
- * However the OdspDocumentDeltaConnection differ in dispose as in there we don't close the socket. There is no
323
- * multiplexing here, so we need to close the socket here.
324
- */
325
- public dispose() {
326
- this.logger.sendTelemetryEvent({
327
- eventName: "ClientClosingDeltaConnection",
328
- driverVersion,
329
- details: JSON.stringify({ disposed: this._disposed, socketConnected: this.socket.connected }),
330
- });
331
- this.disconnect(createGenericNetworkError(
332
- // pre-0.58 error message: clientClosingConnection
333
- "Client closing delta connection", { canRetry: true }, { driverVersion }),
334
- );
335
- }
336
-
337
- protected disconnect(err: IAnyDriverError) {
338
- // Can't check this.disposed here, as we get here on socket closure,
339
- // so _disposed & socket.connected might be not in sync while processing
340
- // "dispose" event.
341
- if (this._disposed) {
342
- return;
343
- }
344
-
345
- // We set the disposed flag as a part of the contract for overriding the disconnect method. This is used by
346
- // DocumentDeltaConnection to determine if emitting messages (ops) on the socket is allowed, which is
347
- // important since OdspDocumentDeltaConnection reuses the socket rather than truly disconnecting it. Note that
348
- // OdspDocumentDeltaConnection may still send disconnect_document which is allowed; this is only intended
349
- // to prevent normal messages from being emitted.
350
- this._disposed = true;
351
-
352
- // Let user of connection object know about disconnect. This has to happen in between setting _disposed and
353
- // removing all listeners!
354
- this.emit("disconnect", err);
355
- this.logger.sendTelemetryEvent({
356
- eventName: "AfterDisconnectEvent",
357
- driverVersion,
358
- details: JSON.stringify({
359
- socketConnected: this.socket.connected,
360
- disconnectListenerCount: this.listenerCount("disconnect"),
361
- }),
362
- });
363
- // user of DeltaConnection should have processed "disconnect" event and removed all listeners. Not clear
364
- // if we want to enforce that, as some users (like LocalDocumentService) do not unregister any handlers
365
- // assert(this.listenerCount("disconnect") === 0, "'disconnect` events should be processed synchronously");
366
-
367
- this.removeTrackedListeners();
368
- this.disconnectCore();
369
- }
370
-
371
- /**
372
- * Disconnect from the websocket.
373
- * @param reason - reason for disconnect
374
- */
375
- protected disconnectCore() {
376
- this.socket.disconnect();
377
- }
378
-
379
- protected async initialize(connectMessage: IConnect, timeout: number) {
380
- this.socket.on("op", this.earlyOpHandler);
381
- this.socket.on("signal", this.earlySignalHandler);
382
- this.earlyOpHandlerAttached = true;
383
-
384
- // Socket.io's reconnect_attempt event is unreliable, so we track connect_error count instead.
385
- let internalSocketConnectionFailureCount: number = 0;
386
- const isInternalSocketReconnectionEnabled = (): boolean => this.socket.io.reconnection();
387
- const getMaxInternalSocketReconnectionAttempts = (): number => isInternalSocketReconnectionEnabled()
388
- ? this.socket.io.reconnectionAttempts()
389
- : 0;
390
- const getMaxAllowedInternalSocketConnectionFailures = (): number =>
391
- getMaxInternalSocketReconnectionAttempts() + 1;
392
-
393
- this._details = await new Promise<IConnected>((resolve, reject) => {
394
- const failAndCloseSocket = (err: IAnyDriverError) => {
395
- this.closeSocket(err);
396
- reject(err);
397
- };
398
-
399
- const failConnection = (err: IAnyDriverError) => {
400
- this.disconnect(err);
401
- reject(err);
402
- };
403
- // Listen for connection issues
404
- this.addConnectionListener("connect_error", (error) => {
405
- internalSocketConnectionFailureCount++;
406
- let isWebSocketTransportError = false;
407
- try {
408
- const description = error?.description;
409
- const context = error?.context;
410
-
411
- if (context && typeof context === "object") {
412
- const statusText = context.statusText?.code;
413
-
414
- // Self-Signed Certificate ErrorCode Found in error.context
415
- if (statusText === "DEPTH_ZERO_SELF_SIGNED_CERT") {
416
- failAndCloseSocket(this.createErrorObject("connect_error", error, false));
417
- return;
418
- }
419
- }
420
- else if (description && typeof description === "object") {
421
- const errorCode = description.error?.code;
422
-
423
- // Self-Signed Certificate ErrorCode Found in error.description
424
- if (errorCode === "DEPTH_ZERO_SELF_SIGNED_CERT") {
425
- failAndCloseSocket(this.createErrorObject("connect_error", error, false));
426
- return;
427
- }
428
-
429
- if (error.type === "TransportError") {
430
- isWebSocketTransportError = true;
431
- }
432
-
433
- // That's a WebSocket. Clear it as we can't log it.
434
- description.target = undefined;
435
- }
436
- } catch (_e) { }
437
-
438
- // Handle socket transport downgrading when not offline.
439
- if (
440
- isWebSocketTransportError &&
441
- this.enableLongPollingDowngrades &&
442
- this.socket.io.opts.transports?.[0] !== "polling") {
443
- // Downgrade transports to polling upgrade mechanism.
444
- this.socket.io.opts.transports = ["polling", "websocket"];
445
- // Don't alter reconnection behavior if already enabled.
446
- if (!isInternalSocketReconnectionEnabled()) {
447
- // Allow single reconnection attempt using polling upgrade mechanism.
448
- this.socket.io.reconnection(true);
449
- this.socket.io.reconnectionAttempts(1);
450
- }
451
- }
452
-
453
- // Allow built-in socket.io reconnection handling.
454
- if (isInternalSocketReconnectionEnabled() &&
455
- internalSocketConnectionFailureCount < getMaxAllowedInternalSocketConnectionFailures()) {
456
- // Reconnection is enabled and maximum reconnect attempts have not been reached.
457
- return;
458
- }
459
-
460
- failAndCloseSocket(this.createErrorObject("connect_error", error));
461
- });
462
-
463
- // Listen for timeouts
464
- this.addConnectionListener("connect_timeout", () => {
465
- failAndCloseSocket(this.createErrorObject("connect_timeout"));
466
- });
467
-
468
- this.addConnectionListener("connect_document_success", (response: IConnected) => {
469
- // If we sent a nonce and the server supports nonces, check that the nonces match
470
- if (connectMessage.nonce !== undefined &&
471
- response.nonce !== undefined &&
472
- response.nonce !== connectMessage.nonce) {
473
- return;
474
- }
475
-
476
- const requestedMode = connectMessage.mode;
477
- const actualMode = response.mode;
478
- const writingPermitted = response.claims.scopes.includes(ScopeType.DocWrite);
479
-
480
- if (writingPermitted) {
481
- // The only time we expect a mismatch in requested/actual is if we lack write permissions
482
- // In this case we will get "read", even if we requested "write"
483
- if (actualMode !== requestedMode) {
484
- failConnection(this.createErrorObject(
485
- "connect_document_success",
486
- "Connected in a different mode than was requested",
487
- false,
488
- ));
489
- return;
490
- }
491
- } else {
492
- if (actualMode === "write") {
493
- failConnection(this.createErrorObject(
494
- "connect_document_success",
495
- "Connected in write mode without write permissions",
496
- false,
497
- ));
498
- return;
499
- }
500
- }
501
-
502
- this.checkpointSequenceNumber = response.checkpointSequenceNumber;
503
-
504
- this.removeConnectionListeners();
505
- resolve(response);
506
- });
507
-
508
- // Socket can be disconnected while waiting for Fluid protocol messages
509
- // (connect_document_error / connect_document_success), as well as before DeltaManager
510
- // had a chance to register its handlers.
511
- this.addTrackedListener("disconnect", (reason) => {
512
- const err = this.createErrorObject("disconnect", reason);
513
- failAndCloseSocket(err);
514
- });
515
-
516
- this.addTrackedListener("error", ((error) => {
517
- // This includes "Invalid namespace" error, which we consider critical (reconnecting will not help)
518
- const err = this.createErrorObject("error", error, error !== "Invalid namespace");
519
- // Disconnect socket - required if happened before initial handshake
520
- failAndCloseSocket(err);
521
- }));
522
-
523
- this.addConnectionListener("connect_document_error", ((error) => {
524
- // If we sent a nonce and the server supports nonces, check that the nonces match
525
- if (connectMessage.nonce !== undefined &&
526
- error.nonce !== undefined &&
527
- error.nonce !== connectMessage.nonce) {
528
- return;
529
- }
530
-
531
- // This is not an socket.io error - it's Fluid protocol error.
532
- // In this case fail connection and indicate that we were unable to create connection
533
- failConnection(this.createErrorObject("connect_document_error", error));
534
- }));
535
-
536
- this.socket.emit("connect_document", connectMessage);
537
-
538
- // Give extra 2 seconds for handshake on top of socket connection timeout
539
- this.socketConnectionTimeout = setTimeout(() => {
540
- failConnection(this.createErrorObject("orderingServiceHandshakeTimeout"));
541
- }, timeout + 2000);
542
- });
543
-
544
- assert(!this.disposed, 0x246 /* "checking consistency of socket & _disposed flags" */);
545
- }
546
-
547
- protected earlyOpHandler = (documentId: string, msgs: ISequencedDocumentMessage[]) => {
548
- this.queuedMessages.push(...msgs);
549
- };
550
-
551
- protected earlySignalHandler = (msg: ISignalMessage) => {
552
- this.queuedSignals.push(msg);
553
- };
554
-
555
- private removeEarlyOpHandler() {
556
- this.socket.removeListener("op", this.earlyOpHandler);
557
- this.earlyOpHandlerAttached = false;
558
- }
559
-
560
- private removeEarlySignalHandler() {
561
- this.socket.removeListener("signal", this.earlySignalHandler);
562
- }
563
-
564
- private addConnectionListener(event: string, listener: (...args: any[]) => void) {
565
- assert(!DocumentDeltaConnection.eventsAlwaysForwarded.includes(event),
566
- 0x247 /* "Use addTrackedListener instead" */);
567
- assert(!DocumentDeltaConnection.eventsToForward.includes(event),
568
- 0x248 /* "should not subscribe to forwarded events" */);
569
- this.socket.on(event, listener);
570
- assert(!this.connectionListeners.has(event), 0x20d /* "double connection listener" */);
571
- this.connectionListeners.set(event, listener);
572
- }
573
-
574
- protected addTrackedListener(event: string, listener: (...args: any[]) => void) {
575
- this.socket.on(event, listener);
576
- assert(!this.trackedListeners.has(event), 0x20e /* "double tracked listener" */);
577
- this.trackedListeners.set(event, listener);
578
- }
579
-
580
- private removeTrackedListeners() {
581
- for (const [event, listener] of this.trackedListeners.entries()) {
582
- this.socket.off(event, listener);
583
- }
584
- // removeTrackedListeners removes all listeners, including connection listeners
585
- this.removeConnectionListeners();
586
-
587
- this.removeEarlyOpHandler();
588
- this.removeEarlySignalHandler();
589
-
590
- this.trackedListeners.clear();
591
- }
592
-
593
- private removeConnectionListeners() {
594
- if (this.socketConnectionTimeout !== undefined) {
595
- clearTimeout(this.socketConnectionTimeout);
596
- }
597
-
598
- for (const [event, listener] of this.connectionListeners.entries()) {
599
- this.socket.off(event, listener);
600
- }
601
- this.connectionListeners.clear();
602
- }
603
-
604
- /**
605
- * Error raising for socket.io issues
606
- */
607
- protected createErrorObject(handler: string, error?: any, canRetry = true): IAnyDriverError {
608
- // Note: we suspect the incoming error object is either:
609
- // - a string: log it in the message (if not a string, it may contain PII but will print as [object Object])
610
- // - an Error object thrown by socket.io engine. Be careful with not recording PII!
611
- let message: string;
612
- if (error?.type === "TransportError") {
613
- // JSON.stringify drops Error.message
614
- const messagePrefix = (error?.message !== undefined)
615
- ? `${error.message}: `
616
- : "";
617
-
618
- // Websocket errors reported by engine.io-client.
619
- // They are Error objects with description containing WS error and description = "TransportError"
620
- // Please see https://github.com/socketio/engine.io-client/blob/7245b80/lib/transport.ts#L44,
621
- message = `${messagePrefix}${JSON.stringify(error, getCircularReplacer())}`;
622
- } else {
623
- message = extractLogSafeErrorProperties(error, true).message;
624
- }
625
-
626
- const errorObj = createGenericNetworkError(
627
- `socket.io (${handler}): ${message}`,
628
- { canRetry },
629
- { driverVersion },
630
- );
631
-
632
- return errorObj;
633
- }
42
+ extends EventEmitterWithErrorHandling<IDocumentDeltaConnectionEvents>
43
+ implements IDocumentDeltaConnection, IDisposable
44
+ {
45
+ static readonly eventsToForward = ["nack", "op", "signal", "pong"];
46
+
47
+ // WARNING: These are critical events that we can't miss, so registration for them has to be in place at all times!
48
+ // Including before handshake is over, and after that (but before DeltaManager had a chance to put its own handlers)
49
+ static readonly eventsAlwaysForwarded = ["disconnect", "error"];
50
+
51
+ /**
52
+ * Last known sequence number to ordering service at the time of connection
53
+ * It may lap actual last sequence number (quite a bit, if container is very active).
54
+ * But it's best information for client to figure out how far it is behind, at least
55
+ * for "read" connections. "write" connections may use own "join" op to similar information,
56
+ * that is likely to be more up-to-date.
57
+ */
58
+ public checkpointSequenceNumber: number | undefined;
59
+
60
+ // Listen for ops sent before we receive a response to connect_document
61
+ protected readonly queuedMessages: ISequencedDocumentMessage[] = [];
62
+ protected readonly queuedSignals: ISignalMessage[] = [];
63
+
64
+ /**
65
+ * A flag to indicate whether we have our handler attached. If it's attached, we're queueing incoming ops
66
+ * to later be retrieved via initialMessages.
67
+ */
68
+ private earlyOpHandlerAttached: boolean = false;
69
+
70
+ private socketConnectionTimeout: ReturnType<typeof setTimeout> | undefined;
71
+
72
+ private _details: IConnected | undefined;
73
+
74
+ // Listeners only needed while the connection is in progress
75
+ private readonly connectionListeners: Map<string, (...args: any[]) => void> = new Map();
76
+ // Listeners used throughout the lifetime of the DocumentDeltaConnection
77
+ private readonly trackedListeners: Map<string, (...args: any[]) => void> = new Map();
78
+
79
+ protected get hasDetails(): boolean {
80
+ return !!this._details;
81
+ }
82
+
83
+ public get disposed() {
84
+ assert(
85
+ this._disposed || this.socket.connected,
86
+ 0x244 /* "Socket is closed, but connection is not!" */,
87
+ );
88
+ return this._disposed;
89
+ }
90
+
91
+ /**
92
+ * Flag to indicate whether the DocumentDeltaConnection is expected to still be capable of sending messages.
93
+ * After disconnection, we flip this to prevent any stale messages from being emitted.
94
+ */
95
+ protected _disposed: boolean = false;
96
+ private readonly mc: MonitoringContext;
97
+
98
+ /**
99
+ * @deprecated Implementors should manage their own logger or monitoring context
100
+ */
101
+ protected get logger(): ITelemetryLogger {
102
+ return this.mc.logger;
103
+ }
104
+
105
+ public get details(): IConnected {
106
+ if (!this._details) {
107
+ throw new Error("Internal error: calling method before _details is initialized!");
108
+ }
109
+ return this._details;
110
+ }
111
+
112
+ /**
113
+ * @param socket - websocket to be used
114
+ * @param documentId - ID of the document
115
+ * @param logger - for reporting telemetry events
116
+ * @param enableLongPollingDowngrades - allow connection to be downgraded to long-polling on websocket failure
117
+ */
118
+ protected constructor(
119
+ protected readonly socket: Socket,
120
+ public documentId: string,
121
+ logger: ITelemetryLogger,
122
+ private readonly enableLongPollingDowngrades: boolean = false,
123
+ ) {
124
+ super((name, error) => {
125
+ logger.sendErrorEvent(
126
+ {
127
+ eventName: "DeltaConnection:EventException",
128
+ name,
129
+ },
130
+ error,
131
+ );
132
+ });
133
+
134
+ this.mc = loggerToMonitoringContext(ChildLogger.create(logger, "DeltaConnection"));
135
+
136
+ this.on("newListener", (event, listener) => {
137
+ assert(!this.disposed, 0x20a /* "register for event on disposed object" */);
138
+
139
+ // Some events are already forwarded - see this.addTrackedListener() calls in initialize().
140
+ if (DocumentDeltaConnection.eventsAlwaysForwarded.includes(event)) {
141
+ assert(this.trackedListeners.has(event), 0x245 /* "tracked listener" */);
142
+ return;
143
+ }
144
+
145
+ if (!DocumentDeltaConnection.eventsToForward.includes(event)) {
146
+ throw new Error(`DocumentDeltaConnection: Registering for unknown event: ${event}`);
147
+ }
148
+
149
+ // Whenever listener is added, we should subscribe on same event on socket, so these two things
150
+ // should be in sync. This currently assumes that nobody unregisters and registers back listeners,
151
+ // and that there are no "internal" listeners installed (like "error" case we skip above)
152
+ // Better flow might be to always unconditionally register all handlers on successful connection,
153
+ // though some logic (naming assert in initialMessages getter) might need to be adjusted (it becomes noop)
154
+ assert(
155
+ (this.listeners(event).length !== 0) === this.trackedListeners.has(event),
156
+ 0x20b /* "mismatch" */,
157
+ );
158
+ if (!this.trackedListeners.has(event)) {
159
+ this.addTrackedListener(event, (...args: any[]) => {
160
+ this.emit(event, ...args);
161
+ });
162
+ }
163
+ });
164
+ }
165
+
166
+ /**
167
+ * Get the ID of the client who is sending the message
168
+ *
169
+ * @returns the client ID
170
+ */
171
+ public get clientId(): string {
172
+ return this.details.clientId;
173
+ }
174
+
175
+ /**
176
+ * Get the mode of the client
177
+ *
178
+ * @returns the client mode
179
+ */
180
+ public get mode(): ConnectionMode {
181
+ return this.details.mode;
182
+ }
183
+
184
+ /**
185
+ * Get the claims of the client who is sending the message
186
+ *
187
+ * @returns client claims
188
+ */
189
+ public get claims(): ITokenClaims {
190
+ return this.details.claims;
191
+ }
192
+
193
+ /**
194
+ * Get whether or not this is an existing document
195
+ *
196
+ * @returns true if the document exists
197
+ */
198
+ public get existing(): boolean {
199
+ return this.details.existing;
200
+ }
201
+
202
+ /**
203
+ * Get the maximum size of a message before chunking is required
204
+ *
205
+ * @returns the maximum size of a message before chunking is required
206
+ */
207
+ public get maxMessageSize(): number {
208
+ return this.details.serviceConfiguration.maxMessageSize;
209
+ }
210
+
211
+ /**
212
+ * Semver of protocol being used with the service
213
+ */
214
+ public get version(): string {
215
+ return this.details.version;
216
+ }
217
+
218
+ /**
219
+ * Configuration details provided by the service
220
+ */
221
+ public get serviceConfiguration(): IClientConfiguration {
222
+ return this.details.serviceConfiguration;
223
+ }
224
+
225
+ private checkNotClosed() {
226
+ assert(!this.disposed, 0x20c /* "connection disposed" */);
227
+ }
228
+
229
+ /**
230
+ * Get messages sent during the connection
231
+ *
232
+ * @returns messages sent during the connection
233
+ */
234
+ public get initialMessages(): ISequencedDocumentMessage[] {
235
+ this.checkNotClosed();
236
+
237
+ // If we call this when the earlyOpHandler is not attached, then the queuedMessages may not include the
238
+ // latest ops. This could possibly indicate that initialMessages was called twice.
239
+ assert(this.earlyOpHandlerAttached, 0x08e /* "Potentially missed initial messages" */);
240
+ // We will lose ops and perf will tank as we need to go to storage to become current!
241
+ assert(this.listeners("op").length !== 0, 0x08f /* "No op handler is setup!" */);
242
+
243
+ this.removeEarlyOpHandler();
244
+
245
+ if (this.queuedMessages.length > 0) {
246
+ // Some messages were queued.
247
+ // add them to the list of initialMessages to be processed
248
+ this.details.initialMessages.push(...this.queuedMessages);
249
+ this.details.initialMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
250
+ this.queuedMessages.length = 0;
251
+ }
252
+ return this.details.initialMessages;
253
+ }
254
+
255
+ /**
256
+ * Get signals sent during the connection
257
+ *
258
+ * @returns signals sent during the connection
259
+ */
260
+ public get initialSignals(): ISignalMessage[] {
261
+ this.checkNotClosed();
262
+ assert(this.listeners("signal").length !== 0, 0x090 /* "No signal handler is setup!" */);
263
+
264
+ this.removeEarlySignalHandler();
265
+
266
+ if (this.queuedSignals.length > 0) {
267
+ // Some signals were queued.
268
+ // add them to the list of initialSignals to be processed
269
+ this.details.initialSignals.push(...this.queuedSignals);
270
+ this.queuedSignals.length = 0;
271
+ }
272
+ return this.details.initialSignals;
273
+ }
274
+
275
+ /**
276
+ * Get initial client list
277
+ *
278
+ * @returns initial client list sent during the connection
279
+ */
280
+ public get initialClients(): ISignalClient[] {
281
+ this.checkNotClosed();
282
+ return this.details.initialClients;
283
+ }
284
+
285
+ protected emitMessages(type: string, messages: IDocumentMessage[][]) {
286
+ // Although the implementation here disconnects the socket and does not reuse it, other subclasses
287
+ // (e.g. OdspDocumentDeltaConnection) may reuse the socket. In these cases, we need to avoid emitting
288
+ // on the still-live socket.
289
+ if (!this.disposed) {
290
+ this.socket.emit(type, this.clientId, messages);
291
+ }
292
+ }
293
+
294
+ protected submitCore(type: string, messages: IDocumentMessage[]) {
295
+ this.emitMessages(type, [messages]);
296
+ }
297
+
298
+ /**
299
+ * Submits a new delta operation to the server
300
+ *
301
+ * @param message - delta operation to submit
302
+ */
303
+ public submit(messages: IDocumentMessage[]): void {
304
+ this.checkNotClosed();
305
+ this.submitCore("submitOp", messages);
306
+ }
307
+
308
+ /**
309
+ * Submits a new signal to the server
310
+ *
311
+ * @param message - signal to submit
312
+ */
313
+ public submitSignal(message: IDocumentMessage): void {
314
+ this.checkNotClosed();
315
+ this.submitCore("submitSignal", [message]);
316
+ }
317
+
318
+ /**
319
+ * Disconnect from the websocket and close the websocket too.
320
+ */
321
+ private closeSocket(error: IAnyDriverError) {
322
+ if (this._disposed) {
323
+ // This would be rare situation due to complexity around socket emitting events.
324
+ this.logger.sendTelemetryEvent(
325
+ {
326
+ eventName: "SocketCloseOnDisposedConnection",
327
+ driverVersion,
328
+ details: JSON.stringify({
329
+ disposed: this._disposed,
330
+ socketConnected: this.socket?.connected,
331
+ trackedListenerCount: this.trackedListeners.size,
332
+ }),
333
+ },
334
+ error,
335
+ );
336
+ return;
337
+ }
338
+ this.closeSocketCore(error);
339
+ }
340
+
341
+ protected closeSocketCore(error: IAnyDriverError) {
342
+ this.disconnect(error);
343
+ }
344
+
345
+ /**
346
+ * Disconnect from the websocket, and permanently disable this DocumentDeltaConnection and close the socket.
347
+ * However the OdspDocumentDeltaConnection differ in dispose as in there we don't close the socket. There is no
348
+ * multiplexing here, so we need to close the socket here.
349
+ */
350
+ public dispose() {
351
+ this.logger.sendTelemetryEvent({
352
+ eventName: "ClientClosingDeltaConnection",
353
+ driverVersion,
354
+ details: JSON.stringify({
355
+ disposed: this._disposed,
356
+ socketConnected: this.socket.connected,
357
+ }),
358
+ });
359
+ this.disconnect(
360
+ createGenericNetworkError(
361
+ // pre-0.58 error message: clientClosingConnection
362
+ "Client closing delta connection",
363
+ { canRetry: true },
364
+ { driverVersion },
365
+ ),
366
+ );
367
+ }
368
+
369
+ protected disconnect(err: IAnyDriverError) {
370
+ // Can't check this.disposed here, as we get here on socket closure,
371
+ // so _disposed & socket.connected might be not in sync while processing
372
+ // "dispose" event.
373
+ if (this._disposed) {
374
+ return;
375
+ }
376
+
377
+ // We set the disposed flag as a part of the contract for overriding the disconnect method. This is used by
378
+ // DocumentDeltaConnection to determine if emitting messages (ops) on the socket is allowed, which is
379
+ // important since OdspDocumentDeltaConnection reuses the socket rather than truly disconnecting it. Note that
380
+ // OdspDocumentDeltaConnection may still send disconnect_document which is allowed; this is only intended
381
+ // to prevent normal messages from being emitted.
382
+ this._disposed = true;
383
+
384
+ // Let user of connection object know about disconnect. This has to happen in between setting _disposed and
385
+ // removing all listeners!
386
+ this.emit("disconnect", err);
387
+ this.logger.sendTelemetryEvent({
388
+ eventName: "AfterDisconnectEvent",
389
+ driverVersion,
390
+ details: JSON.stringify({
391
+ socketConnected: this.socket.connected,
392
+ disconnectListenerCount: this.listenerCount("disconnect"),
393
+ }),
394
+ });
395
+ // user of DeltaConnection should have processed "disconnect" event and removed all listeners. Not clear
396
+ // if we want to enforce that, as some users (like LocalDocumentService) do not unregister any handlers
397
+ // assert(this.listenerCount("disconnect") === 0, "'disconnect` events should be processed synchronously");
398
+
399
+ this.removeTrackedListeners();
400
+ this.disconnectCore();
401
+ }
402
+
403
+ /**
404
+ * Disconnect from the websocket.
405
+ * @param reason - reason for disconnect
406
+ */
407
+ protected disconnectCore() {
408
+ this.socket.disconnect();
409
+ }
410
+
411
+ protected async initialize(connectMessage: IConnect, timeout: number) {
412
+ this.socket.on("op", this.earlyOpHandler);
413
+ this.socket.on("signal", this.earlySignalHandler);
414
+ this.earlyOpHandlerAttached = true;
415
+
416
+ // Socket.io's reconnect_attempt event is unreliable, so we track connect_error count instead.
417
+ let internalSocketConnectionFailureCount: number = 0;
418
+ const isInternalSocketReconnectionEnabled = (): boolean => this.socket.io.reconnection();
419
+ const getMaxInternalSocketReconnectionAttempts = (): number =>
420
+ isInternalSocketReconnectionEnabled() ? this.socket.io.reconnectionAttempts() : 0;
421
+ const getMaxAllowedInternalSocketConnectionFailures = (): number =>
422
+ getMaxInternalSocketReconnectionAttempts() + 1;
423
+
424
+ this._details = await new Promise<IConnected>((resolve, reject) => {
425
+ const failAndCloseSocket = (err: IAnyDriverError) => {
426
+ this.closeSocket(err);
427
+ reject(err);
428
+ };
429
+
430
+ const failConnection = (err: IAnyDriverError) => {
431
+ this.disconnect(err);
432
+ reject(err);
433
+ };
434
+ // Listen for connection issues
435
+ this.addConnectionListener("connect_error", (error) => {
436
+ internalSocketConnectionFailureCount++;
437
+ let isWebSocketTransportError = false;
438
+ try {
439
+ const description = error?.description;
440
+ const context = error?.context;
441
+
442
+ if (context && typeof context === "object") {
443
+ const statusText = context.statusText?.code;
444
+
445
+ // Self-Signed Certificate ErrorCode Found in error.context
446
+ if (statusText === "DEPTH_ZERO_SELF_SIGNED_CERT") {
447
+ failAndCloseSocket(
448
+ this.createErrorObject("connect_error", error, false),
449
+ );
450
+ return;
451
+ }
452
+ } else if (description && typeof description === "object") {
453
+ const errorCode = description.error?.code;
454
+
455
+ // Self-Signed Certificate ErrorCode Found in error.description
456
+ if (errorCode === "DEPTH_ZERO_SELF_SIGNED_CERT") {
457
+ failAndCloseSocket(
458
+ this.createErrorObject("connect_error", error, false),
459
+ );
460
+ return;
461
+ }
462
+
463
+ if (error.type === "TransportError") {
464
+ isWebSocketTransportError = true;
465
+ }
466
+
467
+ // That's a WebSocket. Clear it as we can't log it.
468
+ description.target = undefined;
469
+ }
470
+ } catch (_e) {}
471
+
472
+ // Handle socket transport downgrading when not offline.
473
+ if (
474
+ isWebSocketTransportError &&
475
+ this.enableLongPollingDowngrades &&
476
+ this.socket.io.opts.transports?.[0] !== "polling"
477
+ ) {
478
+ // Downgrade transports to polling upgrade mechanism.
479
+ this.socket.io.opts.transports = ["polling", "websocket"];
480
+ // Don't alter reconnection behavior if already enabled.
481
+ if (!isInternalSocketReconnectionEnabled()) {
482
+ // Allow single reconnection attempt using polling upgrade mechanism.
483
+ this.socket.io.reconnection(true);
484
+ this.socket.io.reconnectionAttempts(1);
485
+ }
486
+ }
487
+
488
+ // Allow built-in socket.io reconnection handling.
489
+ if (
490
+ isInternalSocketReconnectionEnabled() &&
491
+ internalSocketConnectionFailureCount <
492
+ getMaxAllowedInternalSocketConnectionFailures()
493
+ ) {
494
+ // Reconnection is enabled and maximum reconnect attempts have not been reached.
495
+ return;
496
+ }
497
+
498
+ failAndCloseSocket(this.createErrorObject("connect_error", error));
499
+ });
500
+
501
+ // Listen for timeouts
502
+ this.addConnectionListener("connect_timeout", () => {
503
+ failAndCloseSocket(this.createErrorObject("connect_timeout"));
504
+ });
505
+
506
+ this.addConnectionListener("connect_document_success", (response: IConnected) => {
507
+ // If we sent a nonce and the server supports nonces, check that the nonces match
508
+ if (
509
+ connectMessage.nonce !== undefined &&
510
+ response.nonce !== undefined &&
511
+ response.nonce !== connectMessage.nonce
512
+ ) {
513
+ return;
514
+ }
515
+
516
+ const requestedMode = connectMessage.mode;
517
+ const actualMode = response.mode;
518
+ const writingPermitted = response.claims.scopes.includes(ScopeType.DocWrite);
519
+
520
+ if (writingPermitted) {
521
+ // The only time we expect a mismatch in requested/actual is if we lack write permissions
522
+ // In this case we will get "read", even if we requested "write"
523
+ if (actualMode !== requestedMode) {
524
+ failConnection(
525
+ this.createErrorObject(
526
+ "connect_document_success",
527
+ "Connected in a different mode than was requested",
528
+ false,
529
+ ),
530
+ );
531
+ return;
532
+ }
533
+ } else {
534
+ if (actualMode === "write") {
535
+ failConnection(
536
+ this.createErrorObject(
537
+ "connect_document_success",
538
+ "Connected in write mode without write permissions",
539
+ false,
540
+ ),
541
+ );
542
+ return;
543
+ }
544
+ }
545
+
546
+ this.checkpointSequenceNumber = response.checkpointSequenceNumber;
547
+
548
+ this.removeConnectionListeners();
549
+ resolve(response);
550
+ });
551
+
552
+ // Socket can be disconnected while waiting for Fluid protocol messages
553
+ // (connect_document_error / connect_document_success), as well as before DeltaManager
554
+ // had a chance to register its handlers.
555
+ this.addTrackedListener("disconnect", (reason) => {
556
+ const err = this.createErrorObject("disconnect", reason);
557
+ failAndCloseSocket(err);
558
+ });
559
+
560
+ this.addTrackedListener("error", (error) => {
561
+ // This includes "Invalid namespace" error, which we consider critical (reconnecting will not help)
562
+ const err = this.createErrorObject("error", error, error !== "Invalid namespace");
563
+ // Disconnect socket - required if happened before initial handshake
564
+ failAndCloseSocket(err);
565
+ });
566
+
567
+ this.addConnectionListener("connect_document_error", (error) => {
568
+ // If we sent a nonce and the server supports nonces, check that the nonces match
569
+ if (
570
+ connectMessage.nonce !== undefined &&
571
+ error.nonce !== undefined &&
572
+ error.nonce !== connectMessage.nonce
573
+ ) {
574
+ return;
575
+ }
576
+
577
+ // This is not an socket.io error - it's Fluid protocol error.
578
+ // In this case fail connection and indicate that we were unable to create connection
579
+ failConnection(this.createErrorObject("connect_document_error", error));
580
+ });
581
+
582
+ this.socket.emit("connect_document", connectMessage);
583
+
584
+ // Give extra 2 seconds for handshake on top of socket connection timeout
585
+ this.socketConnectionTimeout = setTimeout(() => {
586
+ failConnection(this.createErrorObject("orderingServiceHandshakeTimeout"));
587
+ }, timeout + 2000);
588
+ });
589
+
590
+ assert(!this.disposed, 0x246 /* "checking consistency of socket & _disposed flags" */);
591
+ }
592
+
593
+ protected earlyOpHandler = (documentId: string, msgs: ISequencedDocumentMessage[]) => {
594
+ this.queuedMessages.push(...msgs);
595
+ };
596
+
597
+ protected earlySignalHandler = (msg: ISignalMessage) => {
598
+ this.queuedSignals.push(msg);
599
+ };
600
+
601
+ private removeEarlyOpHandler() {
602
+ this.socket.removeListener("op", this.earlyOpHandler);
603
+ this.earlyOpHandlerAttached = false;
604
+ }
605
+
606
+ private removeEarlySignalHandler() {
607
+ this.socket.removeListener("signal", this.earlySignalHandler);
608
+ }
609
+
610
+ private addConnectionListener(event: string, listener: (...args: any[]) => void) {
611
+ assert(
612
+ !DocumentDeltaConnection.eventsAlwaysForwarded.includes(event),
613
+ 0x247 /* "Use addTrackedListener instead" */,
614
+ );
615
+ assert(
616
+ !DocumentDeltaConnection.eventsToForward.includes(event),
617
+ 0x248 /* "should not subscribe to forwarded events" */,
618
+ );
619
+ this.socket.on(event, listener);
620
+ assert(!this.connectionListeners.has(event), 0x20d /* "double connection listener" */);
621
+ this.connectionListeners.set(event, listener);
622
+ }
623
+
624
+ protected addTrackedListener(event: string, listener: (...args: any[]) => void) {
625
+ this.socket.on(event, listener);
626
+ assert(!this.trackedListeners.has(event), 0x20e /* "double tracked listener" */);
627
+ this.trackedListeners.set(event, listener);
628
+ }
629
+
630
+ private removeTrackedListeners() {
631
+ for (const [event, listener] of this.trackedListeners.entries()) {
632
+ this.socket.off(event, listener);
633
+ }
634
+ // removeTrackedListeners removes all listeners, including connection listeners
635
+ this.removeConnectionListeners();
636
+
637
+ this.removeEarlyOpHandler();
638
+ this.removeEarlySignalHandler();
639
+
640
+ this.trackedListeners.clear();
641
+ }
642
+
643
+ private removeConnectionListeners() {
644
+ if (this.socketConnectionTimeout !== undefined) {
645
+ clearTimeout(this.socketConnectionTimeout);
646
+ }
647
+
648
+ for (const [event, listener] of this.connectionListeners.entries()) {
649
+ this.socket.off(event, listener);
650
+ }
651
+ this.connectionListeners.clear();
652
+ }
653
+
654
+ /**
655
+ * Error raising for socket.io issues
656
+ */
657
+ protected createErrorObject(handler: string, error?: any, canRetry = true): IAnyDriverError {
658
+ // Note: we suspect the incoming error object is either:
659
+ // - a string: log it in the message (if not a string, it may contain PII but will print as [object Object])
660
+ // - an Error object thrown by socket.io engine. Be careful with not recording PII!
661
+ let message: string;
662
+ if (error?.type === "TransportError") {
663
+ // JSON.stringify drops Error.message
664
+ const messagePrefix = error?.message !== undefined ? `${error.message}: ` : "";
665
+
666
+ // Websocket errors reported by engine.io-client.
667
+ // They are Error objects with description containing WS error and description = "TransportError"
668
+ // Please see https://github.com/socketio/engine.io-client/blob/7245b80/lib/transport.ts#L44,
669
+ message = `${messagePrefix}${JSON.stringify(error, getCircularReplacer())}`;
670
+ } else {
671
+ message = extractLogSafeErrorProperties(error, true).message;
672
+ }
673
+
674
+ const errorObj = createGenericNetworkError(
675
+ `socket.io (${handler}): ${message}`,
676
+ { canRetry },
677
+ { driverVersion },
678
+ );
679
+
680
+ return errorObj;
681
+ }
634
682
  }