@fluidframework/container-loader 0.53.0-46105 → 0.54.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/connectionManager.d.ts +153 -0
- package/dist/connectionManager.d.ts.map +1 -0
- package/dist/connectionManager.js +664 -0
- package/dist/connectionManager.js.map +1 -0
- package/dist/container.d.ts +2 -1
- package/dist/container.d.ts.map +1 -1
- package/dist/container.js +25 -28
- package/dist/container.js.map +1 -1
- package/dist/contracts.d.ts +112 -0
- package/dist/contracts.d.ts.map +1 -0
- package/dist/contracts.js +14 -0
- package/dist/contracts.js.map +1 -0
- package/dist/deltaManager.d.ts +26 -135
- package/dist/deltaManager.d.ts.map +1 -1
- package/dist/deltaManager.js +142 -768
- package/dist/deltaManager.js.map +1 -1
- package/dist/loader.d.ts +9 -4
- package/dist/loader.d.ts.map +1 -1
- package/dist/loader.js +5 -4
- package/dist/loader.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/protocolTreeDocumentStorageService.d.ts +2 -2
- package/dist/protocolTreeDocumentStorageService.d.ts.map +1 -1
- package/lib/connectionManager.d.ts +153 -0
- package/lib/connectionManager.d.ts.map +1 -0
- package/lib/connectionManager.js +660 -0
- package/lib/connectionManager.js.map +1 -0
- package/lib/container.d.ts +2 -1
- package/lib/container.d.ts.map +1 -1
- package/lib/container.js +25 -28
- package/lib/container.js.map +1 -1
- package/lib/contracts.d.ts +112 -0
- package/lib/contracts.d.ts.map +1 -0
- package/lib/contracts.js +11 -0
- package/lib/contracts.js.map +1 -0
- package/lib/deltaManager.d.ts +26 -135
- package/lib/deltaManager.d.ts.map +1 -1
- package/lib/deltaManager.js +146 -772
- package/lib/deltaManager.js.map +1 -1
- package/lib/loader.d.ts +9 -4
- package/lib/loader.d.ts.map +1 -1
- package/lib/loader.js +6 -5
- package/lib/loader.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/protocolTreeDocumentStorageService.d.ts +2 -2
- package/lib/protocolTreeDocumentStorageService.d.ts.map +1 -1
- package/package.json +9 -9
- package/src/connectionManager.ts +892 -0
- package/src/container.ts +39 -39
- package/src/contracts.ts +156 -0
- package/src/deltaManager.ts +181 -979
- package/src/loader.ts +31 -9
- package/src/packageVersion.ts +1 -1
package/dist/deltaManager.js
CHANGED
|
@@ -7,7 +7,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
|
7
7
|
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
8
8
|
};
|
|
9
9
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
10
|
-
exports.DeltaManager =
|
|
10
|
+
exports.DeltaManager = void 0;
|
|
11
11
|
const abort_controller_1 = __importDefault(require("abort-controller"));
|
|
12
12
|
const uuid_1 = require("uuid");
|
|
13
13
|
const common_utils_1 = require("@fluidframework/common-utils");
|
|
@@ -18,79 +18,16 @@ const protocol_definitions_1 = require("@fluidframework/protocol-definitions");
|
|
|
18
18
|
const driver_utils_1 = require("@fluidframework/driver-utils");
|
|
19
19
|
const container_utils_1 = require("@fluidframework/container-utils");
|
|
20
20
|
const deltaQueue_1 = require("./deltaQueue");
|
|
21
|
-
const MaxReconnectDelayInMs = 8000;
|
|
22
|
-
const InitialReconnectDelayInMs = 1000;
|
|
23
|
-
const DefaultChunkSize = 16 * 1024;
|
|
24
|
-
function getNackReconnectInfo(nackContent) {
|
|
25
|
-
const message = `Nack (${nackContent.type}): ${nackContent.message}`;
|
|
26
|
-
const canRetry = nackContent.code !== 403;
|
|
27
|
-
const retryAfterMs = nackContent.retryAfter !== undefined ? nackContent.retryAfter * 1000 : undefined;
|
|
28
|
-
return driver_utils_1.createGenericNetworkError(`nack [${nackContent.code}]`, message, canRetry, retryAfterMs, { statusCode: nackContent.code });
|
|
29
|
-
}
|
|
30
|
-
const createReconnectError = (fluidErrorCode, err) => telemetry_utils_1.wrapError(err, (errorMessage) => new driver_utils_1.GenericNetworkError(fluidErrorCode, errorMessage, true /* canRetry */));
|
|
31
|
-
const fatalConnectErrorProp = { fatalConnectError: true };
|
|
32
|
-
var ReconnectMode;
|
|
33
|
-
(function (ReconnectMode) {
|
|
34
|
-
ReconnectMode["Never"] = "Never";
|
|
35
|
-
ReconnectMode["Disabled"] = "Disabled";
|
|
36
|
-
ReconnectMode["Enabled"] = "Enabled";
|
|
37
|
-
})(ReconnectMode = exports.ReconnectMode || (exports.ReconnectMode = {}));
|
|
38
|
-
/**
|
|
39
|
-
* Implementation of IDocumentDeltaConnection that does not support submitting
|
|
40
|
-
* or receiving ops. Used in storage-only mode.
|
|
41
|
-
*/
|
|
42
|
-
class NoDeltaStream extends common_utils_1.TypedEventEmitter {
|
|
43
|
-
constructor() {
|
|
44
|
-
super(...arguments);
|
|
45
|
-
this.clientId = "storage-only client";
|
|
46
|
-
this.claims = {
|
|
47
|
-
scopes: [protocol_definitions_1.ScopeType.DocRead],
|
|
48
|
-
};
|
|
49
|
-
this.mode = "read";
|
|
50
|
-
this.existing = true;
|
|
51
|
-
this.maxMessageSize = 0;
|
|
52
|
-
this.version = "";
|
|
53
|
-
this.initialMessages = [];
|
|
54
|
-
this.initialSignals = [];
|
|
55
|
-
this.initialClients = [];
|
|
56
|
-
this.serviceConfiguration = {
|
|
57
|
-
maxMessageSize: 0,
|
|
58
|
-
blockSize: 0,
|
|
59
|
-
summary: undefined,
|
|
60
|
-
};
|
|
61
|
-
this.checkpointSequenceNumber = undefined;
|
|
62
|
-
this._disposed = false;
|
|
63
|
-
}
|
|
64
|
-
submit(messages) {
|
|
65
|
-
this.emit("nack", this.clientId, messages.map((operation) => {
|
|
66
|
-
return {
|
|
67
|
-
operation,
|
|
68
|
-
content: { message: "Cannot submit with storage-only connection", code: 403 },
|
|
69
|
-
};
|
|
70
|
-
}));
|
|
71
|
-
}
|
|
72
|
-
submitSignal(message) {
|
|
73
|
-
this.emit("nack", this.clientId, {
|
|
74
|
-
operation: message,
|
|
75
|
-
content: { message: "Cannot submit signal with storage-only connection", code: 403 },
|
|
76
|
-
});
|
|
77
|
-
}
|
|
78
|
-
get disposed() { return this._disposed; }
|
|
79
|
-
dispose() { this._disposed = true; }
|
|
80
|
-
}
|
|
81
21
|
/**
|
|
82
22
|
* Manages the flow of both inbound and outbound messages. This class ensures that shared objects receive delta
|
|
83
23
|
* messages in order regardless of possible network conditions or timings causing out of order delivery.
|
|
84
24
|
*/
|
|
85
25
|
class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
86
|
-
constructor(serviceProvider,
|
|
26
|
+
constructor(serviceProvider, logger, _active, createConnectionManager) {
|
|
87
27
|
super();
|
|
88
28
|
this.serviceProvider = serviceProvider;
|
|
89
|
-
this.client = client;
|
|
90
29
|
this.logger = logger;
|
|
91
30
|
this._active = _active;
|
|
92
|
-
// tracks host requiring read-only mode.
|
|
93
|
-
this._forceReadonly = false;
|
|
94
31
|
this.pending = [];
|
|
95
32
|
// The minimum sequence number and last sequence number received from the server
|
|
96
33
|
this.minSequenceNumber = 0;
|
|
@@ -107,82 +44,30 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
107
44
|
this.baseTerm = 0;
|
|
108
45
|
// The sequence number we initially loaded from
|
|
109
46
|
this.initSequenceNumber = 0;
|
|
110
|
-
this.clientSequenceNumber = 0;
|
|
111
|
-
this.clientSequenceNumberObserved = 0;
|
|
112
|
-
// Counts the number of noops sent by the client which may not be acked.
|
|
113
|
-
this.trailingNoopCount = 0;
|
|
114
47
|
this.closed = false;
|
|
115
|
-
this.deltaStreamDelayId = uuid_1.v4();
|
|
116
|
-
this.deltaStorageDelayId = uuid_1.v4();
|
|
117
|
-
this.messageBuffer = [];
|
|
118
|
-
this.connectFirstConnection = true;
|
|
119
48
|
this.throttlingIdSet = new Set();
|
|
120
49
|
this.timeTillThrottling = 0;
|
|
121
|
-
this.connectionStateProps = {};
|
|
122
|
-
// True if current connection has checkpoint information
|
|
123
|
-
// I.e. we know how far behind the client was at the time of establishing connection
|
|
124
|
-
this._hasCheckpointSequenceNumber = false;
|
|
125
50
|
this.closeAbortController = new abort_controller_1.default();
|
|
126
|
-
|
|
127
|
-
this.
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
this.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
this.
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
this.nackHandler = (documentId, messages) => {
|
|
139
|
-
const message = messages[0];
|
|
140
|
-
// TODO: we should remove this check when service updates?
|
|
141
|
-
// eslint-disable-next-line @typescript-eslint/strict-boolean-expressions
|
|
142
|
-
if (this._readonlyPermissions) {
|
|
143
|
-
this.close(driver_utils_1.createWriteError("writeOnReadOnlyDocument"));
|
|
144
|
-
}
|
|
145
|
-
// check message.content for Back-compat with old service.
|
|
146
|
-
const reconnectInfo = message.content !== undefined
|
|
147
|
-
? getNackReconnectInfo(message.content) :
|
|
148
|
-
driver_utils_1.createGenericNetworkError("nackReasonUnknown", undefined, true);
|
|
149
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
150
|
-
this.reconnectOnError("write", reconnectInfo);
|
|
151
|
-
};
|
|
152
|
-
// Connection mode is always read on disconnect/error unless the system mode was write.
|
|
153
|
-
this.disconnectHandler = (disconnectReason) => {
|
|
154
|
-
// Note: we might get multiple disconnect calls on same socket, as early disconnect notification
|
|
155
|
-
// ("server_disconnect", ODSP-specific) is mapped to "disconnect"
|
|
156
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
157
|
-
this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionDisconnected", disconnectReason));
|
|
158
|
-
};
|
|
159
|
-
this.errorHandler = (error) => {
|
|
160
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
161
|
-
this.reconnectOnError(this.defaultReconnectionMode, createReconnectError("dmDocumentDeltaConnectionError", error));
|
|
162
|
-
};
|
|
163
|
-
this.pongHandler = (latency) => {
|
|
164
|
-
this.emit("pong", latency);
|
|
51
|
+
this.deltaStorageDelayId = uuid_1.v4();
|
|
52
|
+
this.deltaStreamDelayId = uuid_1.v4();
|
|
53
|
+
this.messageBuffer = [];
|
|
54
|
+
const props = {
|
|
55
|
+
incomingOpHandler: (messages, reason) => this.enqueueMessages(messages, reason),
|
|
56
|
+
signalHandler: (message) => this._inboundSignal.push(message),
|
|
57
|
+
reconnectionDelayHandler: (delayMs, error) => this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error),
|
|
58
|
+
closeHandler: (error) => this.close(error),
|
|
59
|
+
disconnectHandler: (reason) => this.disconnectHandler(reason),
|
|
60
|
+
connectHandler: (connection) => this.connectHandler(connection),
|
|
61
|
+
pongHandler: (latency) => this.emit("pong", latency),
|
|
62
|
+
readonlyChangeHandler: (readonly) => telemetry_utils_1.safeRaiseEvent(this, this.logger, "readonly", readonly),
|
|
165
63
|
};
|
|
166
|
-
this.
|
|
167
|
-
this.defaultReconnectionMode = this.client.mode;
|
|
168
|
-
this._reconnectMode = reconnectAllowed ? ReconnectMode.Enabled : ReconnectMode.Never;
|
|
64
|
+
this.connectionManager = createConnectionManager(props);
|
|
169
65
|
this._inbound = new deltaQueue_1.DeltaQueue((op) => {
|
|
170
66
|
this.processInboundMessage(op);
|
|
171
67
|
});
|
|
172
68
|
this._inbound.on("error", (error) => {
|
|
173
69
|
this.close(container_utils_1.CreateProcessingError(error, "deltaManagerInboundErrorHandler", this.lastMessage));
|
|
174
70
|
});
|
|
175
|
-
// Outbound message queue. The outbound queue is represented as a queue of an array of ops. Ops contained
|
|
176
|
-
// within an array *must* fit within the maxMessageSize and are guaranteed to be ordered sequentially.
|
|
177
|
-
this._outbound = new deltaQueue_1.DeltaQueue((messages) => {
|
|
178
|
-
if (this.connection === undefined) {
|
|
179
|
-
throw new Error("Attempted to submit an outbound message without connection");
|
|
180
|
-
}
|
|
181
|
-
this.connection.submit(messages);
|
|
182
|
-
});
|
|
183
|
-
this._outbound.on("error", (error) => {
|
|
184
|
-
this.close(telemetry_utils_1.normalizeError(error));
|
|
185
|
-
});
|
|
186
71
|
// Inbound signal queue
|
|
187
72
|
this._inboundSignal = new deltaQueue_1.DeltaQueue((message) => {
|
|
188
73
|
if (this.handler === undefined) {
|
|
@@ -203,21 +88,9 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
203
88
|
get active() { return this._active(); }
|
|
204
89
|
get disposed() { return this.closed; }
|
|
205
90
|
get IDeltaSender() { return this; }
|
|
206
|
-
/**
|
|
207
|
-
* Tells if current connection has checkpoint information.
|
|
208
|
-
* I.e. we know how far behind the client was at the time of establishing connection
|
|
209
|
-
*/
|
|
210
|
-
get hasCheckpointSequenceNumber() {
|
|
211
|
-
// Valid to be called only if we have active connection.
|
|
212
|
-
common_utils_1.assert(this.connection !== undefined, 0x0df /* "Missing active connection" */);
|
|
213
|
-
return this._hasCheckpointSequenceNumber;
|
|
214
|
-
}
|
|
215
91
|
get inbound() {
|
|
216
92
|
return this._inbound;
|
|
217
93
|
}
|
|
218
|
-
get outbound() {
|
|
219
|
-
return this._outbound;
|
|
220
|
-
}
|
|
221
94
|
get inboundSignal() {
|
|
222
95
|
return this._inboundSignal;
|
|
223
96
|
}
|
|
@@ -239,39 +112,22 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
239
112
|
get minimumSequenceNumber() {
|
|
240
113
|
return this.minSequenceNumber;
|
|
241
114
|
}
|
|
242
|
-
get maxMessageSize() {
|
|
243
|
-
var _a, _b, _c;
|
|
244
|
-
return (_c = (_b = (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration) === null || _b === void 0 ? void 0 : _b.maxMessageSize) !== null && _c !== void 0 ? _c : DefaultChunkSize;
|
|
245
|
-
}
|
|
246
|
-
get version() {
|
|
247
|
-
if (this.connection === undefined) {
|
|
248
|
-
throw new Error("Cannot check version without a connection");
|
|
249
|
-
}
|
|
250
|
-
return this.connection.version;
|
|
251
|
-
}
|
|
252
|
-
get serviceConfiguration() {
|
|
253
|
-
var _a;
|
|
254
|
-
return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.serviceConfiguration;
|
|
255
|
-
}
|
|
256
|
-
get scopes() {
|
|
257
|
-
var _a;
|
|
258
|
-
return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.scopes;
|
|
259
|
-
}
|
|
260
|
-
get socketDocumentId() {
|
|
261
|
-
var _a;
|
|
262
|
-
return (_a = this.connection) === null || _a === void 0 ? void 0 : _a.claims.documentId;
|
|
263
|
-
}
|
|
264
115
|
/**
|
|
265
|
-
*
|
|
116
|
+
* Tells if current connection has checkpoint information.
|
|
117
|
+
* I.e. we know how far behind the client was at the time of establishing connection
|
|
266
118
|
*/
|
|
267
|
-
get
|
|
268
|
-
|
|
269
|
-
common_utils_1.assert(
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
}
|
|
119
|
+
get hasCheckpointSequenceNumber() {
|
|
120
|
+
// Valid to be called only if we have active connection.
|
|
121
|
+
common_utils_1.assert(this.connectionManager.connected, 0x0df /* "Missing active connection" */);
|
|
122
|
+
return this._checkpointSequenceNumber !== undefined;
|
|
123
|
+
}
|
|
124
|
+
// Forwarding connection manager properties / IDeltaManager implementation
|
|
125
|
+
get maxMessageSize() { return this.connectionManager.maxMessageSize; }
|
|
126
|
+
get version() { return this.connectionManager.version; }
|
|
127
|
+
get serviceConfiguration() { return this.connectionManager.serviceConfiguration; }
|
|
128
|
+
get outbound() { return this.connectionManager.outbound; }
|
|
129
|
+
get readOnlyInfo() { return this.connectionManager.readOnlyInfo; }
|
|
130
|
+
get clientDetails() { return this.connectionManager.clientDetails; }
|
|
275
131
|
/**
|
|
276
132
|
* Tells if container is in read-only mode.
|
|
277
133
|
* Data stores should listen for "readonly" notifications and disallow user
|
|
@@ -283,109 +139,50 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
283
139
|
* @deprecated - use readOnlyInfo
|
|
284
140
|
*/
|
|
285
141
|
get readonly() {
|
|
286
|
-
|
|
287
|
-
return true;
|
|
288
|
-
}
|
|
289
|
-
return this._readonlyPermissions;
|
|
290
|
-
}
|
|
291
|
-
get readOnlyInfo() {
|
|
292
|
-
const storageOnly = this.connection !== undefined && this.connection instanceof NoDeltaStream;
|
|
293
|
-
if (storageOnly || this._forceReadonly || this._readonlyPermissions === true) {
|
|
294
|
-
return {
|
|
295
|
-
readonly: true,
|
|
296
|
-
forced: this._forceReadonly,
|
|
297
|
-
permissions: this._readonlyPermissions,
|
|
298
|
-
storageOnly,
|
|
299
|
-
};
|
|
300
|
-
}
|
|
301
|
-
return { readonly: this._readonlyPermissions };
|
|
302
|
-
}
|
|
303
|
-
/**
|
|
304
|
-
* Automatic reconnecting enabled or disabled.
|
|
305
|
-
* If set to Never, then reconnecting will never be allowed.
|
|
306
|
-
*/
|
|
307
|
-
get reconnectMode() {
|
|
308
|
-
return this._reconnectMode;
|
|
142
|
+
return this.readOnlyInfo.readonly;
|
|
309
143
|
}
|
|
310
|
-
|
|
311
|
-
//
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
const
|
|
320
|
-
|
|
144
|
+
submit(type, contents, batch = false, metadata) {
|
|
145
|
+
// Start adding trace for the op.
|
|
146
|
+
const traces = [
|
|
147
|
+
{
|
|
148
|
+
action: "start",
|
|
149
|
+
service: "client",
|
|
150
|
+
timestamp: Date.now(),
|
|
151
|
+
}
|
|
152
|
+
];
|
|
153
|
+
const messagePartial = {
|
|
154
|
+
contents: JSON.stringify(contents),
|
|
155
|
+
metadata,
|
|
156
|
+
referenceSequenceNumber: this.lastProcessedSequenceNumber,
|
|
157
|
+
traces,
|
|
158
|
+
type,
|
|
321
159
|
};
|
|
322
|
-
if (
|
|
323
|
-
|
|
160
|
+
if (!batch) {
|
|
161
|
+
this.flush();
|
|
324
162
|
}
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
sentOps: this.clientSequenceNumber });
|
|
163
|
+
const message = this.connectionManager.prepareMessageToSend(messagePartial);
|
|
164
|
+
if (message === undefined) {
|
|
165
|
+
return -1;
|
|
329
166
|
}
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
|
|
333
|
-
* Will throw an error if reconnectMode set to Never.
|
|
334
|
-
*/
|
|
335
|
-
setAutoReconnect(mode) {
|
|
336
|
-
common_utils_1.assert(mode !== ReconnectMode.Never && this._reconnectMode !== ReconnectMode.Never, 0x278 /* "API is not supported for non-connecting or closed container" */);
|
|
337
|
-
this._reconnectMode = mode;
|
|
338
|
-
if (mode !== ReconnectMode.Enabled) {
|
|
339
|
-
// immediately disconnect - do not rely on service eventually dropping connection.
|
|
340
|
-
this.disconnectFromDeltaStream("setAutoReconnect");
|
|
167
|
+
this.messageBuffer.push(message);
|
|
168
|
+
if (!batch) {
|
|
169
|
+
this.flush();
|
|
341
170
|
}
|
|
171
|
+
this.emit("submitOp", message);
|
|
172
|
+
return message.clientSequenceNumber;
|
|
342
173
|
}
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
347
|
-
* (server can return "write" mode even when asked for "read")
|
|
348
|
-
* Leveraging same "readonly" event as runtime & data stores should behave the same in such case
|
|
349
|
-
* as in read-only permissions.
|
|
350
|
-
* But this.active can be used by some DDSes to figure out if ops can be sent
|
|
351
|
-
* (for example, read-only view still participates in code proposals / upgrades decisions)
|
|
352
|
-
*
|
|
353
|
-
* Forcing Readonly does not prevent DDS from generating ops. It is up to user code to honour
|
|
354
|
-
* the readonly flag. If ops are generated, they will accumulate locally and not be sent. If
|
|
355
|
-
* there are pending in the outbound queue, it will stop sending until force readonly is
|
|
356
|
-
* cleared.
|
|
357
|
-
*
|
|
358
|
-
* @param readonly - set or clear force readonly.
|
|
359
|
-
*/
|
|
360
|
-
forceReadonly(readonly) {
|
|
361
|
-
if (readonly !== this._forceReadonly) {
|
|
362
|
-
this.logger.sendTelemetryEvent({
|
|
363
|
-
eventName: "ForceReadOnly",
|
|
364
|
-
value: readonly,
|
|
365
|
-
});
|
|
366
|
-
}
|
|
367
|
-
const oldValue = this.readOnlyInfo.readonly;
|
|
368
|
-
this._forceReadonly = readonly;
|
|
369
|
-
if (oldValue !== this.readOnlyInfo.readonly) {
|
|
370
|
-
common_utils_1.assert(this._reconnectMode !== ReconnectMode.Never, 0x279 /* "API is not supported for non-connecting or closed container" */);
|
|
371
|
-
let reconnect = false;
|
|
372
|
-
if (this.readOnlyInfo.readonly === true) {
|
|
373
|
-
// If we switch to readonly while connected, we should disconnect first
|
|
374
|
-
// See comment in the "readonly" event handler to deltaManager set up by
|
|
375
|
-
// the ContainerRuntime constructor
|
|
376
|
-
if (this.shouldJoinWrite()) {
|
|
377
|
-
// If we have pending changes, then we will never send them - it smells like
|
|
378
|
-
// host logic error.
|
|
379
|
-
this.logger.sendErrorEvent({ eventName: "ForceReadonlyPendingChanged" });
|
|
380
|
-
}
|
|
381
|
-
reconnect = this.disconnectFromDeltaStream("Force readonly");
|
|
382
|
-
}
|
|
383
|
-
telemetry_utils_1.safeRaiseEvent(this, this.logger, "readonly", this.readOnlyInfo.readonly);
|
|
384
|
-
if (reconnect) {
|
|
385
|
-
// reconnect if we disconnected from before.
|
|
386
|
-
this.triggerConnect({ reason: "forceReadonly", mode: "read", fetchOpsFromStorage: false });
|
|
387
|
-
}
|
|
174
|
+
submitSignal(content) { return this.connectionManager.submitSignal(content); }
|
|
175
|
+
flush() {
|
|
176
|
+
if (this.messageBuffer.length === 0) {
|
|
177
|
+
return;
|
|
388
178
|
}
|
|
179
|
+
// The prepareFlush event allows listeners to append metadata to the batch prior to submission.
|
|
180
|
+
this.emit("prepareSend", this.messageBuffer);
|
|
181
|
+
this.connectionManager.sendMessages(this.messageBuffer);
|
|
182
|
+
this.messageBuffer = [];
|
|
183
|
+
}
|
|
184
|
+
get connectionProps() {
|
|
185
|
+
return Object.assign({ sequenceNumber: this.lastSequenceNumber }, this.connectionManager.connectionProps);
|
|
389
186
|
}
|
|
390
187
|
/**
|
|
391
188
|
* Log error event with a bunch of internal to DeltaManager information about state of op processing
|
|
@@ -395,20 +192,46 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
395
192
|
*/
|
|
396
193
|
logConnectionIssue(event) {
|
|
397
194
|
var _a;
|
|
398
|
-
common_utils_1.assert(this.
|
|
195
|
+
common_utils_1.assert(this.connectionManager.connected, 0x238 /* "called only in connected state" */);
|
|
399
196
|
const pendingSorted = this.pending.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
400
197
|
this.logger.sendErrorEvent(Object.assign(Object.assign(Object.assign(Object.assign({}, event), {
|
|
401
198
|
// This directly tells us if fetching ops is in flight, and thus likely the reason of
|
|
402
199
|
// stalled op processing
|
|
403
200
|
fetchReason: this.fetchReason,
|
|
404
201
|
// A bunch of useful sequence numbers to understand if we are holding some ops from processing
|
|
405
|
-
lastQueuedSequenceNumber: this.lastQueuedSequenceNumber, lastProcessedSequenceNumber: this.lastProcessedSequenceNumber, lastObserved: this.lastObservedSeqNumber }), this.
|
|
202
|
+
lastQueuedSequenceNumber: this.lastQueuedSequenceNumber, lastProcessedSequenceNumber: this.lastProcessedSequenceNumber, lastObserved: this.lastObservedSeqNumber }), this.connectionManager.connectionVerboseProps), { pendingOps: this.pending.length, pendingFirst: (_a = pendingSorted[0]) === null || _a === void 0 ? void 0 : _a.sequenceNumber, haveHandler: this.handler !== undefined, inboundLength: this.inbound.length, inboundPaused: this.inbound.paused }));
|
|
406
203
|
}
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
204
|
+
connectHandler(connection) {
|
|
205
|
+
this.refreshDelayInfo(this.deltaStreamDelayId);
|
|
206
|
+
const props = this.connectionManager.connectionVerboseProps;
|
|
207
|
+
props.connectionLastQueuedSequenceNumber = this.lastQueuedSequenceNumber;
|
|
208
|
+
props.connectionLastObservedSeqNumber = this.lastObservedSeqNumber;
|
|
209
|
+
const checkpointSequenceNumber = connection.checkpointSequenceNumber;
|
|
210
|
+
this._checkpointSequenceNumber = checkpointSequenceNumber;
|
|
211
|
+
if (checkpointSequenceNumber !== undefined) {
|
|
212
|
+
this.updateLatestKnownOpSeqNumber(checkpointSequenceNumber);
|
|
213
|
+
}
|
|
214
|
+
// We cancel all ops on lost of connectivity, and rely on DDSes to resubmit them.
|
|
215
|
+
// Semantics are not well defined for batches (and they are broken right now on disconnects anyway),
|
|
216
|
+
// but it's safe to assume (until better design is put into place) that batches should not exist
|
|
217
|
+
// across multiple connections. Right now we assume runtime will not submit any ops in disconnected
|
|
218
|
+
// state. As requirements change, so should these checks.
|
|
219
|
+
common_utils_1.assert(this.messageBuffer.length === 0, 0x0e9 /* "messageBuffer is not empty on new connection" */);
|
|
220
|
+
this.emit("connect", connection, checkpointSequenceNumber !== undefined ?
|
|
221
|
+
this.lastObservedSeqNumber - this.lastSequenceNumber : undefined);
|
|
222
|
+
// If we got some initial ops, then we know the gap and call above fetched ops to fill it.
|
|
223
|
+
// Same is true for "write" mode even if we have no ops - we will get "join" own op very very soon.
|
|
224
|
+
// However if we are connecting as view-only, then there is no good signal to realize if client is behind.
|
|
225
|
+
// Thus we have to hit storage to see if any ops are there.
|
|
226
|
+
if (checkpointSequenceNumber !== undefined) {
|
|
227
|
+
// We know how far we are behind (roughly). If it's non-zero gap, fetch ops right away.
|
|
228
|
+
if (checkpointSequenceNumber > this.lastQueuedSequenceNumber) {
|
|
229
|
+
this.fetchMissingDeltas("AfterConnection");
|
|
230
|
+
}
|
|
231
|
+
// we do not know the gap, and we will not learn about it if socket is quite - have to ask.
|
|
232
|
+
}
|
|
233
|
+
else if (connection.mode === "read") {
|
|
234
|
+
this.fetchMissingDeltas("AfterReadConnection");
|
|
412
235
|
}
|
|
413
236
|
}
|
|
414
237
|
dispose() {
|
|
@@ -449,66 +272,16 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
449
272
|
// (which in most cases will happen when we are done processing cached ops)
|
|
450
273
|
if (cacheOnly) {
|
|
451
274
|
// fire and forget
|
|
452
|
-
this.fetchMissingDeltas("DocumentOpen"
|
|
275
|
+
this.fetchMissingDeltas("DocumentOpen");
|
|
453
276
|
}
|
|
454
277
|
}
|
|
455
278
|
// Ensure there is no need to call this.processPendingOps() at the end of boot sequence
|
|
456
279
|
common_utils_1.assert(this.fetchReason !== undefined || this.pending.length === 0, 0x269 /* "pending ops are not dropped" */);
|
|
457
280
|
}
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
claims: connection.claims,
|
|
461
|
-
clientId: connection.clientId,
|
|
462
|
-
existing: connection.existing,
|
|
463
|
-
checkpointSequenceNumber: connection.checkpointSequenceNumber,
|
|
464
|
-
get initialClients() { return connection.initialClients; },
|
|
465
|
-
maxMessageSize: connection.serviceConfiguration.maxMessageSize,
|
|
466
|
-
mode: connection.mode,
|
|
467
|
-
serviceConfiguration: connection.serviceConfiguration,
|
|
468
|
-
version: connection.version,
|
|
469
|
-
};
|
|
470
|
-
}
|
|
471
|
-
async connect(args) {
|
|
472
|
-
const connection = await this.connectCore(args);
|
|
473
|
-
return DeltaManager.detailsFromConnection(connection);
|
|
474
|
-
}
|
|
475
|
-
/**
|
|
476
|
-
* Start the connection. Any error should result in container being close.
|
|
477
|
-
* And report the error if it excape for any reason.
|
|
478
|
-
* @param args - The connection arguments
|
|
479
|
-
*/
|
|
480
|
-
triggerConnect(args) {
|
|
481
|
-
common_utils_1.assert(this.connection === undefined, 0x239 /* "called only in disconnected state" */);
|
|
482
|
-
if (this.reconnectMode !== ReconnectMode.Enabled) {
|
|
483
|
-
return;
|
|
484
|
-
}
|
|
485
|
-
this.connectCore(args).catch((err) => {
|
|
486
|
-
// Errors are raised as "error" event and close container.
|
|
487
|
-
// Have a catch-all case in case we missed something
|
|
488
|
-
if (!this.closed) {
|
|
489
|
-
this.logger.sendErrorEvent({ eventName: "ConnectException" }, err);
|
|
490
|
-
}
|
|
491
|
-
});
|
|
492
|
-
}
|
|
493
|
-
async connectCore(args) {
|
|
494
|
-
var _a, _b, _c;
|
|
495
|
-
common_utils_1.assert(!this.closed, 0x26a /* "not closed" */);
|
|
496
|
-
if (this.connection !== undefined) {
|
|
497
|
-
return this.connection;
|
|
498
|
-
}
|
|
499
|
-
if (this.connectionP !== undefined) {
|
|
500
|
-
return this.connectionP;
|
|
501
|
-
}
|
|
281
|
+
connect(args) {
|
|
282
|
+
var _a;
|
|
502
283
|
const fetchOpsFromStorage = (_a = args.fetchOpsFromStorage) !== null && _a !== void 0 ? _a : true;
|
|
503
|
-
|
|
504
|
-
// if we have any non-acked ops from last connection, reconnect as "write".
|
|
505
|
-
// without that we would connect in view-only mode, which will result in immediate
|
|
506
|
-
// firing of "connected" event from Container and switch of current clientId (as tracked
|
|
507
|
-
// by all DDSes). This will make it impossible to figure out if ops actually made it through,
|
|
508
|
-
// so DDSes will immediately resubmit all pending ops, and some of them will be duplicates, corrupting document
|
|
509
|
-
if (this.shouldJoinWrite()) {
|
|
510
|
-
requestedMode = "write";
|
|
511
|
-
}
|
|
284
|
+
telemetry_utils_1.logIfFalse(this.handler !== undefined || !fetchOpsFromStorage, this.logger, "CantFetchWithoutBaseline"); // can't fetch if no baseline
|
|
512
285
|
// Note: There is race condition here.
|
|
513
286
|
// We want to issue request to storage as soon as possible, to
|
|
514
287
|
// reduce latency of becoming current, thus this code here.
|
|
@@ -518,213 +291,11 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
518
291
|
// own "join" message and realize any gap client has in ops.
|
|
519
292
|
// But for view-only connection, we have no such signal, and with no traffic
|
|
520
293
|
// on the wire, we might be always behind.
|
|
521
|
-
// See comment at the end of
|
|
522
|
-
telemetry_utils_1.logIfFalse(this.handler !== undefined || !fetchOpsFromStorage, this.logger, "CantFetchWithoutBaseline"); // can't fetch if no baseline
|
|
294
|
+
// See comment at the end of "connect" handler
|
|
523
295
|
if (fetchOpsFromStorage) {
|
|
524
|
-
this.fetchMissingDeltas(args.reason
|
|
525
|
-
}
|
|
526
|
-
const docService = this.serviceProvider();
|
|
527
|
-
common_utils_1.assert(docService !== undefined, 0x2a7 /* "Container is not attached" */);
|
|
528
|
-
if (((_c = docService.policies) === null || _c === void 0 ? void 0 : _c.storageOnly) === true) {
|
|
529
|
-
const connection = new NoDeltaStream();
|
|
530
|
-
this.connectionP = Promise.resolve(connection); // to keep setupNewSuccessfulConnection happy
|
|
531
|
-
this.setupNewSuccessfulConnection(connection, "read");
|
|
532
|
-
return connection;
|
|
533
|
-
}
|
|
534
|
-
// The promise returned from connectCore will settle with a resolved connection or reject with error
|
|
535
|
-
const connectCore = async () => {
|
|
536
|
-
let connection;
|
|
537
|
-
let delayMs = InitialReconnectDelayInMs;
|
|
538
|
-
let connectRepeatCount = 0;
|
|
539
|
-
const connectStartTime = common_utils_1.performance.now();
|
|
540
|
-
let lastError;
|
|
541
|
-
// This loop will keep trying to connect until successful, with a delay between each iteration.
|
|
542
|
-
while (connection === undefined) {
|
|
543
|
-
if (this.closed) {
|
|
544
|
-
throw new Error("Attempting to connect a closed DeltaManager");
|
|
545
|
-
}
|
|
546
|
-
connectRepeatCount++;
|
|
547
|
-
try {
|
|
548
|
-
this.client.mode = requestedMode;
|
|
549
|
-
connection = await docService.connectToDeltaStream(this.client);
|
|
550
|
-
if (connection.disposed) {
|
|
551
|
-
// Nobody observed this connection, so drop it on the floor and retry.
|
|
552
|
-
this.logger.sendTelemetryEvent({ eventName: "ReceivedClosedConnection" });
|
|
553
|
-
connection = undefined;
|
|
554
|
-
}
|
|
555
|
-
}
|
|
556
|
-
catch (origError) {
|
|
557
|
-
if (typeof origError === "object" && origError !== null &&
|
|
558
|
-
(origError === null || origError === void 0 ? void 0 : origError.errorType) === driver_utils_1.DeltaStreamConnectionForbiddenError.errorType) {
|
|
559
|
-
connection = new NoDeltaStream();
|
|
560
|
-
requestedMode = "read";
|
|
561
|
-
break;
|
|
562
|
-
}
|
|
563
|
-
// Socket.io error when we connect to wrong socket, or hit some multiplexing bug
|
|
564
|
-
if (!driver_utils_1.canRetryOnError(origError)) {
|
|
565
|
-
const error = telemetry_utils_1.normalizeError(origError, { props: fatalConnectErrorProp });
|
|
566
|
-
this.close(error);
|
|
567
|
-
throw error;
|
|
568
|
-
}
|
|
569
|
-
// Log error once - we get too many errors in logs when we are offline,
|
|
570
|
-
// and unfortunately there is no reliable way to detect that.
|
|
571
|
-
if (connectRepeatCount === 1) {
|
|
572
|
-
driver_utils_1.logNetworkFailure(this.logger, {
|
|
573
|
-
delay: delayMs,
|
|
574
|
-
eventName: "DeltaConnectionFailureToConnect",
|
|
575
|
-
duration: telemetry_utils_1.TelemetryLogger.formatTick(common_utils_1.performance.now() - connectStartTime),
|
|
576
|
-
}, origError);
|
|
577
|
-
}
|
|
578
|
-
lastError = origError;
|
|
579
|
-
const retryDelayFromError = driver_utils_1.getRetryDelayFromError(origError);
|
|
580
|
-
delayMs = retryDelayFromError !== null && retryDelayFromError !== void 0 ? retryDelayFromError : Math.min(delayMs * 2, MaxReconnectDelayInMs);
|
|
581
|
-
if (retryDelayFromError !== undefined) {
|
|
582
|
-
this.emitDelayInfo(this.deltaStreamDelayId, retryDelayFromError, origError);
|
|
583
|
-
}
|
|
584
|
-
await driver_utils_1.waitForConnectedState(delayMs);
|
|
585
|
-
}
|
|
586
|
-
}
|
|
587
|
-
// If we retried more than once, log an event about how long it took
|
|
588
|
-
if (connectRepeatCount > 1) {
|
|
589
|
-
this.logger.sendTelemetryEvent({
|
|
590
|
-
eventName: "MultipleDeltaConnectionFailures",
|
|
591
|
-
attempts: connectRepeatCount,
|
|
592
|
-
duration: telemetry_utils_1.TelemetryLogger.formatTick(common_utils_1.performance.now() - connectStartTime),
|
|
593
|
-
}, lastError);
|
|
594
|
-
}
|
|
595
|
-
this.setupNewSuccessfulConnection(connection, requestedMode);
|
|
596
|
-
return connection;
|
|
597
|
-
};
|
|
598
|
-
// This promise settles as soon as we know the outcome of the connection attempt
|
|
599
|
-
// Set it upfront, such that if connection is established (NoDeltaConnection) or rejected (bug in
|
|
600
|
-
// connectToDeltaStream() implementation - throwing exception vs. returning rejected promise) in
|
|
601
|
-
// synchronous way, we have this.connectionP setup for all the code to assert correctness of the flow.
|
|
602
|
-
const deferred = new common_utils_1.Deferred();
|
|
603
|
-
this.connectionP = deferred.promise;
|
|
604
|
-
// Regardless of how the connection attempt concludes, we'll clear the promise and remove the listener
|
|
605
|
-
// Reject the connection promise if the DeltaManager gets closed during connection
|
|
606
|
-
const cleanupAndReject = (error) => {
|
|
607
|
-
this.connectionP = undefined;
|
|
608
|
-
this.removeListener("closed", cleanupAndReject);
|
|
609
|
-
// This error came from some logic error in this file. Fail-fast to learn and fix the issue faster
|
|
610
|
-
const normalizedError = telemetry_utils_1.normalizeError(error, { props: fatalConnectErrorProp });
|
|
611
|
-
this.close(normalizedError);
|
|
612
|
-
deferred.reject(normalizedError);
|
|
613
|
-
};
|
|
614
|
-
this.on("closed", cleanupAndReject);
|
|
615
|
-
// Attempt the connection
|
|
616
|
-
connectCore().then((connection) => {
|
|
617
|
-
this.removeListener("closed", cleanupAndReject);
|
|
618
|
-
deferred.resolve(connection);
|
|
619
|
-
}).catch(cleanupAndReject);
|
|
620
|
-
return this.connectionP;
|
|
621
|
-
}
|
|
622
|
-
flush() {
|
|
623
|
-
if (this.messageBuffer.length === 0) {
|
|
624
|
-
return;
|
|
625
|
-
}
|
|
626
|
-
// The prepareFlush event allows listeners to append metadata to the batch prior to submission.
|
|
627
|
-
this.emit("prepareSend", this.messageBuffer);
|
|
628
|
-
this._outbound.push(this.messageBuffer);
|
|
629
|
-
this.messageBuffer = [];
|
|
630
|
-
}
|
|
631
|
-
/**
|
|
632
|
-
* Submits the given delta returning the client sequence number for the message. Contents is the actual
|
|
633
|
-
* contents of the message. appData is optional metadata that can be attached to the op by the app.
|
|
634
|
-
*
|
|
635
|
-
* If batch is set to true then the submit will be batched - and as a result guaranteed to be ordered sequentially
|
|
636
|
-
* in the global sequencing space. The batch will be flushed either when flush is called or when a non-batched
|
|
637
|
-
* op is submitted.
|
|
638
|
-
*/
|
|
639
|
-
submit(type, contents, batch = false, metadata) {
|
|
640
|
-
// TODO need to fail if gets too large
|
|
641
|
-
// const serializedContent = JSON.stringify(this.messageBuffer);
|
|
642
|
-
// const maxOpSize = this.context.deltaManager.maxMessageSize;
|
|
643
|
-
var _a, _b;
|
|
644
|
-
if (this.readOnlyInfo.readonly === true) {
|
|
645
|
-
common_utils_1.assert(this.readOnlyInfo.readonly === true, 0x1f0 /* "Unexpected mismatch in readonly" */);
|
|
646
|
-
const error = new container_utils_1.GenericError("deltaManagerReadonlySubmit", undefined /* error */, {
|
|
647
|
-
readonly: this.readOnlyInfo.readonly,
|
|
648
|
-
forcedReadonly: this.readOnlyInfo.forced,
|
|
649
|
-
readonlyPermissions: this.readOnlyInfo.permissions,
|
|
650
|
-
storageOnly: this.readOnlyInfo.storageOnly,
|
|
651
|
-
});
|
|
652
|
-
this.close(error);
|
|
653
|
-
return -1;
|
|
654
|
-
}
|
|
655
|
-
// reset clientSequenceNumber if we are using new clientId.
|
|
656
|
-
// we keep info about old connection as long as possible to be able to account for all non-acked ops
|
|
657
|
-
// that we pick up on next connection.
|
|
658
|
-
common_utils_1.assert(!!this.connection, 0x0e4 /* "Lost old connection!" */);
|
|
659
|
-
if (this.lastSubmittedClientId !== ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
|
|
660
|
-
this.lastSubmittedClientId = (_b = this.connection) === null || _b === void 0 ? void 0 : _b.clientId;
|
|
661
|
-
this.clientSequenceNumber = 0;
|
|
662
|
-
this.clientSequenceNumberObserved = 0;
|
|
663
|
-
}
|
|
664
|
-
// If connection is "read" or implicit "read" (got leave op for "write" connection),
|
|
665
|
-
// then op can't make it through - we will get a nack if op is sent.
|
|
666
|
-
// We can short-circuit this process.
|
|
667
|
-
// Note that we also want nacks to be rare and be treated as catastrophic failures.
|
|
668
|
-
// Be careful with reentrancy though - disconnected event should not be be raised in the
|
|
669
|
-
// middle of the current workflow, but rather on clean stack!
|
|
670
|
-
if (this.connectionMode === "read" || this.downgradedConnection) {
|
|
671
|
-
if (!this.pendingReconnect) {
|
|
672
|
-
this.pendingReconnect = true;
|
|
673
|
-
Promise.resolve().then(async () => {
|
|
674
|
-
if (this.pendingReconnect) { // still valid?
|
|
675
|
-
return this.reconnectOnErrorCore("write", // connectionMode
|
|
676
|
-
"Switch to write");
|
|
677
|
-
}
|
|
678
|
-
})
|
|
679
|
-
.catch(() => { });
|
|
680
|
-
}
|
|
681
|
-
// Can return -1 here, but no other path does it (other than error path in Container),
|
|
682
|
-
// so it's better not to introduce new states.
|
|
683
|
-
return ++this.clientSequenceNumber;
|
|
684
|
-
}
|
|
685
|
-
const service = this.clientDetails.type === undefined || this.clientDetails.type === ""
|
|
686
|
-
? "unknown"
|
|
687
|
-
: this.clientDetails.type;
|
|
688
|
-
// Start adding trace for the op.
|
|
689
|
-
const traces = [
|
|
690
|
-
{
|
|
691
|
-
action: "start",
|
|
692
|
-
service,
|
|
693
|
-
timestamp: Date.now(),
|
|
694
|
-
}
|
|
695
|
-
];
|
|
696
|
-
const message = {
|
|
697
|
-
clientSequenceNumber: ++this.clientSequenceNumber,
|
|
698
|
-
contents: JSON.stringify(contents),
|
|
699
|
-
metadata,
|
|
700
|
-
referenceSequenceNumber: this.lastProcessedSequenceNumber,
|
|
701
|
-
traces,
|
|
702
|
-
type,
|
|
703
|
-
};
|
|
704
|
-
if (type === protocol_definitions_1.MessageType.NoOp) {
|
|
705
|
-
this.trailingNoopCount++;
|
|
706
|
-
}
|
|
707
|
-
else {
|
|
708
|
-
this.trailingNoopCount = 0;
|
|
709
|
-
}
|
|
710
|
-
this.emit("submitOp", message);
|
|
711
|
-
if (!batch) {
|
|
712
|
-
this.flush();
|
|
713
|
-
this.messageBuffer.push(message);
|
|
714
|
-
this.flush();
|
|
715
|
-
}
|
|
716
|
-
else {
|
|
717
|
-
this.messageBuffer.push(message);
|
|
718
|
-
}
|
|
719
|
-
return message.clientSequenceNumber;
|
|
720
|
-
}
|
|
721
|
-
submitSignal(content) {
|
|
722
|
-
if (this.connection !== undefined) {
|
|
723
|
-
this.connection.submitSignal(content);
|
|
724
|
-
}
|
|
725
|
-
else {
|
|
726
|
-
this.logger.sendErrorEvent({ eventName: "submitSignalDisconnected" });
|
|
296
|
+
this.fetchMissingDeltas(args.reason);
|
|
727
297
|
}
|
|
298
|
+
this.connectionManager.connect(args.mode);
|
|
728
299
|
}
|
|
729
300
|
async getDeltas(from, // inclusive
|
|
730
301
|
to, // exclusive
|
|
@@ -736,9 +307,6 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
736
307
|
if (this.deltaStorage === undefined) {
|
|
737
308
|
this.deltaStorage = await docService.connectToDeltaStorage();
|
|
738
309
|
}
|
|
739
|
-
common_utils_1.assert(this.closeAbortController.signal.onabort === null, 0x1e8 /* "reentrancy" */);
|
|
740
|
-
const controller = new abort_controller_1.default();
|
|
741
|
-
this.closeAbortController.signal.onabort = () => controller.abort();
|
|
742
310
|
let cancelFetch;
|
|
743
311
|
if (to !== undefined) {
|
|
744
312
|
const lastExpectedOp = to - 1; // make it inclusive!
|
|
@@ -746,7 +314,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
746
314
|
// received through delta stream. Validate that before moving forward.
|
|
747
315
|
if (this.lastQueuedSequenceNumber >= lastExpectedOp) {
|
|
748
316
|
this.logger.sendPerformanceEvent(Object.assign({ reason: this.fetchReason, eventName: "ExtraStorageCall", early: true, from,
|
|
749
|
-
to }, this.
|
|
317
|
+
to }, this.connectionManager.connectionVerboseProps));
|
|
750
318
|
return;
|
|
751
319
|
}
|
|
752
320
|
// Be prepared for the case where webSocket would receive the ops that we are trying to fill through
|
|
@@ -763,6 +331,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
763
331
|
// That said, if we have socket connection, make sure we got ops up to checkpointSequenceNumber!
|
|
764
332
|
cancelFetch = (op) => op.sequenceNumber >= this.lastObservedSeqNumber;
|
|
765
333
|
}
|
|
334
|
+
const controller = new abort_controller_1.default();
|
|
766
335
|
let opsFromFetch = false;
|
|
767
336
|
const opListener = (op) => {
|
|
768
337
|
common_utils_1.assert(op.sequenceNumber === this.lastQueuedSequenceNumber, 0x23a /* "seq#'s" */);
|
|
@@ -774,8 +343,10 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
774
343
|
this._inbound.off("push", opListener);
|
|
775
344
|
}
|
|
776
345
|
};
|
|
777
|
-
this._inbound.on("push", opListener);
|
|
778
346
|
try {
|
|
347
|
+
this._inbound.on("push", opListener);
|
|
348
|
+
common_utils_1.assert(this.closeAbortController.signal.onabort === null, 0x1e8 /* "reentrancy" */);
|
|
349
|
+
this.closeAbortController.signal.onabort = () => controller.abort();
|
|
779
350
|
const stream = this.deltaStorage.fetchMessages(from, // inclusive
|
|
780
351
|
to, // exclusive
|
|
781
352
|
controller.signal, cacheOnly, this.fetchReason);
|
|
@@ -795,9 +366,9 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
795
366
|
}
|
|
796
367
|
}
|
|
797
368
|
finally {
|
|
798
|
-
common_utils_1.assert(!opsFromFetch, 0x289 /* "logic error" */);
|
|
799
369
|
this.closeAbortController.signal.onabort = null;
|
|
800
370
|
this._inbound.off("push", opListener);
|
|
371
|
+
common_utils_1.assert(!opsFromFetch, 0x289 /* "logic error" */);
|
|
801
372
|
}
|
|
802
373
|
}
|
|
803
374
|
/**
|
|
@@ -808,16 +379,9 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
808
379
|
return;
|
|
809
380
|
}
|
|
810
381
|
this.closed = true;
|
|
811
|
-
|
|
812
|
-
this._reconnectMode = ReconnectMode.Never;
|
|
382
|
+
this.connectionManager.dispose(error);
|
|
813
383
|
this.closeAbortController.abort();
|
|
814
|
-
const disconnectReason = error !== undefined
|
|
815
|
-
? `Closing DeltaManager (${error.message})`
|
|
816
|
-
: "Closing DeltaManager";
|
|
817
|
-
// This raises "disconnect" event if we have active connection.
|
|
818
|
-
this.disconnectFromDeltaStream(disconnectReason);
|
|
819
384
|
this._inbound.clear();
|
|
820
|
-
this._outbound.clear();
|
|
821
385
|
this._inboundSignal.clear();
|
|
822
386
|
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
823
387
|
this._inbound.pause();
|
|
@@ -825,10 +389,6 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
825
389
|
this._inboundSignal.pause();
|
|
826
390
|
// Drop pending messages - this will ensure catchUp() does not go into infinite loop
|
|
827
391
|
this.pending = [];
|
|
828
|
-
// Notify everyone we are in read-only state.
|
|
829
|
-
// Useful for data stores in case we hit some critical error,
|
|
830
|
-
// to switch to a mode where user edits are not accepted
|
|
831
|
-
this.set_readonlyPermissions(true);
|
|
832
392
|
// This needs to be the last thing we do (before removing listeners), as it causes
|
|
833
393
|
// Container to dispose context and break ability of data stores / runtime to "hear"
|
|
834
394
|
// from delta manager, including notification (above) about readonly state.
|
|
@@ -841,6 +401,20 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
841
401
|
this.timeTillThrottling = 0;
|
|
842
402
|
}
|
|
843
403
|
}
|
|
404
|
+
disconnectHandler(reason) {
|
|
405
|
+
if (this.messageBuffer.length > 0) {
|
|
406
|
+
// Behavior is not well defined here RE batches across connections / disconnect.
|
|
407
|
+
// DeltaManager overall policy - drop all ops on disconnection and rely on
|
|
408
|
+
// container runtime to deal with resubmitting any ops that did not make it through.
|
|
409
|
+
// So drop them, but also raise error event to look into details.
|
|
410
|
+
this.logger.sendErrorEvent({
|
|
411
|
+
eventName: "OpenBatchOnDisconnect",
|
|
412
|
+
length: this.messageBuffer.length,
|
|
413
|
+
});
|
|
414
|
+
this.messageBuffer.length = 0;
|
|
415
|
+
}
|
|
416
|
+
this.emit("disconnect", reason);
|
|
417
|
+
}
|
|
844
418
|
/**
|
|
845
419
|
* Emit info about a delay in service communication on account of throttling.
|
|
846
420
|
* @param id - Id of the connection that is delayed
|
|
@@ -856,183 +430,6 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
856
430
|
this.emit("throttled", throttlingWarning);
|
|
857
431
|
}
|
|
858
432
|
}
|
|
859
|
-
/**
|
|
860
|
-
* Once we've successfully gotten a connection, we need to set up state, attach event listeners, and process
|
|
861
|
-
* initial messages.
|
|
862
|
-
* @param connection - The newly established connection
|
|
863
|
-
*/
|
|
864
|
-
setupNewSuccessfulConnection(connection, requestedMode) {
|
|
865
|
-
// Old connection should have been cleaned up before establishing a new one
|
|
866
|
-
common_utils_1.assert(this.connection === undefined, 0x0e6 /* "old connection exists on new connection setup" */);
|
|
867
|
-
common_utils_1.assert(this.connectionP !== undefined || this.closed, 0x27f /* "reentrancy may result in incorrect behavior" */);
|
|
868
|
-
common_utils_1.assert(!connection.disposed, 0x28a /* "can't be disposed - Callers need to ensure that!" */);
|
|
869
|
-
this.connectionP = undefined;
|
|
870
|
-
this.connection = connection;
|
|
871
|
-
// Does information in scopes & mode matches?
|
|
872
|
-
// If we asked for "write" and got "read", then file is read-only
|
|
873
|
-
// But if we ask read, server can still give us write.
|
|
874
|
-
const readonly = !connection.claims.scopes.includes(protocol_definitions_1.ScopeType.DocWrite);
|
|
875
|
-
// This connection mode validation logic is moving to the driver layer in 0.44. These two asserts can be
|
|
876
|
-
// removed after those packages have released and become ubiquitous.
|
|
877
|
-
common_utils_1.assert(requestedMode === "read" || readonly === (this.connectionMode === "read"), 0x0e7 /* "claims/connectionMode mismatch" */);
|
|
878
|
-
common_utils_1.assert(!readonly || this.connectionMode === "read", 0x0e8 /* "readonly perf with write connection" */);
|
|
879
|
-
this.set_readonlyPermissions(readonly);
|
|
880
|
-
this.refreshDelayInfo(this.deltaStreamDelayId);
|
|
881
|
-
if (this.closed) {
|
|
882
|
-
// Raise proper events, Log telemetry event and close connection.
|
|
883
|
-
this.disconnectFromDeltaStream("DeltaManager already closed");
|
|
884
|
-
return;
|
|
885
|
-
}
|
|
886
|
-
// We cancel all ops on lost of connectivity, and rely on DDSes to resubmit them.
|
|
887
|
-
// Semantics are not well defined for batches (and they are broken right now on disconnects anyway),
|
|
888
|
-
// but it's safe to assume (until better design is put into place) that batches should not exist
|
|
889
|
-
// across multiple connections. Right now we assume runtime will not submit any ops in disconnected
|
|
890
|
-
// state. As requirements change, so should these checks.
|
|
891
|
-
common_utils_1.assert(this.messageBuffer.length === 0, 0x0e9 /* "messageBuffer is not empty on new connection" */);
|
|
892
|
-
this._outbound.resume();
|
|
893
|
-
connection.on("op", this.opHandler);
|
|
894
|
-
connection.on("signal", this.signalHandler);
|
|
895
|
-
connection.on("nack", this.nackHandler);
|
|
896
|
-
connection.on("disconnect", this.disconnectHandler);
|
|
897
|
-
connection.on("error", this.errorHandler);
|
|
898
|
-
connection.on("pong", this.pongHandler);
|
|
899
|
-
// Initial messages are always sorted. However, due to early op handler installed by drivers and appending those
|
|
900
|
-
// ops to initialMessages, resulting set is no longer sorted, which would result in client hitting storage to
|
|
901
|
-
// fill in gap. We will recover by cancelling this request once we process remaining ops, but it's a waste that
|
|
902
|
-
// we could avoid
|
|
903
|
-
const initialMessages = connection.initialMessages.sort((a, b) => a.sequenceNumber - b.sequenceNumber);
|
|
904
|
-
this.connectionStateProps = {
|
|
905
|
-
connectionLastQueuedSequenceNumber: this.lastQueuedSequenceNumber,
|
|
906
|
-
connectionLastObservedSeqNumber: this.lastObservedSeqNumber,
|
|
907
|
-
clientId: connection.clientId,
|
|
908
|
-
mode: connection.mode,
|
|
909
|
-
};
|
|
910
|
-
if (connection.relayServiceAgent !== undefined) {
|
|
911
|
-
this.connectionStateProps.relayServiceAgent = connection.relayServiceAgent;
|
|
912
|
-
}
|
|
913
|
-
this._hasCheckpointSequenceNumber = false;
|
|
914
|
-
// Some storages may provide checkpointSequenceNumber to identify how far client is behind.
|
|
915
|
-
const checkpointSequenceNumber = connection.checkpointSequenceNumber;
|
|
916
|
-
if (checkpointSequenceNumber !== undefined) {
|
|
917
|
-
this._hasCheckpointSequenceNumber = true;
|
|
918
|
-
this.updateLatestKnownOpSeqNumber(checkpointSequenceNumber);
|
|
919
|
-
}
|
|
920
|
-
// Update knowledge of how far we are behind, before raising "connect" event
|
|
921
|
-
// This is duplication of what enqueueMessages() does, but we have to raise event before we get there,
|
|
922
|
-
// so duplicating update logic here as well.
|
|
923
|
-
const last = initialMessages.length > 0 ? initialMessages[initialMessages.length - 1].sequenceNumber : -1;
|
|
924
|
-
if (initialMessages.length > 0) {
|
|
925
|
-
this._hasCheckpointSequenceNumber = true;
|
|
926
|
-
this.updateLatestKnownOpSeqNumber(last);
|
|
927
|
-
}
|
|
928
|
-
// Notify of the connection
|
|
929
|
-
// WARNING: This has to happen before processInitialMessages() call below.
|
|
930
|
-
// If not, we may not update Container.pendingClientId in time before seeing our own join session op.
|
|
931
|
-
this.emit("connect", DeltaManager.detailsFromConnection(connection), this._hasCheckpointSequenceNumber ? this.lastObservedSeqNumber - this.lastSequenceNumber : undefined);
|
|
932
|
-
this.enqueueMessages(initialMessages, this.connectFirstConnection ? "InitialOps" : "ReconnectOps");
|
|
933
|
-
if (connection.initialSignals !== undefined) {
|
|
934
|
-
for (const signal of connection.initialSignals) {
|
|
935
|
-
this._inboundSignal.push(signal);
|
|
936
|
-
}
|
|
937
|
-
}
|
|
938
|
-
// If we got some initial ops, then we know the gap and call above fetched ops to fill it.
|
|
939
|
-
// Same is true for "write" mode even if we have no ops - we will get self "join" ops very very soon.
|
|
940
|
-
// However if we are connecting as view-only, then there is no good signal to realize if client is behind.
|
|
941
|
-
// Thus we have to hit storage to see if any ops are there.
|
|
942
|
-
if (initialMessages.length === 0) {
|
|
943
|
-
if (checkpointSequenceNumber !== undefined) {
|
|
944
|
-
// We know how far we are behind (roughly). If it's non-zero gap, fetch ops right away.
|
|
945
|
-
if (checkpointSequenceNumber > this.lastQueuedSequenceNumber) {
|
|
946
|
-
this.fetchMissingDeltas("AfterConnection", this.lastQueuedSequenceNumber);
|
|
947
|
-
}
|
|
948
|
-
// we do not know the gap, and we will not learn about it if socket is quite - have to ask.
|
|
949
|
-
}
|
|
950
|
-
else if (connection.mode === "read") {
|
|
951
|
-
this.fetchMissingDeltas("AfterReadConnection", this.lastQueuedSequenceNumber);
|
|
952
|
-
}
|
|
953
|
-
}
|
|
954
|
-
else {
|
|
955
|
-
this.connectionStateProps.connectionInitialOpsFrom = initialMessages[0].sequenceNumber;
|
|
956
|
-
this.connectionStateProps.connectionInitialOpsTo = last + 1;
|
|
957
|
-
}
|
|
958
|
-
this.connectFirstConnection = false;
|
|
959
|
-
}
|
|
960
|
-
/**
|
|
961
|
-
* Disconnect the current connection.
|
|
962
|
-
* @param reason - Text description of disconnect reason to emit with disconnect event
|
|
963
|
-
*/
|
|
964
|
-
disconnectFromDeltaStream(reason) {
|
|
965
|
-
this.pendingReconnect = false;
|
|
966
|
-
this.downgradedConnection = false;
|
|
967
|
-
if (this.connection === undefined) {
|
|
968
|
-
return false;
|
|
969
|
-
}
|
|
970
|
-
common_utils_1.assert(this.connectionP === undefined, 0x27b /* "reentrancy may result in incorrect behavior" */);
|
|
971
|
-
const connection = this.connection;
|
|
972
|
-
// Avoid any re-entrancy - clear object reference
|
|
973
|
-
this.connection = undefined;
|
|
974
|
-
// Remove listeners first so we don't try to retrigger this flow accidentally through reconnectOnError
|
|
975
|
-
connection.off("op", this.opHandler);
|
|
976
|
-
connection.off("signal", this.signalHandler);
|
|
977
|
-
connection.off("nack", this.nackHandler);
|
|
978
|
-
connection.off("disconnect", this.disconnectHandler);
|
|
979
|
-
connection.off("error", this.errorHandler);
|
|
980
|
-
connection.off("pong", this.pongHandler);
|
|
981
|
-
// eslint-disable-next-line @typescript-eslint/no-floating-promises
|
|
982
|
-
this._outbound.pause();
|
|
983
|
-
this._outbound.clear();
|
|
984
|
-
this.emit("disconnect", reason);
|
|
985
|
-
connection.dispose();
|
|
986
|
-
this.connectionStateProps = {};
|
|
987
|
-
return true;
|
|
988
|
-
}
|
|
989
|
-
/**
|
|
990
|
-
* Disconnect the current connection and reconnect.
|
|
991
|
-
* @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
|
|
992
|
-
* @param requestedMode - Read or write
|
|
993
|
-
* @param error - Error reconnect information including whether or not to reconnect
|
|
994
|
-
* @returns A promise that resolves when the connection is reestablished or we stop trying
|
|
995
|
-
*/
|
|
996
|
-
async reconnectOnError(requestedMode, error) {
|
|
997
|
-
return this.reconnectOnErrorCore(requestedMode, error.message, error);
|
|
998
|
-
}
|
|
999
|
-
/**
|
|
1000
|
-
* Disconnect the current connection and reconnect.
|
|
1001
|
-
* @param connection - The connection that wants to reconnect - no-op if it's different from this.connection
|
|
1002
|
-
* @param requestedMode - Read or write
|
|
1003
|
-
* @param error - Error reconnect information including whether or not to reconnect
|
|
1004
|
-
* @returns A promise that resolves when the connection is reestablished or we stop trying
|
|
1005
|
-
*/
|
|
1006
|
-
async reconnectOnErrorCore(requestedMode, disconnectMessage, error) {
|
|
1007
|
-
// We quite often get protocol errors before / after observing nack/disconnect
|
|
1008
|
-
// we do not want to run through same sequence twice.
|
|
1009
|
-
// If we're already disconnected/disconnecting it's not appropriate to call this again.
|
|
1010
|
-
common_utils_1.assert(this.connection !== undefined, 0x0eb /* "Missing connection for reconnect" */);
|
|
1011
|
-
this.disconnectFromDeltaStream(disconnectMessage);
|
|
1012
|
-
const canRetry = error !== undefined ? driver_utils_1.canRetryOnError(error) : true;
|
|
1013
|
-
// If reconnection is not an option, close the DeltaManager
|
|
1014
|
-
if (!canRetry) {
|
|
1015
|
-
this.close(telemetry_utils_1.normalizeError(error, { props: fatalConnectErrorProp }));
|
|
1016
|
-
}
|
|
1017
|
-
else if (this.reconnectMode === ReconnectMode.Never) {
|
|
1018
|
-
// Do not raise container error if we are closing just because we lost connection.
|
|
1019
|
-
// Those errors (like IdleDisconnect) would show up in telemetry dashboards and
|
|
1020
|
-
// are very misleading, as first initial reaction - some logic is broken.
|
|
1021
|
-
this.close();
|
|
1022
|
-
}
|
|
1023
|
-
// If closed then we can't reconnect
|
|
1024
|
-
if (this.closed) {
|
|
1025
|
-
return;
|
|
1026
|
-
}
|
|
1027
|
-
if (this.reconnectMode === ReconnectMode.Enabled) {
|
|
1028
|
-
const delayMs = error !== undefined ? driver_utils_1.getRetryDelayFromError(error) : undefined;
|
|
1029
|
-
if (delayMs !== undefined) {
|
|
1030
|
-
this.emitDelayInfo(this.deltaStreamDelayId, delayMs, error);
|
|
1031
|
-
await driver_utils_1.waitForConnectedState(delayMs);
|
|
1032
|
-
}
|
|
1033
|
-
this.triggerConnect({ reason: "reconnect", mode: requestedMode, fetchOpsFromStorage: false });
|
|
1034
|
-
}
|
|
1035
|
-
}
|
|
1036
433
|
// returns parts of message (in string format) that should never change for a given message.
|
|
1037
434
|
// Used for message comparison. It attempts to avoid comparing fields that potentially may differ.
|
|
1038
435
|
// for example, it's not clear if serverMetadata or timestamp property is a property of message or server state.
|
|
@@ -1044,7 +441,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1044
441
|
return `${m.clientId}-${m.type}-${m.minimumSequenceNumber}-${m.referenceSequenceNumber}-${m.timestamp}`;
|
|
1045
442
|
}
|
|
1046
443
|
enqueueMessages(messages, reason, allowGaps = false) {
|
|
1047
|
-
var _a, _b
|
|
444
|
+
var _a, _b;
|
|
1048
445
|
if (this.handler === undefined) {
|
|
1049
446
|
// We did not setup handler yet.
|
|
1050
447
|
// This happens when we connect to web socket faster than we get attributes for container
|
|
@@ -1104,7 +501,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1104
501
|
// correctly take into account pending ops.
|
|
1105
502
|
if (eventName !== undefined) {
|
|
1106
503
|
this.logger.sendPerformanceEvent(Object.assign({ eventName,
|
|
1107
|
-
reason, previousReason: this.prevEnqueueMessagesReason, from, to: last + 1, length: messages.length, fetchReason: this.fetchReason, duplicate: duplicate > 0 ? duplicate : undefined, initialGap: initialGap !== 0 ? initialGap : undefined, gap: gap > 0 ? gap : undefined, firstMissing, dmInitialSeqNumber: this.initialSequenceNumber }, this.
|
|
504
|
+
reason, previousReason: this.prevEnqueueMessagesReason, from, to: last + 1, length: messages.length, fetchReason: this.fetchReason, duplicate: duplicate > 0 ? duplicate : undefined, initialGap: initialGap !== 0 ? initialGap : undefined, gap: gap > 0 ? gap : undefined, firstMissing, dmInitialSeqNumber: this.initialSequenceNumber }, this.connectionManager.connectionVerboseProps));
|
|
1108
505
|
}
|
|
1109
506
|
}
|
|
1110
507
|
this.updateLatestKnownOpSeqNumber(messages[messages.length - 1].sequenceNumber);
|
|
@@ -1120,7 +517,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1120
517
|
const message2 = this.comparableMessagePayload(message);
|
|
1121
518
|
if (message1 !== message2) {
|
|
1122
519
|
const error = new driver_utils_1.NonRetryableError("twoMessagesWithSameSeqNumAndDifferentPayload", undefined, driver_definitions_1.DriverErrorType.fileOverwrittenInStorage, {
|
|
1123
|
-
clientId:
|
|
520
|
+
clientId: this.connectionManager.clientId,
|
|
1124
521
|
sequenceNumber: message.sequenceNumber,
|
|
1125
522
|
message1,
|
|
1126
523
|
message2,
|
|
@@ -1131,7 +528,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1131
528
|
}
|
|
1132
529
|
else if (message.sequenceNumber !== this.lastQueuedSequenceNumber + 1) {
|
|
1133
530
|
this.pending.push(message);
|
|
1134
|
-
this.fetchMissingDeltas(reason,
|
|
531
|
+
this.fetchMissingDeltas(reason, message.sequenceNumber);
|
|
1135
532
|
}
|
|
1136
533
|
else {
|
|
1137
534
|
this.lastQueuedSequenceNumber = message.sequenceNumber;
|
|
@@ -1145,56 +542,34 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1145
542
|
this.prevEnqueueMessagesReason = this.pending.length > 0 ? "unknown" : reason;
|
|
1146
543
|
}
|
|
1147
544
|
processInboundMessage(message) {
|
|
1148
|
-
var _a, _b, _c;
|
|
1149
545
|
const startTime = Date.now();
|
|
1150
546
|
this.lastProcessedMessage = message;
|
|
1151
547
|
// All non-system messages are coming from some client, and should have clientId
|
|
1152
548
|
// System messages may have no clientId (but some do, like propose, noop, summarize)
|
|
1153
549
|
common_utils_1.assert(message.clientId !== undefined
|
|
1154
550
|
|| protocol_base_1.isSystemMessage(message), 0x0ed /* "non-system message have to have clientId" */);
|
|
1155
|
-
// if we have connection, and message is local, then we better treat is as local!
|
|
1156
|
-
common_utils_1.assert(this.connection === undefined
|
|
1157
|
-
|| this.connection.clientId !== message.clientId
|
|
1158
|
-
|| this.lastSubmittedClientId === message.clientId, 0x0ee /* "Not accounting local messages correctly" */);
|
|
1159
|
-
if (this.lastSubmittedClientId !== undefined && this.lastSubmittedClientId === message.clientId) {
|
|
1160
|
-
const clientSequenceNumber = message.clientSequenceNumber;
|
|
1161
|
-
common_utils_1.assert(this.clientSequenceNumberObserved < clientSequenceNumber, 0x0ef /* "client seq# not growing" */);
|
|
1162
|
-
common_utils_1.assert(clientSequenceNumber <= this.clientSequenceNumber, 0x0f0 /* "Incoming local client seq# > generated by this client" */);
|
|
1163
|
-
this.clientSequenceNumberObserved = clientSequenceNumber;
|
|
1164
|
-
}
|
|
1165
551
|
// TODO Remove after SPO picks up the latest build.
|
|
1166
552
|
if (typeof message.contents === "string"
|
|
1167
553
|
&& message.contents !== ""
|
|
1168
554
|
&& message.type !== protocol_definitions_1.MessageType.ClientLeave) {
|
|
1169
555
|
message.contents = JSON.parse(message.contents);
|
|
1170
556
|
}
|
|
1171
|
-
|
|
1172
|
-
const systemLeaveMessage = message;
|
|
1173
|
-
const clientId = JSON.parse(systemLeaveMessage.data);
|
|
1174
|
-
if (clientId === ((_a = this.connection) === null || _a === void 0 ? void 0 : _a.clientId)) {
|
|
1175
|
-
// We have been kicked out from quorum
|
|
1176
|
-
this.logger.sendPerformanceEvent({ eventName: "ReadConnectionTransition" });
|
|
1177
|
-
this.downgradedConnection = true;
|
|
1178
|
-
}
|
|
1179
|
-
}
|
|
557
|
+
this.connectionManager.beforeProcessingIncomingOp(message);
|
|
1180
558
|
// Add final ack trace.
|
|
1181
559
|
if (message.traces !== undefined && message.traces.length > 0) {
|
|
1182
|
-
const service = this.clientDetails.type === undefined || this.clientDetails.type === ""
|
|
1183
|
-
? "unknown"
|
|
1184
|
-
: this.clientDetails.type;
|
|
1185
560
|
message.traces.push({
|
|
1186
561
|
action: "end",
|
|
1187
|
-
service,
|
|
562
|
+
service: "client",
|
|
1188
563
|
timestamp: Date.now(),
|
|
1189
564
|
});
|
|
1190
565
|
}
|
|
1191
566
|
// Watch the minimum sequence number and be ready to update as needed
|
|
1192
567
|
if (this.minSequenceNumber > message.minimumSequenceNumber) {
|
|
1193
|
-
throw new container_utils_1.DataCorruptionError("msnMovesBackwards", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId:
|
|
568
|
+
throw new container_utils_1.DataCorruptionError("msnMovesBackwards", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: this.connectionManager.clientId }));
|
|
1194
569
|
}
|
|
1195
570
|
this.minSequenceNumber = message.minimumSequenceNumber;
|
|
1196
571
|
if (message.sequenceNumber !== this.lastProcessedSequenceNumber + 1) {
|
|
1197
|
-
throw new container_utils_1.DataCorruptionError("nonSequentialSequenceNumber", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId:
|
|
572
|
+
throw new container_utils_1.DataCorruptionError("nonSequentialSequenceNumber", Object.assign(Object.assign({}, extractLogSafeMessageProperties(message)), { clientId: this.connectionManager.clientId }));
|
|
1198
573
|
}
|
|
1199
574
|
this.lastProcessedSequenceNumber = message.sequenceNumber;
|
|
1200
575
|
// a bunch of code assumes that this is true
|
|
@@ -1216,15 +591,15 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1216
591
|
/**
|
|
1217
592
|
* Retrieves the missing deltas between the given sequence numbers
|
|
1218
593
|
*/
|
|
1219
|
-
fetchMissingDeltas(reasonArg,
|
|
1220
|
-
this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */,
|
|
594
|
+
fetchMissingDeltas(reasonArg, to) {
|
|
595
|
+
this.fetchMissingDeltasCore(reasonArg, false /* cacheOnly */, to).catch((error) => {
|
|
1221
596
|
this.logger.sendErrorEvent({ eventName: "fetchMissingDeltasException" }, error);
|
|
1222
597
|
});
|
|
1223
598
|
}
|
|
1224
599
|
/**
|
|
1225
600
|
* Retrieves the missing deltas between the given sequence numbers
|
|
1226
601
|
*/
|
|
1227
|
-
async fetchMissingDeltasCore(reason, cacheOnly,
|
|
602
|
+
async fetchMissingDeltasCore(reason, cacheOnly, to) {
|
|
1228
603
|
var _a;
|
|
1229
604
|
// Exit out early if we're already fetching deltas
|
|
1230
605
|
if (this.fetchReason !== undefined) {
|
|
@@ -1236,12 +611,11 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1236
611
|
}
|
|
1237
612
|
if (this.handler === undefined) {
|
|
1238
613
|
// We do not poses yet any information
|
|
1239
|
-
common_utils_1.assert(
|
|
614
|
+
common_utils_1.assert(this.lastQueuedSequenceNumber === 0, 0x26b /* "initial state" */);
|
|
1240
615
|
return;
|
|
1241
616
|
}
|
|
1242
617
|
try {
|
|
1243
|
-
|
|
1244
|
-
let from = lastKnowOp + 1;
|
|
618
|
+
let from = this.lastQueuedSequenceNumber + 1;
|
|
1245
619
|
const n = (_a = this.previouslyProcessedMessage) === null || _a === void 0 ? void 0 : _a.sequenceNumber;
|
|
1246
620
|
if (n !== undefined) {
|
|
1247
621
|
// If we already processed at least one op, then we have this.previouslyProcessedMessage populated
|
|
@@ -1249,7 +623,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1249
623
|
// Knowing about this mechanism, we could ask for op we already observed to increase validation.
|
|
1250
624
|
// This is especially useful when coming out of offline mode or loading from
|
|
1251
625
|
// very old cached (by client / driver) snapshot.
|
|
1252
|
-
common_utils_1.assert(n ===
|
|
626
|
+
common_utils_1.assert(n === this.lastQueuedSequenceNumber, 0x0f2 /* "previouslyProcessedMessage" */);
|
|
1253
627
|
common_utils_1.assert(from > 1, 0x0f3 /* "not positive" */);
|
|
1254
628
|
from--;
|
|
1255
629
|
}
|
|
@@ -1297,7 +671,7 @@ class DeltaManager extends common_utils_1.TypedEventEmitter {
|
|
|
1297
671
|
// (the other 50%), and thus these errors below should be looked at even if code below results in
|
|
1298
672
|
// recovery.
|
|
1299
673
|
if (this.lastQueuedSequenceNumber < this.lastObservedSeqNumber) {
|
|
1300
|
-
this.fetchMissingDeltas("OpsBehind"
|
|
674
|
+
this.fetchMissingDeltas("OpsBehind");
|
|
1301
675
|
}
|
|
1302
676
|
}
|
|
1303
677
|
}
|