@fluidframework/container-runtime 0.59.4000-71130 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.eslintrc.js +1 -1
- package/dist/blobManager.d.ts +2 -2
- package/dist/blobManager.d.ts.map +1 -1
- package/dist/blobManager.js +12 -11
- package/dist/blobManager.js.map +1 -1
- package/dist/connectionTelemetry.js +3 -3
- package/dist/connectionTelemetry.js.map +1 -1
- package/dist/containerRuntime.d.ts +125 -29
- package/dist/containerRuntime.d.ts.map +1 -1
- package/dist/containerRuntime.js +242 -110
- package/dist/containerRuntime.js.map +1 -1
- package/dist/dataStoreContext.d.ts +4 -2
- package/dist/dataStoreContext.d.ts.map +1 -1
- package/dist/dataStoreContext.js +16 -5
- package/dist/dataStoreContext.js.map +1 -1
- package/dist/dataStores.d.ts +4 -3
- package/dist/dataStores.d.ts.map +1 -1
- package/dist/dataStores.js +9 -3
- package/dist/dataStores.js.map +1 -1
- package/dist/garbageCollection.d.ts +14 -3
- package/dist/garbageCollection.d.ts.map +1 -1
- package/dist/garbageCollection.js +56 -26
- package/dist/garbageCollection.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/orderedClientElection.js +0 -4
- package/dist/orderedClientElection.js.map +1 -1
- package/dist/packageVersion.d.ts +1 -1
- package/dist/packageVersion.d.ts.map +1 -1
- package/dist/packageVersion.js +1 -1
- package/dist/packageVersion.js.map +1 -1
- package/dist/pendingStateManager.d.ts +30 -29
- package/dist/pendingStateManager.d.ts.map +1 -1
- package/dist/pendingStateManager.js +72 -109
- package/dist/pendingStateManager.js.map +1 -1
- package/dist/runningSummarizer.d.ts +4 -3
- package/dist/runningSummarizer.d.ts.map +1 -1
- package/dist/runningSummarizer.js +11 -6
- package/dist/runningSummarizer.js.map +1 -1
- package/dist/serializedSnapshotStorage.d.ts +58 -0
- package/dist/serializedSnapshotStorage.d.ts.map +1 -0
- package/dist/serializedSnapshotStorage.js +108 -0
- package/dist/serializedSnapshotStorage.js.map +1 -0
- package/dist/summarizer.d.ts +11 -4
- package/dist/summarizer.d.ts.map +1 -1
- package/dist/summarizer.js +18 -9
- package/dist/summarizer.js.map +1 -1
- package/dist/summarizerHeuristics.d.ts +5 -3
- package/dist/summarizerHeuristics.d.ts.map +1 -1
- package/dist/summarizerHeuristics.js +10 -3
- package/dist/summarizerHeuristics.js.map +1 -1
- package/dist/summarizerTypes.d.ts +4 -2
- package/dist/summarizerTypes.d.ts.map +1 -1
- package/dist/summarizerTypes.js.map +1 -1
- package/dist/summaryManager.d.ts +3 -3
- package/dist/summaryManager.d.ts.map +1 -1
- package/dist/summaryManager.js +7 -7
- package/dist/summaryManager.js.map +1 -1
- package/garbageCollection.md +9 -1
- package/lib/blobManager.d.ts +2 -2
- package/lib/blobManager.d.ts.map +1 -1
- package/lib/blobManager.js +12 -11
- package/lib/blobManager.js.map +1 -1
- package/lib/connectionTelemetry.js +3 -3
- package/lib/connectionTelemetry.js.map +1 -1
- package/lib/containerRuntime.d.ts +125 -29
- package/lib/containerRuntime.d.ts.map +1 -1
- package/lib/containerRuntime.js +243 -111
- package/lib/containerRuntime.js.map +1 -1
- package/lib/dataStoreContext.d.ts +4 -2
- package/lib/dataStoreContext.d.ts.map +1 -1
- package/lib/dataStoreContext.js +16 -5
- package/lib/dataStoreContext.js.map +1 -1
- package/lib/dataStores.d.ts +4 -3
- package/lib/dataStores.d.ts.map +1 -1
- package/lib/dataStores.js +9 -3
- package/lib/dataStores.js.map +1 -1
- package/lib/garbageCollection.d.ts +14 -3
- package/lib/garbageCollection.d.ts.map +1 -1
- package/lib/garbageCollection.js +54 -6
- package/lib/garbageCollection.js.map +1 -1
- package/lib/index.d.ts +2 -2
- package/lib/index.d.ts.map +1 -1
- package/lib/index.js +1 -1
- package/lib/index.js.map +1 -1
- package/lib/orderedClientElection.js +0 -4
- package/lib/orderedClientElection.js.map +1 -1
- package/lib/packageVersion.d.ts +1 -1
- package/lib/packageVersion.d.ts.map +1 -1
- package/lib/packageVersion.js +1 -1
- package/lib/packageVersion.js.map +1 -1
- package/lib/pendingStateManager.d.ts +30 -29
- package/lib/pendingStateManager.d.ts.map +1 -1
- package/lib/pendingStateManager.js +72 -109
- package/lib/pendingStateManager.js.map +1 -1
- package/lib/runningSummarizer.d.ts +4 -3
- package/lib/runningSummarizer.d.ts.map +1 -1
- package/lib/runningSummarizer.js +11 -6
- package/lib/runningSummarizer.js.map +1 -1
- package/lib/serializedSnapshotStorage.d.ts +58 -0
- package/lib/serializedSnapshotStorage.d.ts.map +1 -0
- package/lib/serializedSnapshotStorage.js +104 -0
- package/lib/serializedSnapshotStorage.js.map +1 -0
- package/lib/summarizer.d.ts +11 -4
- package/lib/summarizer.d.ts.map +1 -1
- package/lib/summarizer.js +18 -9
- package/lib/summarizer.js.map +1 -1
- package/lib/summarizerHeuristics.d.ts +5 -3
- package/lib/summarizerHeuristics.d.ts.map +1 -1
- package/lib/summarizerHeuristics.js +10 -3
- package/lib/summarizerHeuristics.js.map +1 -1
- package/lib/summarizerTypes.d.ts +4 -2
- package/lib/summarizerTypes.d.ts.map +1 -1
- package/lib/summarizerTypes.js.map +1 -1
- package/lib/summaryManager.d.ts +3 -3
- package/lib/summaryManager.d.ts.map +1 -1
- package/lib/summaryManager.js +7 -7
- package/lib/summaryManager.js.map +1 -1
- package/package.json +47 -33
- package/src/blobManager.ts +29 -15
- package/src/connectionTelemetry.ts +3 -3
- package/src/containerRuntime.ts +388 -135
- package/src/dataStoreContext.ts +27 -5
- package/src/dataStores.ts +15 -3
- package/src/garbageCollection.ts +69 -12
- package/src/index.ts +7 -1
- package/src/orderedClientElection.ts +1 -1
- package/src/packageVersion.ts +1 -1
- package/src/pendingStateManager.ts +104 -123
- package/src/runningSummarizer.ts +20 -10
- package/src/serializedSnapshotStorage.ts +146 -0
- package/src/summarizer.ts +20 -16
- package/src/summarizerHeuristics.ts +21 -5
- package/src/summarizerTypes.ts +4 -2
- package/src/summaryManager.ts +5 -6
|
@@ -5,13 +5,15 @@
|
|
|
5
5
|
|
|
6
6
|
import { IDisposable } from "@fluidframework/common-definitions";
|
|
7
7
|
import { assert, Lazy } from "@fluidframework/common-utils";
|
|
8
|
+
import { ICriticalContainerError } from "@fluidframework/container-definitions";
|
|
8
9
|
import { DataProcessingError } from "@fluidframework/container-utils";
|
|
9
10
|
import {
|
|
10
11
|
ISequencedDocumentMessage,
|
|
11
12
|
} from "@fluidframework/protocol-definitions";
|
|
12
13
|
import { FlushMode } from "@fluidframework/runtime-definitions";
|
|
14
|
+
import { wrapError } from "@fluidframework/telemetry-utils";
|
|
13
15
|
import Deque from "double-ended-queue";
|
|
14
|
-
import {
|
|
16
|
+
import { ContainerMessageType } from "./containerRuntime";
|
|
15
17
|
|
|
16
18
|
/**
|
|
17
19
|
* This represents a message that has been submitted and is added to the pending queue when `submit` is called on the
|
|
@@ -47,16 +49,31 @@ export interface IPendingFlush {
|
|
|
47
49
|
export type IPendingState = IPendingMessage | IPendingFlushMode | IPendingFlush;
|
|
48
50
|
|
|
49
51
|
export interface IPendingLocalState {
|
|
50
|
-
/**
|
|
51
|
-
* client ID we most recently connected with, or undefined if we never connected
|
|
52
|
-
*/
|
|
53
|
-
clientId?: string;
|
|
54
52
|
/**
|
|
55
53
|
* list of pending states, including ops and batch information
|
|
56
54
|
*/
|
|
57
55
|
pendingStates: IPendingState[];
|
|
58
56
|
}
|
|
59
57
|
|
|
58
|
+
export interface IRuntimeStateHandler{
|
|
59
|
+
connected(): boolean;
|
|
60
|
+
clientId(): string | undefined;
|
|
61
|
+
flushMode(): FlushMode;
|
|
62
|
+
setFlushMode(mode: FlushMode): void;
|
|
63
|
+
close(error?: ICriticalContainerError): void;
|
|
64
|
+
applyStashedOp: (type: ContainerMessageType, content: ISequencedDocumentMessage) => Promise<unknown>;
|
|
65
|
+
flush(): void;
|
|
66
|
+
reSubmit(
|
|
67
|
+
type: ContainerMessageType,
|
|
68
|
+
content: any,
|
|
69
|
+
localOpMetadata: unknown,
|
|
70
|
+
opMetadata: Record<string, unknown> | undefined): void;
|
|
71
|
+
rollback(
|
|
72
|
+
type: ContainerMessageType,
|
|
73
|
+
content: any,
|
|
74
|
+
localOpMetadata: unknown): void;
|
|
75
|
+
}
|
|
76
|
+
|
|
60
77
|
/**
|
|
61
78
|
* PendingStateManager is responsible for maintaining the messages that have not been sent or have not yet been
|
|
62
79
|
* acknowledged by the server. It also maintains the batch information for both automatically and manually flushed
|
|
@@ -69,15 +86,16 @@ export interface IPendingLocalState {
|
|
|
69
86
|
export class PendingStateManager implements IDisposable {
|
|
70
87
|
private readonly pendingStates = new Deque<IPendingState>();
|
|
71
88
|
private readonly initialStates: Deque<IPendingState>;
|
|
72
|
-
private readonly previousClientIds = new Set<string>();
|
|
73
|
-
private readonly firstStashedCSN: number = -1;
|
|
74
89
|
private readonly disposeOnce = new Lazy<void>(() => {
|
|
75
90
|
this.initialStates.clear();
|
|
76
91
|
this.pendingStates.clear();
|
|
77
92
|
});
|
|
78
93
|
|
|
79
94
|
// Maintains the count of messages that are currently unacked.
|
|
80
|
-
private
|
|
95
|
+
private _pendingMessagesCount: number = 0;
|
|
96
|
+
public get pendingMessagesCount(): number {
|
|
97
|
+
return this._pendingMessagesCount;
|
|
98
|
+
}
|
|
81
99
|
|
|
82
100
|
// Indicates whether we are processing a batch.
|
|
83
101
|
private isProcessingBatch: boolean = false;
|
|
@@ -95,22 +113,18 @@ export class PendingStateManager implements IDisposable {
|
|
|
95
113
|
|
|
96
114
|
private clientId: string | undefined;
|
|
97
115
|
|
|
98
|
-
private get connected(): boolean {
|
|
99
|
-
return this.containerRuntime.connected;
|
|
100
|
-
}
|
|
101
|
-
|
|
102
116
|
/**
|
|
103
117
|
* Called to check if there are any pending messages in the pending state queue.
|
|
104
118
|
* @returns A boolean indicating whether there are messages or not.
|
|
105
119
|
*/
|
|
106
120
|
public hasPendingMessages(): boolean {
|
|
107
|
-
return this.
|
|
121
|
+
return this._pendingMessagesCount !== 0 || !this.initialStates.isEmpty();
|
|
108
122
|
}
|
|
109
123
|
|
|
110
124
|
public getLocalState(): IPendingLocalState | undefined {
|
|
125
|
+
assert(this.initialStates.isEmpty(), 0x2e9 /* "Must call getLocalState() after applying initial states" */);
|
|
111
126
|
if (this.hasPendingMessages()) {
|
|
112
127
|
return {
|
|
113
|
-
clientId: this.clientId,
|
|
114
128
|
pendingStates: this.pendingStates.toArray().map(
|
|
115
129
|
// delete localOpMetadata since it may not be serializable
|
|
116
130
|
// and will be regenerated by applyStashedOp()
|
|
@@ -120,23 +134,12 @@ export class PendingStateManager implements IDisposable {
|
|
|
120
134
|
}
|
|
121
135
|
|
|
122
136
|
constructor(
|
|
123
|
-
private readonly
|
|
124
|
-
private readonly applyStashedOp: (type, content) => Promise<unknown>,
|
|
137
|
+
private readonly stateHandler: IRuntimeStateHandler,
|
|
125
138
|
initialFlushMode: FlushMode,
|
|
126
139
|
initialLocalState: IPendingLocalState | undefined,
|
|
127
140
|
) {
|
|
128
141
|
this.initialStates = new Deque<IPendingState>(initialLocalState?.pendingStates ?? []);
|
|
129
142
|
|
|
130
|
-
if (initialLocalState) {
|
|
131
|
-
if (initialLocalState?.clientId) {
|
|
132
|
-
this.previousClientIds.add(initialLocalState.clientId);
|
|
133
|
-
}
|
|
134
|
-
// get stashed op count and client sequence number of first op
|
|
135
|
-
const messages = initialLocalState.pendingStates
|
|
136
|
-
.filter((state) => state.type === "message") as IPendingMessage[];
|
|
137
|
-
this.firstStashedCSN = messages[0].clientSequenceNumber;
|
|
138
|
-
}
|
|
139
|
-
|
|
140
143
|
this.flushModeForNextMessage = initialFlushMode;
|
|
141
144
|
this.onFlushModeUpdated(initialFlushMode);
|
|
142
145
|
}
|
|
@@ -172,7 +175,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
172
175
|
|
|
173
176
|
this.pendingStates.push(pendingMessage);
|
|
174
177
|
|
|
175
|
-
this.
|
|
178
|
+
this._pendingMessagesCount++;
|
|
176
179
|
}
|
|
177
180
|
|
|
178
181
|
/**
|
|
@@ -189,7 +192,7 @@ export class PendingStateManager implements IDisposable {
|
|
|
189
192
|
public onFlush() {
|
|
190
193
|
// If the FlushMode is Immediate, we don't need to track an explicit flush call because every message is
|
|
191
194
|
// automatically flushed. So, flush is a no-op.
|
|
192
|
-
if (this.
|
|
195
|
+
if (this.stateHandler.flushMode() === FlushMode.Immediate) {
|
|
193
196
|
return;
|
|
194
197
|
}
|
|
195
198
|
|
|
@@ -205,21 +208,25 @@ export class PendingStateManager implements IDisposable {
|
|
|
205
208
|
|
|
206
209
|
/**
|
|
207
210
|
* Applies stashed ops at their reference sequence number so they are ready to be ACKed or resubmitted
|
|
211
|
+
* @param seqNum - Sequence number at which to apply ops. Will apply all ops if seqNum is undefined.
|
|
208
212
|
*/
|
|
209
|
-
public async applyStashedOpsAt(seqNum
|
|
213
|
+
public async applyStashedOpsAt(seqNum?: number) {
|
|
210
214
|
// apply stashed ops at sequence number
|
|
211
215
|
while (!this.initialStates.isEmpty()) {
|
|
212
216
|
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
213
217
|
const nextState = this.initialStates.peekFront()!;
|
|
214
218
|
if (nextState.type === "message") {
|
|
215
|
-
if (
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
+
if (seqNum !== undefined) {
|
|
220
|
+
if (nextState.referenceSequenceNumber > seqNum) {
|
|
221
|
+
break; // nothing left to do at this sequence number
|
|
222
|
+
} else if (nextState.referenceSequenceNumber < seqNum) {
|
|
223
|
+
throw new Error("loaded from snapshot too recent to apply stashed ops");
|
|
224
|
+
}
|
|
219
225
|
}
|
|
220
226
|
|
|
221
227
|
// applyStashedOp will cause the DDS to behave as if it has sent the op but not actually send it
|
|
222
|
-
const localOpMetadata =
|
|
228
|
+
const localOpMetadata =
|
|
229
|
+
await this.stateHandler.applyStashedOp(nextState.messageType, nextState.content);
|
|
223
230
|
nextState.localOpMetadata = localOpMetadata;
|
|
224
231
|
}
|
|
225
232
|
|
|
@@ -229,88 +236,12 @@ export class PendingStateManager implements IDisposable {
|
|
|
229
236
|
}
|
|
230
237
|
}
|
|
231
238
|
|
|
232
|
-
/**
|
|
233
|
-
* Processes a local message once it's ack'd by the server to verify that there was no data corruption and that
|
|
234
|
-
* the batch information was preserved for batch messages. Also process remote messages that might have been
|
|
235
|
-
* sent from a previous container.
|
|
236
|
-
* @param message - The message that got ack'd and needs to be processed.
|
|
237
|
-
*/
|
|
238
|
-
public processMessage(message: ISequencedDocumentMessage, local: boolean) {
|
|
239
|
-
// Do not process chunked ops until all pieces are available.
|
|
240
|
-
if (message.type === ContainerMessageType.ChunkedOp) {
|
|
241
|
-
return { localAck: false, localOpMetadata: undefined };
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
if (local) {
|
|
245
|
-
return { localAck: false, localOpMetadata: this.processPendingLocalMessage(message) };
|
|
246
|
-
} else {
|
|
247
|
-
return this.processRemoteMessage(message);
|
|
248
|
-
}
|
|
249
|
-
}
|
|
250
|
-
|
|
251
|
-
/**
|
|
252
|
-
* Listens for ACKs of stashed ops
|
|
253
|
-
*/
|
|
254
|
-
private processRemoteMessage(message: ISequencedDocumentMessage) {
|
|
255
|
-
if (!isRuntimeMessage(message)) {
|
|
256
|
-
return { localAck: false, localOpMetadata: undefined };
|
|
257
|
-
}
|
|
258
|
-
|
|
259
|
-
// this message was a pending op that was actually sent successfully
|
|
260
|
-
const isOriginalClientId = message.clientId === Array.from(this.previousClientIds)[0] &&
|
|
261
|
-
message.clientSequenceNumber >= this.firstStashedCSN;
|
|
262
|
-
// this message is a pending or stashed op that was resubmitted
|
|
263
|
-
const isNewClientId = Array.from(this.previousClientIds).indexOf(message.clientId) > 0;
|
|
264
|
-
|
|
265
|
-
// if this is an ack for a stashed op, dequeue one message.
|
|
266
|
-
// we should have seen its ref seq num by now and the DDS should be ready for it to be ACKed
|
|
267
|
-
if (isOriginalClientId || isNewClientId) {
|
|
268
|
-
assert(this.clientId === undefined, 0x28b /* "multiple clients connected with stashed ops" */);
|
|
269
|
-
while (!this.pendingStates.isEmpty()) {
|
|
270
|
-
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
271
|
-
const nextState = this.pendingStates.shift()!;
|
|
272
|
-
// if it's not a message just drop it and keep looking
|
|
273
|
-
if (nextState.type === "message") {
|
|
274
|
-
this.assertOpMatch(nextState, message, isOriginalClientId);
|
|
275
|
-
return { localAck: true, localOpMetadata: nextState.localOpMetadata };
|
|
276
|
-
}
|
|
277
|
-
}
|
|
278
|
-
}
|
|
279
|
-
|
|
280
|
-
if (message.type === ContainerMessageType.Rejoin && this.previousClientIds.has(message.contents?.clientId)) {
|
|
281
|
-
this.previousClientIds.add(message.clientId);
|
|
282
|
-
}
|
|
283
|
-
|
|
284
|
-
return { localAck: false, localOpMetadata: undefined };
|
|
285
|
-
}
|
|
286
|
-
|
|
287
|
-
private assertOpMatch(state: IPendingMessage, message: ISequencedDocumentMessage, isOriginalClientId: boolean) {
|
|
288
|
-
assert(message.type === state.messageType, 0x28c /* "different message type" */);
|
|
289
|
-
assert(message.clientSequenceNumber === state.clientSequenceNumber || !isOriginalClientId,
|
|
290
|
-
0x28d /* "client sequence number doesn't match" */);
|
|
291
|
-
switch (message.type) {
|
|
292
|
-
case ContainerMessageType.Attach:
|
|
293
|
-
assert(message.contents.id === state.content.id, 0x28e /* "datastore ID doesn't match" */);
|
|
294
|
-
break;
|
|
295
|
-
case ContainerMessageType.FluidDataStoreOp:
|
|
296
|
-
assert(message.contents.address === state.content.address, 0x28f /* "address doesn't match" */);
|
|
297
|
-
break;
|
|
298
|
-
case ContainerMessageType.BlobAttach:
|
|
299
|
-
// todo: assert we have blob storage, assert blob IDs match, remove blob from blob storage since it made
|
|
300
|
-
// it through successfully
|
|
301
|
-
break;
|
|
302
|
-
case ContainerMessageType.Rejoin:
|
|
303
|
-
default:
|
|
304
|
-
throw new Error(`${message.type} not expected`);
|
|
305
|
-
}
|
|
306
|
-
}
|
|
307
|
-
|
|
308
239
|
/**
|
|
309
240
|
* Processes a local message once its ack'd by the server. It verifies that there was no data corruption and that
|
|
310
241
|
* the batch information was preserved for batch messages.
|
|
311
242
|
* @param message - The message that got ack'd and needs to be processed.
|
|
312
243
|
*/
|
|
313
|
-
|
|
244
|
+
public processPendingLocalMessage(message: ISequencedDocumentMessage): unknown {
|
|
314
245
|
// Pre-processing part - This may be the start of a batch.
|
|
315
246
|
this.maybeProcessBatchBegin(message);
|
|
316
247
|
|
|
@@ -330,11 +261,11 @@ export class PendingStateManager implements IDisposable {
|
|
|
330
261
|
{ expectedClientSequenceNumber: pendingState.clientSequenceNumber },
|
|
331
262
|
);
|
|
332
263
|
|
|
333
|
-
this.
|
|
264
|
+
this.stateHandler.close(error);
|
|
334
265
|
return;
|
|
335
266
|
}
|
|
336
267
|
|
|
337
|
-
this.
|
|
268
|
+
this._pendingMessagesCount--;
|
|
338
269
|
|
|
339
270
|
// Post-processing part - If we are processing a batch then this could be the last message in the batch.
|
|
340
271
|
this.maybeProcessBatchEnd(message);
|
|
@@ -446,6 +377,31 @@ export class PendingStateManager implements IDisposable {
|
|
|
446
377
|
this.isProcessingBatch = false;
|
|
447
378
|
}
|
|
448
379
|
|
|
380
|
+
/**
|
|
381
|
+
* Capture the pending state at this point
|
|
382
|
+
*/
|
|
383
|
+
public checkpoint() {
|
|
384
|
+
const checkpointHead = this.pendingStates.peekBack();
|
|
385
|
+
return {
|
|
386
|
+
rollback: () => {
|
|
387
|
+
try {
|
|
388
|
+
while (this.pendingStates.peekBack() !== checkpointHead) {
|
|
389
|
+
this.rollbackNextPendingState();
|
|
390
|
+
}
|
|
391
|
+
} catch (err) {
|
|
392
|
+
const error = wrapError(err, (message) => {
|
|
393
|
+
return DataProcessingError.create(
|
|
394
|
+
`RollbackError: ${message}`,
|
|
395
|
+
"checkpointRollback",
|
|
396
|
+
undefined) as DataProcessingError;
|
|
397
|
+
});
|
|
398
|
+
this.stateHandler.close(error);
|
|
399
|
+
throw error;
|
|
400
|
+
}
|
|
401
|
+
},
|
|
402
|
+
};
|
|
403
|
+
}
|
|
404
|
+
|
|
449
405
|
/**
|
|
450
406
|
* Returns the next pending state from the pending state queue.
|
|
451
407
|
*/
|
|
@@ -455,17 +411,42 @@ export class PendingStateManager implements IDisposable {
|
|
|
455
411
|
return nextPendingState;
|
|
456
412
|
}
|
|
457
413
|
|
|
414
|
+
/**
|
|
415
|
+
* Undo the last pending state
|
|
416
|
+
*/
|
|
417
|
+
private rollbackNextPendingState() {
|
|
418
|
+
const pendingStatesCount = this.pendingStates.length;
|
|
419
|
+
if (pendingStatesCount === 0) {
|
|
420
|
+
return;
|
|
421
|
+
}
|
|
422
|
+
|
|
423
|
+
this._pendingMessagesCount--;
|
|
424
|
+
|
|
425
|
+
// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
|
|
426
|
+
const pendingState = this.pendingStates.pop()!;
|
|
427
|
+
switch (pendingState.type) {
|
|
428
|
+
case "message":
|
|
429
|
+
this.stateHandler.rollback(
|
|
430
|
+
pendingState.messageType,
|
|
431
|
+
pendingState.content,
|
|
432
|
+
pendingState.localOpMetadata);
|
|
433
|
+
break;
|
|
434
|
+
default:
|
|
435
|
+
throw new Error(`Can't rollback state ${pendingState.type}`);
|
|
436
|
+
}
|
|
437
|
+
}
|
|
438
|
+
|
|
458
439
|
/**
|
|
459
440
|
* Called when the Container's connection state changes. If the Container gets connected, it replays all the pending
|
|
460
441
|
* states in its queue. This includes setting the FlushMode and triggering resubmission of unacked ops.
|
|
461
442
|
*/
|
|
462
443
|
public replayPendingStates() {
|
|
463
|
-
assert(this.connected, 0x172 /* "The connection state is not consistent with the runtime" */);
|
|
444
|
+
assert(this.stateHandler.connected(), 0x172 /* "The connection state is not consistent with the runtime" */);
|
|
464
445
|
|
|
465
446
|
// This assert suggests we are about to send same ops twice, which will result in data loss.
|
|
466
|
-
assert(this.clientId !== this.
|
|
447
|
+
assert(this.clientId !== this.stateHandler.clientId(),
|
|
467
448
|
0x173 /* "replayPendingStates called twice for same clientId!" */);
|
|
468
|
-
this.clientId = this.
|
|
449
|
+
this.clientId = this.stateHandler.clientId();
|
|
469
450
|
|
|
470
451
|
assert(this.initialStates.isEmpty(), 0x174 /* "initial states should be empty before replaying pending" */);
|
|
471
452
|
|
|
@@ -475,14 +456,14 @@ export class PendingStateManager implements IDisposable {
|
|
|
475
456
|
}
|
|
476
457
|
|
|
477
458
|
// Reset the pending message count because all these messages will be removed from the queue.
|
|
478
|
-
this.
|
|
459
|
+
this._pendingMessagesCount = 0;
|
|
479
460
|
|
|
480
461
|
// Save the current FlushMode so that we can revert it back after replaying the states.
|
|
481
|
-
const savedFlushMode = this.
|
|
462
|
+
const savedFlushMode = this.stateHandler.flushMode();
|
|
482
463
|
|
|
483
464
|
// Set the flush mode for the next message. This step is important because the flush mode may have been changed
|
|
484
465
|
// after the next pending message was sent.
|
|
485
|
-
this.
|
|
466
|
+
this.stateHandler.setFlushMode(this.flushModeForNextMessage);
|
|
486
467
|
|
|
487
468
|
// Process exactly `pendingStatesCount` items in the queue as it represents the number of states that were
|
|
488
469
|
// pending when we connected. This is important because the `reSubmitFn` might add more items in the queue
|
|
@@ -492,17 +473,17 @@ export class PendingStateManager implements IDisposable {
|
|
|
492
473
|
const pendingState = this.pendingStates.shift()!;
|
|
493
474
|
switch (pendingState.type) {
|
|
494
475
|
case "message":
|
|
495
|
-
this.
|
|
476
|
+
this.stateHandler.reSubmit(
|
|
496
477
|
pendingState.messageType,
|
|
497
478
|
pendingState.content,
|
|
498
479
|
pendingState.localOpMetadata,
|
|
499
480
|
pendingState.opMetadata);
|
|
500
481
|
break;
|
|
501
482
|
case "flushMode":
|
|
502
|
-
this.
|
|
483
|
+
this.stateHandler.setFlushMode(pendingState.flushMode);
|
|
503
484
|
break;
|
|
504
485
|
case "flush":
|
|
505
|
-
this.
|
|
486
|
+
this.stateHandler.flush();
|
|
506
487
|
break;
|
|
507
488
|
default:
|
|
508
489
|
break;
|
|
@@ -511,6 +492,6 @@ export class PendingStateManager implements IDisposable {
|
|
|
511
492
|
}
|
|
512
493
|
|
|
513
494
|
// Revert the FlushMode.
|
|
514
|
-
this.
|
|
495
|
+
this.stateHandler.setFlushMode(savedFlushMode);
|
|
515
496
|
}
|
|
516
497
|
}
|
package/src/runningSummarizer.ts
CHANGED
|
@@ -8,17 +8,18 @@ import { assert, delay, Deferred, PromiseTimer } from "@fluidframework/common-ut
|
|
|
8
8
|
import { UsageError } from "@fluidframework/container-utils";
|
|
9
9
|
import {
|
|
10
10
|
ISequencedDocumentMessage,
|
|
11
|
-
ISummaryConfiguration,
|
|
12
11
|
MessageType,
|
|
13
12
|
} from "@fluidframework/protocol-definitions";
|
|
14
13
|
import { ChildLogger } from "@fluidframework/telemetry-utils";
|
|
14
|
+
import {
|
|
15
|
+
ISummaryConfiguration,
|
|
16
|
+
} from "./containerRuntime";
|
|
15
17
|
import { SummarizeHeuristicRunner } from "./summarizerHeuristics";
|
|
16
18
|
import {
|
|
17
19
|
IEnqueueSummarizeOptions,
|
|
18
20
|
ISummarizeOptions,
|
|
19
21
|
ISummarizeHeuristicData,
|
|
20
22
|
ISummarizeHeuristicRunner,
|
|
21
|
-
ISummarizerOptions,
|
|
22
23
|
IOnDemandSummarizeOptions,
|
|
23
24
|
EnqueueSummarizeResult,
|
|
24
25
|
SummarizerStopReason,
|
|
@@ -57,7 +58,6 @@ export class RunningSummarizer implements IDisposable {
|
|
|
57
58
|
summaryCollection: SummaryCollection,
|
|
58
59
|
cancellationToken: ISummaryCancellationToken,
|
|
59
60
|
stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
60
|
-
options?: Readonly<Partial<ISummarizerOptions>>,
|
|
61
61
|
): Promise<RunningSummarizer> {
|
|
62
62
|
const summarizer = new RunningSummarizer(
|
|
63
63
|
logger,
|
|
@@ -68,8 +68,7 @@ export class RunningSummarizer implements IDisposable {
|
|
|
68
68
|
raiseSummarizingError,
|
|
69
69
|
summaryCollection,
|
|
70
70
|
cancellationToken,
|
|
71
|
-
stopSummarizerCallback
|
|
72
|
-
options);
|
|
71
|
+
stopSummarizerCallback);
|
|
73
72
|
|
|
74
73
|
await summarizer.waitStart();
|
|
75
74
|
|
|
@@ -107,7 +106,6 @@ export class RunningSummarizer implements IDisposable {
|
|
|
107
106
|
private readonly summaryCollection: SummaryCollection,
|
|
108
107
|
private readonly cancellationToken: ISummaryCancellationToken,
|
|
109
108
|
private readonly stopSummarizerCallback: (reason: SummarizerStopReason) => void,
|
|
110
|
-
{ disableHeuristics = false }: Readonly<Partial<ISummarizerOptions>> = {},
|
|
111
109
|
) {
|
|
112
110
|
const telemetryProps: ISummarizeRunnerTelemetry = {
|
|
113
111
|
summarizeCount: () => this.summarizeCount,
|
|
@@ -121,15 +119,23 @@ export class RunningSummarizer implements IDisposable {
|
|
|
121
119
|
},
|
|
122
120
|
);
|
|
123
121
|
|
|
124
|
-
if (
|
|
122
|
+
if (configuration.state !== "disableHeuristics") {
|
|
123
|
+
assert(this.configuration.state === "enabled", 0x2ea /* "Configuration state should be enabled" */);
|
|
125
124
|
this.heuristicRunner = new SummarizeHeuristicRunner(
|
|
126
125
|
heuristicData,
|
|
127
|
-
configuration,
|
|
128
|
-
(reason) => this.trySummarize(reason)
|
|
126
|
+
this.configuration,
|
|
127
|
+
(reason) => this.trySummarize(reason),
|
|
128
|
+
this.logger);
|
|
129
129
|
}
|
|
130
130
|
|
|
131
|
+
assert(
|
|
132
|
+
this.configuration.state !== "disabled",
|
|
133
|
+
0x2eb /* "Summary not supported with configuration disabled" */,
|
|
134
|
+
);
|
|
135
|
+
|
|
131
136
|
// Cap the maximum amount of time client will wait for a summarize op ack to maxSummarizeAckWaitTime
|
|
132
137
|
// configuration.maxAckWaitTime is composed from defaults, server values, and runtime overrides
|
|
138
|
+
|
|
133
139
|
const maxAckWaitTime = Math.min(this.configuration.maxAckWaitTime, maxSummarizeAckWaitTime);
|
|
134
140
|
|
|
135
141
|
this.pendingAckTimer = new PromiseTimer(
|
|
@@ -360,7 +366,11 @@ export class RunningSummarizer implements IDisposable {
|
|
|
360
366
|
return;
|
|
361
367
|
}
|
|
362
368
|
|
|
363
|
-
|
|
369
|
+
// We only want to attempt 1 summary when reason is "lastSummary"
|
|
370
|
+
if (++summaryAttempts > 1 && reason === "lastSummary") {
|
|
371
|
+
return;
|
|
372
|
+
}
|
|
373
|
+
|
|
364
374
|
summaryAttemptsPerPhase++;
|
|
365
375
|
|
|
366
376
|
const { delaySeconds: regularDelaySeconds = 0, ...options } = attempts[summaryAttemptPhase];
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
/*!
|
|
2
|
+
* Copyright (c) Microsoft Corporation and contributors. All rights reserved.
|
|
3
|
+
* Licensed under the MIT License.
|
|
4
|
+
*/
|
|
5
|
+
|
|
6
|
+
import { assert, bufferToString, stringToBuffer } from "@fluidframework/common-utils";
|
|
7
|
+
import { IDocumentStorageService, ISummaryContext } from "@fluidframework/driver-definitions";
|
|
8
|
+
import {
|
|
9
|
+
ICreateBlobResponse,
|
|
10
|
+
ISnapshotTree,
|
|
11
|
+
ISummaryHandle,
|
|
12
|
+
ISummaryTree,
|
|
13
|
+
IVersion,
|
|
14
|
+
} from "@fluidframework/protocol-definitions";
|
|
15
|
+
import { ISnapshotTreeWithBlobContents } from "@fluidframework/container-definitions";
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Serialized blobs from a snapshot. Used to load offline.
|
|
19
|
+
*/
|
|
20
|
+
export interface ISerializedBaseSnapshotBlobs {
|
|
21
|
+
[id: string]: string;
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* A storage wrapper that can serialize blobs from a snapshot tree and then use them to rehydrate.
|
|
26
|
+
* Used in offline load/attached dehydration to save snapshot blobs that are still needed but may have been deleted.
|
|
27
|
+
*/
|
|
28
|
+
export class SerializedSnapshotStorage implements IDocumentStorageService {
|
|
29
|
+
constructor(
|
|
30
|
+
private readonly storageGetter: () => IDocumentStorageService,
|
|
31
|
+
private readonly blobs: ISerializedBaseSnapshotBlobs,
|
|
32
|
+
) { }
|
|
33
|
+
|
|
34
|
+
public static async serializeTree(
|
|
35
|
+
snapshot: ISnapshotTree,
|
|
36
|
+
storage: IDocumentStorageService,
|
|
37
|
+
): Promise<ISerializedBaseSnapshotBlobs> {
|
|
38
|
+
const blobs = {};
|
|
39
|
+
await this.serializeTreeCore(snapshot, blobs, storage);
|
|
40
|
+
return blobs;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
private static async serializeTreeCore(
|
|
44
|
+
tree: ISnapshotTree,
|
|
45
|
+
blobs: ISerializedBaseSnapshotBlobs,
|
|
46
|
+
storage: IDocumentStorageService,
|
|
47
|
+
) {
|
|
48
|
+
const treePs: Promise<any>[] = [];
|
|
49
|
+
for (const subTree of Object.values(tree.trees)) {
|
|
50
|
+
treePs.push(this.serializeTreeCore(subTree, blobs, storage));
|
|
51
|
+
}
|
|
52
|
+
for (const id of Object.values(tree.blobs)) {
|
|
53
|
+
const blob = await storage.readBlob(id);
|
|
54
|
+
// ArrayBufferLike will not survive JSON.stringify()
|
|
55
|
+
blobs[id] = bufferToString(blob, "utf8");
|
|
56
|
+
}
|
|
57
|
+
return Promise.all(treePs);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
public static serializeTreeWithBlobContents(
|
|
61
|
+
snapshot: ISnapshotTreeWithBlobContents,
|
|
62
|
+
): ISerializedBaseSnapshotBlobs {
|
|
63
|
+
const blobs = {};
|
|
64
|
+
this.serializeTreeWithBlobContentsCore(snapshot, blobs);
|
|
65
|
+
return blobs;
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
private static serializeTreeWithBlobContentsCore(
|
|
69
|
+
tree: ISnapshotTreeWithBlobContents,
|
|
70
|
+
blobs: ISerializedBaseSnapshotBlobs,
|
|
71
|
+
) {
|
|
72
|
+
for (const subTree of Object.values(tree.trees)) {
|
|
73
|
+
this.serializeTreeWithBlobContentsCore(subTree, blobs);
|
|
74
|
+
}
|
|
75
|
+
for (const id of Object.values(tree.blobs)) {
|
|
76
|
+
const blob = tree.blobsContents[id];
|
|
77
|
+
assert(!!blob, 0x2ec /* "Blob must be present in blobsContents" */);
|
|
78
|
+
// ArrayBufferLike will not survive JSON.stringify()
|
|
79
|
+
blobs[id] = bufferToString(blob, "utf8");
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
private _storage?: IDocumentStorageService;
|
|
84
|
+
private get storage(): IDocumentStorageService {
|
|
85
|
+
// avoid calling it until we need it since it will be undefined if we're not connected
|
|
86
|
+
// and we shouldn't need it in this case anyway
|
|
87
|
+
if (this._storage) {
|
|
88
|
+
return this._storage;
|
|
89
|
+
}
|
|
90
|
+
this._storage = this.storageGetter();
|
|
91
|
+
return this._storage;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
public get repositoryUrl(): string { return this.storage.repositoryUrl; }
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Reads the object with the given ID, returns content in arrayBufferLike
|
|
98
|
+
*/
|
|
99
|
+
public async readBlob(id: string): Promise<ArrayBufferLike> {
|
|
100
|
+
if (this.blobs[id] !== undefined) {
|
|
101
|
+
return stringToBuffer(this.blobs[id], "utf8");
|
|
102
|
+
}
|
|
103
|
+
return this.storage.readBlob(id);
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
/**
|
|
107
|
+
* Returns the snapshot tree.
|
|
108
|
+
*/
|
|
109
|
+
// eslint-disable-next-line @rushstack/no-new-null
|
|
110
|
+
public async getSnapshotTree(version?: IVersion): Promise<ISnapshotTree | null> {
|
|
111
|
+
return this.storage.getSnapshotTree(version);
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* Retrieves all versions of the document starting at the specified versionId - or null if from the head
|
|
116
|
+
*/
|
|
117
|
+
// eslint-disable-next-line @rushstack/no-new-null
|
|
118
|
+
public async getVersions(versionId: string | null, count: number): Promise<IVersion[]> {
|
|
119
|
+
return this.storage.getVersions(versionId, count);
|
|
120
|
+
}
|
|
121
|
+
|
|
122
|
+
/**
|
|
123
|
+
* Creates a blob out of the given buffer
|
|
124
|
+
*/
|
|
125
|
+
public async createBlob(file: ArrayBufferLike): Promise<ICreateBlobResponse> {
|
|
126
|
+
return this.storage.createBlob(file);
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
/**
|
|
130
|
+
* Uploads a summary tree to storage using the given context for reference of previous summary handle.
|
|
131
|
+
* The ISummaryHandles in the uploaded tree should have paths to indicate which summary object they are
|
|
132
|
+
* referencing from the previously acked summary.
|
|
133
|
+
* Returns the uploaded summary handle.
|
|
134
|
+
*/
|
|
135
|
+
public async uploadSummaryWithContext(summary: ISummaryTree, context: ISummaryContext): Promise<string> {
|
|
136
|
+
return this.storage.uploadSummaryWithContext(summary, context);
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/**
|
|
140
|
+
* Retrieves the commit that matches the packfile handle. If the packfile has already been committed and the
|
|
141
|
+
* server has deleted it this call may result in a broken promise.
|
|
142
|
+
*/
|
|
143
|
+
public async downloadSummary(handle: ISummaryHandle): Promise<ISummaryTree> {
|
|
144
|
+
return this.storage.downloadSummary(handle);
|
|
145
|
+
}
|
|
146
|
+
}
|