foldkit-sync 0.15.0 → 0.16.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +558 -40
- package/dist/diagnostics.d.ts +74 -3
- package/dist/diagnostics.d.ts.map +1 -1
- package/dist/diagnostics.js +60 -4
- package/dist/diagnostics.js.map +1 -1
- package/dist/do.d.ts +49 -7
- package/dist/do.d.ts.map +1 -1
- package/dist/do.js +91 -8
- package/dist/do.js.map +1 -1
- package/dist/errors.d.ts +92 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +87 -1
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +9 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -2
- package/dist/index.js.map +1 -1
- package/dist/journal.d.ts +52 -27
- package/dist/journal.d.ts.map +1 -1
- package/dist/journal.js +176 -64
- package/dist/journal.js.map +1 -1
- package/dist/make.d.ts +48 -17
- package/dist/make.d.ts.map +1 -1
- package/dist/make.js +11 -4
- package/dist/make.js.map +1 -1
- package/dist/migrations.d.ts +37 -0
- package/dist/migrations.d.ts.map +1 -0
- package/dist/migrations.js +56 -0
- package/dist/migrations.js.map +1 -0
- package/dist/mount.d.ts +1 -1
- package/dist/mount.d.ts.map +1 -1
- package/dist/mount.js +12 -6
- package/dist/mount.js.map +1 -1
- package/dist/refusal.d.ts +33 -0
- package/dist/refusal.d.ts.map +1 -0
- package/dist/refusal.js +47 -0
- package/dist/refusal.js.map +1 -0
- package/dist/serverJournal.d.ts +143 -0
- package/dist/serverJournal.d.ts.map +1 -0
- package/dist/serverJournal.js +503 -0
- package/dist/serverJournal.js.map +1 -0
- package/dist/sync.d.ts +132 -11
- package/dist/sync.d.ts.map +1 -1
- package/dist/sync.js +273 -64
- package/dist/sync.js.map +1 -1
- package/dist/transport.d.ts +5 -5
- package/dist/transport.d.ts.map +1 -1
- package/dist/transport.js.map +1 -1
- package/package.json +11 -8
package/dist/sync.js
CHANGED
|
@@ -1,14 +1,27 @@
|
|
|
1
|
-
import { Effect, Exit, Metric, Option, PubSub, Queue, Ref, Schema, Stream, SynchronizedRef } from "effect";
|
|
1
|
+
import { Effect, Exit, Metric, Option, PubSub, Queue, Ref, Result, Schema, Stream, SynchronizedRef } from "effect";
|
|
2
2
|
import { Diagnostics } from "foldkit-surface";
|
|
3
|
-
import { exchangeFailed, historyRebuilt } from "./diagnostics.js";
|
|
3
|
+
import { exchangeFailed, historyRebuilt, unknownRefusal } from "./diagnostics.js";
|
|
4
4
|
import { Transport } from "./transport.js";
|
|
5
|
+
import { historyOf, migrations } from "./migrations.js";
|
|
6
|
+
import { SyncRefusal, fitWords } from "./refusal.js";
|
|
5
7
|
import { DocumentId, LocalSequence, OpId, ReplicaId, Sequence, localSequence, sequence } from "./ids.js";
|
|
6
|
-
import { CheckpointRegressionError, CommittedOrderError, ForeignAcknowledgementError, ForeignRejectionError, InvalidExchangeError, InvalidOutboxError, InvalidReplicaHistoryError, ReplayError, ReplicaClosedError, UnsupportedReplicaVersionError, WrongReplicaStorageError } from "./errors.js";
|
|
8
|
+
import { AmbiguousMessageError, CheckpointRegressionError, CommittedOrderError, ForeignAcknowledgementError, ForeignRejectionError, InvalidExchangeError, InvalidOutboxError, InvalidReplicaHistoryError, ReplayError, ReplicaClosedError, ServerOnlyMessageError, UnreadableMessageError, UnsupportedMessageVersionError, UnsupportedReplicaVersionError, WrongReplicaStorageError } from "./errors.js";
|
|
7
9
|
// src/sync.ts
|
|
8
10
|
/** Wire and persisted-format versions. A bump must handle the older value explicitly. */
|
|
9
11
|
const PROTOCOL_VERSION = 1;
|
|
10
12
|
const SCHEMA_VERSION = 1;
|
|
11
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* Replica ids the server's own operations use (`foldkit-sync/journal`'s
|
|
15
|
+
* `serverJournal`): a sent entry may not claim one, or it could bind the id to
|
|
16
|
+
* its actor first and have the server's append refused.
|
|
17
|
+
*/
|
|
18
|
+
const SERVER_REPLICA_PREFIX = "server/";
|
|
19
|
+
/** The longest replica id a sent entry may name. A stored one is read whatever its length. */
|
|
20
|
+
const MAX_SENT_REPLICA_ID = 128;
|
|
21
|
+
/**
|
|
22
|
+
* A client-authored operation envelope. `message` is the application's
|
|
23
|
+
* Message, encoded at the current version of the document's history.
|
|
24
|
+
*/
|
|
12
25
|
const OperationSchema = Schema.Struct({
|
|
13
26
|
protocolVersion: Schema.Literal(PROTOCOL_VERSION),
|
|
14
27
|
schemaVersion: Schema.Literal(SCHEMA_VERSION),
|
|
@@ -19,14 +32,49 @@ const OperationSchema = Schema.Struct({
|
|
|
19
32
|
baseCursor: Sequence,
|
|
20
33
|
message: Schema.Unknown
|
|
21
34
|
});
|
|
22
|
-
/**
|
|
23
|
-
|
|
35
|
+
/**
|
|
36
|
+
* An operation as it was written: what a replica keeps in its outbox and
|
|
37
|
+
* sends, with its Message at the version it was written at. `messageVersion`
|
|
38
|
+
* is absent for version 1, so a document with no migrations writes today's
|
|
39
|
+
* envelope byte for byte.
|
|
40
|
+
*/
|
|
41
|
+
const WrittenFields = Schema.Struct({
|
|
24
42
|
...OperationSchema.fields,
|
|
43
|
+
messageVersion: Schema.optionalKey(Schema.Int.check(Schema.isGreaterThanOrEqualTo(1)))
|
|
44
|
+
});
|
|
45
|
+
/**
|
|
46
|
+
* Branded, so only Sync makes one: decoding an entry, or writing an
|
|
47
|
+
* `Operation` at the current version (`journalContract().operation.encode`).
|
|
48
|
+
* A decoded `Operation` is not one, since its missing `messageVersion` would
|
|
49
|
+
* read as version 1 and be upcast again.
|
|
50
|
+
*/
|
|
51
|
+
const WrittenOperationSchema = WrittenFields.pipe(Schema.brand("foldkit-sync/WrittenOperation"));
|
|
52
|
+
/** What the codec has checked or written itself. */
|
|
53
|
+
const vouched = (operation) => operation;
|
|
54
|
+
const committedFields = {
|
|
25
55
|
serverSequence: Sequence.check(Schema.isGreaterThanOrEqualTo(1)),
|
|
26
56
|
actorId: Schema.NonEmptyString
|
|
57
|
+
};
|
|
58
|
+
Schema.Struct({
|
|
59
|
+
...OperationSchema.fields,
|
|
60
|
+
...committedFields
|
|
27
61
|
});
|
|
28
|
-
|
|
29
|
-
const
|
|
62
|
+
/** A committed operation as the server sends it: written at the server's current version. */
|
|
63
|
+
const CommittedWrittenSchema = Schema.Struct({
|
|
64
|
+
...WrittenFields.fields,
|
|
65
|
+
...committedFields
|
|
66
|
+
});
|
|
67
|
+
const decodeWritten = Schema.decodeUnknownSync(WrittenOperationSchema, { onExcessProperty: "error" });
|
|
68
|
+
/**
|
|
69
|
+
* An envelope as written, checked for its shape and nothing else: for the
|
|
70
|
+
* server's own operations (`serverJournal`), whose bytes it wrote or recorded.
|
|
71
|
+
* Never for an entry someone sent.
|
|
72
|
+
*/
|
|
73
|
+
const writtenEnvelope = (input) => decodeWritten(input);
|
|
74
|
+
const decodeCommittedWritten = Schema.decodeUnknownSync(CommittedWrittenSchema, { onExcessProperty: "error" });
|
|
75
|
+
/** The version an operation was written at: absent is 1. */
|
|
76
|
+
const versionOf = (operation) => operation.messageVersion ?? 1;
|
|
77
|
+
const decodeRefusal = Schema.decodeUnknownResult(SyncRefusal, { onExcessProperty: "error" });
|
|
30
78
|
/** How long `start` waits after a failed exchange: doubling from the base, up to the cap, in ms. */
|
|
31
79
|
const START_RETRY_BASE = 500;
|
|
32
80
|
const START_RETRY_MAX = 3e4;
|
|
@@ -53,8 +101,11 @@ const syncMetrics = {
|
|
|
53
101
|
*/
|
|
54
102
|
const defineSync = (definition) => {
|
|
55
103
|
const documentId = definition.documentId;
|
|
56
|
-
const
|
|
57
|
-
const
|
|
104
|
+
const messages = definition.migrations ?? migrations(definition.message);
|
|
105
|
+
const history = historyOf(messages);
|
|
106
|
+
const current = history.version;
|
|
107
|
+
const decodeMessage = Schema.decodeUnknownSync(messages.current, { onExcessProperty: "error" });
|
|
108
|
+
const encodeMessage = Schema.encodeSync(messages.current);
|
|
58
109
|
const decodeShared = Schema.decodeUnknownSync(definition.shared, { onExcessProperty: "error" });
|
|
59
110
|
const encodeShared = Schema.encodeSync(definition.shared);
|
|
60
111
|
const ReplicaStateSchema = Schema.Struct({
|
|
@@ -67,7 +118,8 @@ const defineSync = (definition) => {
|
|
|
67
118
|
cursor: Sequence,
|
|
68
119
|
committed: definition.shared,
|
|
69
120
|
committedIds: Schema.Array(Schema.String),
|
|
70
|
-
pending: Schema.Array(
|
|
121
|
+
pending: Schema.Array(WrittenOperationSchema),
|
|
122
|
+
refused: Schema.optionalKey(Schema.Array(WrittenOperationSchema)),
|
|
71
123
|
epoch: Schema.optionalKey(Schema.NonEmptyString)
|
|
72
124
|
});
|
|
73
125
|
const CheckpointSchema = Schema.Struct({
|
|
@@ -77,6 +129,10 @@ const defineSync = (definition) => {
|
|
|
77
129
|
const ExchangeSchema = Schema.Struct({
|
|
78
130
|
operations: Schema.Array(Schema.Unknown),
|
|
79
131
|
rejected: Schema.Array(OpId),
|
|
132
|
+
refusals: Schema.optional(Schema.Array(Schema.Struct({
|
|
133
|
+
opId: OpId,
|
|
134
|
+
refusal: Schema.Unknown
|
|
135
|
+
}))),
|
|
80
136
|
reasons: Schema.optional(Schema.Array(Schema.Struct({
|
|
81
137
|
opId: OpId,
|
|
82
138
|
reason: Schema.String.check(Schema.isMaxLength(500))
|
|
@@ -88,7 +144,7 @@ const defineSync = (definition) => {
|
|
|
88
144
|
});
|
|
89
145
|
const decodeState = Schema.decodeUnknownSync(ReplicaStateSchema, { onExcessProperty: "error" });
|
|
90
146
|
const encodeState = Schema.encodeSync(ReplicaStateSchema);
|
|
91
|
-
const
|
|
147
|
+
const encodeWritten = Schema.encodeSync(WrittenOperationSchema);
|
|
92
148
|
const VersionProbe = Schema.Struct({
|
|
93
149
|
protocolVersion: Schema.optional(Schema.Number),
|
|
94
150
|
schemaVersion: Schema.optional(Schema.Number)
|
|
@@ -117,44 +173,129 @@ const defineSync = (definition) => {
|
|
|
117
173
|
const checkIdentity = (operation) => {
|
|
118
174
|
if (operation.opId !== `${operation.replicaId}:${operation.localSequence}`) throw new Error("Invalid operation identity");
|
|
119
175
|
};
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
176
|
+
/**
|
|
177
|
+
* An operation's Message, upcast to the current version and decoded. The
|
|
178
|
+
* durable check runs on the current Message, so a step that maps an old
|
|
179
|
+
* variant onto a local-only one is refused as that one is.
|
|
180
|
+
*/
|
|
181
|
+
const newer = (opId, version) => new UnsupportedMessageVersionError({
|
|
182
|
+
opId,
|
|
183
|
+
found: version,
|
|
184
|
+
supported: current,
|
|
185
|
+
message: `Operation ${opId} was written at Message version ${version}; this build reads up to ${current}`
|
|
186
|
+
});
|
|
187
|
+
const serverOnly = definition.serverOnly ?? (() => false);
|
|
188
|
+
/**
|
|
189
|
+
* `sent` is an entry someone sent, which may carry only what a replica
|
|
190
|
+
* submits; `history` is what the journal stored, the server's Messages too.
|
|
191
|
+
*/
|
|
192
|
+
const messageOf = (operation, kind) => {
|
|
193
|
+
const version = versionOf(operation);
|
|
194
|
+
if (version > current) throw newer(operation.opId, version);
|
|
195
|
+
let decoded;
|
|
196
|
+
if (version === current) decoded = decodeMessage(operation.message);
|
|
197
|
+
else try {
|
|
198
|
+
decoded = decodeMessage(history.upcast(version, operation.message));
|
|
199
|
+
} catch (cause) {
|
|
200
|
+
throw new UnreadableMessageError({
|
|
201
|
+
opId: operation.opId,
|
|
202
|
+
version,
|
|
203
|
+
message: `No step of the Message history reads operation ${operation.opId}, written at version ${version}`,
|
|
204
|
+
cause
|
|
205
|
+
});
|
|
206
|
+
}
|
|
207
|
+
const durable = definition.durable(decoded);
|
|
208
|
+
const server = serverOnly(decoded);
|
|
209
|
+
if (durable && server) throw new AmbiguousMessageError({
|
|
210
|
+
opId: operation.opId,
|
|
211
|
+
message: `Operation ${operation.opId} carries a Message both durable and server-only`
|
|
212
|
+
});
|
|
213
|
+
if (server && kind === "sent") throw new ServerOnlyMessageError({
|
|
214
|
+
opId: operation.opId,
|
|
215
|
+
message: `Operation ${operation.opId} carries a Message only the server appends`
|
|
216
|
+
});
|
|
217
|
+
if (!durable && !server) throw new Error("Message is local-only");
|
|
123
218
|
return decoded;
|
|
124
219
|
};
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
220
|
+
/** The envelope alone: the decoded side holds no version, only the current Message. */
|
|
221
|
+
const envelopeOf = ({ messageVersion: _, ...envelope }) => envelope;
|
|
222
|
+
/** Written at the current version, which an explicit 1 is not. */
|
|
223
|
+
const write = (operation) => {
|
|
224
|
+
const envelope = envelopeOf(operation);
|
|
225
|
+
return vouched(current === 1 ? envelope : {
|
|
226
|
+
...envelope,
|
|
227
|
+
messageVersion: current
|
|
228
|
+
});
|
|
131
229
|
};
|
|
132
230
|
const assertDocument = (operation, key) => {
|
|
133
231
|
if (operation.documentId !== key) throw new Error("Wrong document");
|
|
134
232
|
return operation;
|
|
135
233
|
};
|
|
136
|
-
|
|
137
|
-
const
|
|
234
|
+
/** What a sent entry may not name: the server's replica ids, or one past the bound. */
|
|
235
|
+
const checkSentReplica = (operation) => {
|
|
236
|
+
if (operation.replicaId.startsWith("server/")) throw new Error("The replica id is reserved for the server");
|
|
237
|
+
if (operation.replicaId.length > MAX_SENT_REPLICA_ID) throw new Error("The replica id is too long");
|
|
238
|
+
};
|
|
239
|
+
const read = (written, kind) => {
|
|
240
|
+
checkIdentity(written);
|
|
241
|
+
if (kind === "sent") checkSentReplica(written);
|
|
242
|
+
return {
|
|
243
|
+
...envelopeOf(written),
|
|
244
|
+
message: encodeMessage(messageOf(written, kind))
|
|
245
|
+
};
|
|
246
|
+
};
|
|
247
|
+
const normalizeOperation = (input) => read(decodeWritten(input), "sent");
|
|
248
|
+
const operationFrom = (input, key) => read(assertDocument(decodeWritten(input), key), "sent");
|
|
249
|
+
const writtenFrom = (input, key) => {
|
|
250
|
+
const written = assertDocument(decodeWritten(input), key);
|
|
251
|
+
read(written, "sent");
|
|
252
|
+
return written;
|
|
253
|
+
};
|
|
254
|
+
/** Stored history: the server's Messages read, and a stored replica id of any length. */
|
|
255
|
+
const readStored = (input) => read(decodeWritten(input), "history");
|
|
256
|
+
/** An outbox entry this replica wrote: history, since it may predate a deploy that moved its variant. */
|
|
257
|
+
const storedFrom = (input, key) => read(assertDocument(decodeWritten(input), key), "history");
|
|
138
258
|
/**
|
|
139
259
|
* A committed operation arrives with its message already encoded, so decode it
|
|
140
260
|
* once for the contract check and hand it to `replay` instead of decoding and
|
|
141
|
-
* re-encoding it on the way
|
|
261
|
+
* re-encoding it on the way.
|
|
142
262
|
*/
|
|
143
263
|
const decodeCommittedOperation = (input, key) => {
|
|
144
|
-
const
|
|
145
|
-
checkIdentity(
|
|
264
|
+
const written = assertDocument(decodeCommittedWritten(input), key);
|
|
265
|
+
checkIdentity(written);
|
|
266
|
+
const message = messageOf(written, "history");
|
|
146
267
|
return {
|
|
147
|
-
committed
|
|
148
|
-
|
|
268
|
+
committed: {
|
|
269
|
+
...envelopeOf(written),
|
|
270
|
+
message: encodeMessage(message)
|
|
271
|
+
},
|
|
272
|
+
message
|
|
149
273
|
};
|
|
150
274
|
};
|
|
151
275
|
const committedFrom = (input, key) => decodeCommittedOperation(input, key).committed;
|
|
276
|
+
/**
|
|
277
|
+
* Normalized at the version it was written at, with no upcast: what the
|
|
278
|
+
* build that wrote it hashed. An explicit `messageVersion: 1` is dropped,
|
|
279
|
+
* so it and an absent one compare alike.
|
|
280
|
+
*/
|
|
281
|
+
const compared = (input) => {
|
|
282
|
+
const written = decodeWritten(input);
|
|
283
|
+
const version = versionOf(written);
|
|
284
|
+
if (version > current) throw newer(written.opId, version);
|
|
285
|
+
const envelope = envelopeOf(written);
|
|
286
|
+
return vouched({
|
|
287
|
+
...envelope,
|
|
288
|
+
...version === 1 ? {} : { messageVersion: version },
|
|
289
|
+
message: history.normalize(version, written.message)
|
|
290
|
+
});
|
|
291
|
+
};
|
|
152
292
|
const journalContract = () => ({
|
|
153
293
|
replicaId: (operation) => operation.replicaId,
|
|
154
294
|
legacyReplicaId: (opId) => opId.slice(0, opId.lastIndexOf(":")),
|
|
155
295
|
operation: {
|
|
156
|
-
encode:
|
|
157
|
-
decode:
|
|
296
|
+
encode: write,
|
|
297
|
+
decode: readStored,
|
|
298
|
+
compared
|
|
158
299
|
},
|
|
159
300
|
snapshot: {
|
|
160
301
|
encode: encodeShared,
|
|
@@ -170,7 +311,12 @@ const defineSync = (definition) => {
|
|
|
170
311
|
}))
|
|
171
312
|
}) }
|
|
172
313
|
});
|
|
173
|
-
|
|
314
|
+
/** `state` with `refused` set; the key is absent when none is, so such a state is as before. */
|
|
315
|
+
const withRefused = ({ refused: _, ...state }, refused) => refused.length === 0 ? state : {
|
|
316
|
+
...state,
|
|
317
|
+
refused
|
|
318
|
+
};
|
|
319
|
+
const optimistic = (state) => state.pending.reduce((model, operation) => definition.replay(model, messageOf(operation, "history")), state.committed);
|
|
174
320
|
const openReplica = Effect.fn("Sync.openReplica")(function* (storage, options) {
|
|
175
321
|
const saved = yield* storage.load();
|
|
176
322
|
const initial = {
|
|
@@ -205,13 +351,17 @@ const defineSync = (definition) => {
|
|
|
205
351
|
const committedIds = new Set(state.committedIds);
|
|
206
352
|
if (state.nextLocalSequence < 1 || committedIds.size !== state.committedIds.length) return yield* new InvalidReplicaHistoryError({ message: "Invalid replica history" });
|
|
207
353
|
const pendingIds = /* @__PURE__ */ new Set();
|
|
354
|
+
const unreadable = /* @__PURE__ */ new Set();
|
|
208
355
|
for (const pending of state.pending) {
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
|
|
214
|
-
|
|
356
|
+
if (pending.replicaId !== replicaId || pending.localSequence >= state.nextLocalSequence || committedIds.has(pending.opId) || pendingIds.has(pending.opId)) return yield* new InvalidOutboxError({ message: "Invalid outbox" });
|
|
357
|
+
pendingIds.add(pending.opId);
|
|
358
|
+
try {
|
|
359
|
+
storedFrom(pending, documentId);
|
|
360
|
+
} catch (cause) {
|
|
361
|
+
if (cause instanceof UnsupportedMessageVersionError) return yield* cause;
|
|
362
|
+
if (cause instanceof UnreadableMessageError) unreadable.add(pending);
|
|
363
|
+
else return yield* new InvalidOutboxError({ message: "Invalid outbox" });
|
|
364
|
+
}
|
|
215
365
|
}
|
|
216
366
|
const store = Effect.fn("Sync.store")(function* (next, expectedRevision) {
|
|
217
367
|
const encoded = yield* Effect.try({
|
|
@@ -224,7 +374,13 @@ const defineSync = (definition) => {
|
|
|
224
374
|
yield* storage.save(encoded, expectedRevision);
|
|
225
375
|
});
|
|
226
376
|
if (saved === void 0) yield* store(state, null);
|
|
227
|
-
const
|
|
377
|
+
const opened = unreadable.size === 0 ? state : withRefused({
|
|
378
|
+
...state,
|
|
379
|
+
revision: state.revision + 1,
|
|
380
|
+
pending: state.pending.filter((operation) => !unreadable.has(operation))
|
|
381
|
+
}, [...state.refused ?? [], ...unreadable]);
|
|
382
|
+
if (opened !== state) yield* store(opened, state.revision);
|
|
383
|
+
const stateRef = yield* SynchronizedRef.make(opened);
|
|
228
384
|
const closed = yield* Ref.make(false);
|
|
229
385
|
const lastError = yield* Ref.make(void 0);
|
|
230
386
|
const rejectedOps = yield* Ref.make([]);
|
|
@@ -251,6 +407,7 @@ const defineSync = (definition) => {
|
|
|
251
407
|
cursor: current.cursor,
|
|
252
408
|
lastError: yield* Ref.get(lastError),
|
|
253
409
|
rejected: yield* Ref.get(rejectedOps),
|
|
410
|
+
refused: current.refused ?? [],
|
|
254
411
|
epoch: Option.fromUndefinedOr(current.epoch)
|
|
255
412
|
},
|
|
256
413
|
shared: projected,
|
|
@@ -268,24 +425,26 @@ const defineSync = (definition) => {
|
|
|
268
425
|
});
|
|
269
426
|
const result = yield* SynchronizedRef.modifyEffect(stateRef, (current) => Effect.gen(function* () {
|
|
270
427
|
if (yield* Ref.get(closed)) return yield* new ReplicaClosedError({ message: "Replica is closed" });
|
|
428
|
+
if (serverOnly(message)) return yield* new InvalidOutboxError({ message: "Message is one only the server appends" });
|
|
271
429
|
if (!definition.durable(message)) return yield* new InvalidOutboxError({ message: "Message is local-only" });
|
|
272
430
|
const last = current.pending[current.pending.length - 1];
|
|
273
|
-
const merged = last !== void 0 && definition.coalesce !== void 0 && (yield* Ref.get(unsent)).has(last.opId) ? definition.coalesce(
|
|
431
|
+
const merged = last !== void 0 && definition.coalesce !== void 0 && (yield* Ref.get(unsent)).has(last.opId) ? definition.coalesce(messageOf(last, "history"), message) : void 0;
|
|
274
432
|
if (merged !== void 0 && !definition.durable(merged)) return yield* new InvalidOutboxError({ message: "coalesce returned a local-only Message" });
|
|
433
|
+
if (merged !== void 0 && serverOnly(merged)) return yield* new InvalidOutboxError({ message: "coalesce returned a Message only the server appends" });
|
|
275
434
|
const operation = yield* Effect.try({
|
|
276
|
-
try: () =>
|
|
435
|
+
try: () => write(storedFrom(write(merged === void 0 ? {
|
|
277
436
|
protocolVersion: PROTOCOL_VERSION,
|
|
278
437
|
schemaVersion: SCHEMA_VERSION,
|
|
279
438
|
documentId,
|
|
280
439
|
replicaId,
|
|
281
440
|
localSequence: current.nextLocalSequence,
|
|
282
|
-
opId: `${replicaId}:${current.nextLocalSequence}
|
|
441
|
+
opId: OpId.make(`${replicaId}:${current.nextLocalSequence}`),
|
|
283
442
|
baseCursor: current.cursor,
|
|
284
443
|
message: encodeMessage(message)
|
|
285
444
|
} : {
|
|
286
|
-
...last,
|
|
445
|
+
...envelopeOf(last),
|
|
287
446
|
message: encodeMessage(merged)
|
|
288
|
-
}, documentId),
|
|
447
|
+
}), documentId)),
|
|
289
448
|
catch: () => new InvalidOutboxError({ message: "Invalid outbox" })
|
|
290
449
|
});
|
|
291
450
|
const cached = yield* Ref.get(projection);
|
|
@@ -305,7 +464,7 @@ const defineSync = (definition) => {
|
|
|
305
464
|
if (storage.append === void 0) yield* persist(next, current);
|
|
306
465
|
else {
|
|
307
466
|
const encoded = yield* Effect.try({
|
|
308
|
-
try: () =>
|
|
467
|
+
try: () => encodeWritten(operation),
|
|
309
468
|
catch: (cause) => new InvalidReplicaHistoryError({
|
|
310
469
|
message: "Could not encode the operation",
|
|
311
470
|
cause
|
|
@@ -351,7 +510,7 @@ const defineSync = (definition) => {
|
|
|
351
510
|
})
|
|
352
511
|
});
|
|
353
512
|
});
|
|
354
|
-
const restarted = yield* SynchronizedRef.modifyEffect(stateRef, (current) => Effect.gen(function* () {
|
|
513
|
+
const { restarted, refusals } = yield* SynchronizedRef.modifyEffect(stateRef, (current) => Effect.gen(function* () {
|
|
355
514
|
if (yield* Ref.get(closed)) return yield* new ReplicaClosedError({ message: "Replica is closed" });
|
|
356
515
|
const restarted = response.epoch !== void 0 && current.epoch !== void 0 && response.epoch !== current.epoch;
|
|
357
516
|
if (restarted) yield* Diagnostics.report(historyRebuilt(documentId, replicaId));
|
|
@@ -390,15 +549,32 @@ const defineSync = (definition) => {
|
|
|
390
549
|
message: "Server both acknowledged and rejected an operation"
|
|
391
550
|
});
|
|
392
551
|
}
|
|
393
|
-
for (const { opId } of response.reasons ?? []) if (!rejected.has(opId)) return yield* new ForeignRejectionError({
|
|
552
|
+
for (const { opId } of [...response.refusals ?? [], ...response.reasons ?? []]) if (!rejected.has(opId)) return yield* new ForeignRejectionError({
|
|
394
553
|
opId,
|
|
395
554
|
message: "Server gave a reason for an operation it did not reject"
|
|
396
555
|
});
|
|
556
|
+
const refusing = /* @__PURE__ */ new Set();
|
|
557
|
+
for (const { opId } of response.refusals ?? []) {
|
|
558
|
+
if (refusing.has(opId)) return yield* new ForeignRejectionError({
|
|
559
|
+
opId,
|
|
560
|
+
message: "Server gave two refusals for one operation"
|
|
561
|
+
});
|
|
562
|
+
refusing.add(opId);
|
|
563
|
+
}
|
|
564
|
+
const refusals = (response.refusals ?? []).map(({ opId, refusal }) => [opId, decodeRefusal(refusal)]);
|
|
565
|
+
const unreadableIds = new Set(refusals.flatMap(([opId, refusal]) => Result.match(refusal, {
|
|
566
|
+
onFailure: () => [],
|
|
567
|
+
onSuccess: SyncRefusal.match({
|
|
568
|
+
Forbidden: () => [],
|
|
569
|
+
Invalid: () => [],
|
|
570
|
+
Unreadable: () => [opId]
|
|
571
|
+
})
|
|
572
|
+
})));
|
|
397
573
|
let applied = 0;
|
|
398
574
|
for (const raw of response.operations) {
|
|
399
575
|
const { committed: operation, message } = yield* Effect.try({
|
|
400
576
|
try: () => decodeCommittedOperation(raw, documentId),
|
|
401
|
-
catch: (cause) => new InvalidReplicaHistoryError({
|
|
577
|
+
catch: (cause) => cause instanceof UnsupportedMessageVersionError ? cause : new InvalidReplicaHistoryError({
|
|
402
578
|
message: "Invalid committed operation",
|
|
403
579
|
cause
|
|
404
580
|
})
|
|
@@ -423,7 +599,7 @@ const defineSync = (definition) => {
|
|
|
423
599
|
}
|
|
424
600
|
if (applied > 0) yield* Metric.update(syncMetrics.applied, applied);
|
|
425
601
|
const epoch = response.epoch ?? current.epoch;
|
|
426
|
-
const next = {
|
|
602
|
+
const next = withRefused({
|
|
427
603
|
...current,
|
|
428
604
|
revision: current.revision + 1,
|
|
429
605
|
committed,
|
|
@@ -431,19 +607,30 @@ const defineSync = (definition) => {
|
|
|
431
607
|
...epoch === void 0 ? {} : { epoch },
|
|
432
608
|
committedIds: [...ids].slice(-1024),
|
|
433
609
|
pending: current.pending.filter((operation) => !ids.has(operation.opId) && !(acknowledged.has(operation.opId) && response.more !== true) && !rejected.has(operation.opId))
|
|
434
|
-
};
|
|
610
|
+
}, [...current.refused ?? [], ...current.pending.filter((operation) => unreadableIds.has(operation.opId))]);
|
|
435
611
|
yield* persist(next, current);
|
|
436
|
-
return [
|
|
612
|
+
return [{
|
|
613
|
+
restarted,
|
|
614
|
+
refusals
|
|
615
|
+
}, next];
|
|
437
616
|
}));
|
|
438
617
|
yield* Ref.set(lastError, void 0);
|
|
439
618
|
if (response.rejected.length > 0) {
|
|
440
619
|
const reasonOf = new Map((response.reasons ?? []).map(({ opId, reason }) => [opId, reason]));
|
|
620
|
+
const refusalOf = /* @__PURE__ */ new Map();
|
|
621
|
+
for (const [opId, decoded] of refusals) if (Result.isSuccess(decoded)) refusalOf.set(opId, decoded.success);
|
|
622
|
+
else yield* Diagnostics.report(unknownRefusal(documentId, replicaId, opId, fitWords(decoded.failure.message)));
|
|
441
623
|
const sentOf = new Map(sent.pending.map((operation) => [operation.opId, operation]));
|
|
442
|
-
const refused = response.rejected.map((opId) =>
|
|
443
|
-
opId
|
|
444
|
-
|
|
445
|
-
|
|
446
|
-
|
|
624
|
+
const refused = response.rejected.map((opId) => {
|
|
625
|
+
const operation = sentOf.get(opId);
|
|
626
|
+
return {
|
|
627
|
+
opId,
|
|
628
|
+
refusal: Option.fromUndefinedOr(refusalOf.get(opId)),
|
|
629
|
+
reason: Option.fromUndefinedOr(reasonOf.get(opId)),
|
|
630
|
+
operation,
|
|
631
|
+
message: encodeMessage(messageOf(operation, "history"))
|
|
632
|
+
};
|
|
633
|
+
});
|
|
447
634
|
yield* Ref.update(rejectedOps, (previous) => [...refused, ...previous].slice(0, 32));
|
|
448
635
|
}
|
|
449
636
|
yield* PubSub.publish(statusSignals, void 0);
|
|
@@ -464,16 +651,33 @@ const defineSync = (definition) => {
|
|
|
464
651
|
while (!(yield* Ref.get(closed))) {
|
|
465
652
|
const exit = yield* Effect.exit(synchronize);
|
|
466
653
|
if (yield* Ref.get(closed)) return;
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
654
|
+
yield* Exit.match(exit, {
|
|
655
|
+
onSuccess: () => {
|
|
656
|
+
failures = 0;
|
|
657
|
+
return Queue.take(wake);
|
|
658
|
+
},
|
|
659
|
+
onFailure: () => {
|
|
660
|
+
failures += 1;
|
|
661
|
+
const backoff = Math.min(START_RETRY_MAX, START_RETRY_BASE * 2 ** Math.min(failures - 1, 16));
|
|
662
|
+
return Effect.raceFirst(Queue.take(wake), Effect.sleep(backoff));
|
|
663
|
+
}
|
|
664
|
+
});
|
|
475
665
|
}
|
|
476
666
|
});
|
|
667
|
+
const dismiss = Effect.fn("Sync.dismiss")(function* (opId) {
|
|
668
|
+
if (yield* SynchronizedRef.modifyEffect(stateRef, (current) => Effect.gen(function* () {
|
|
669
|
+
if (yield* Ref.get(closed)) return yield* new ReplicaClosedError({ message: "Replica is closed" });
|
|
670
|
+
const refused = current.refused ?? [];
|
|
671
|
+
const left = refused.filter((operation) => operation.opId !== opId);
|
|
672
|
+
if (left.length === refused.length) return [false, current];
|
|
673
|
+
const next = withRefused({
|
|
674
|
+
...current,
|
|
675
|
+
revision: current.revision + 1
|
|
676
|
+
}, left);
|
|
677
|
+
yield* persist(next, current);
|
|
678
|
+
return [true, next];
|
|
679
|
+
}))) yield* PubSub.publish(statusSignals, void 0);
|
|
680
|
+
});
|
|
477
681
|
return {
|
|
478
682
|
replicaId,
|
|
479
683
|
shared,
|
|
@@ -487,6 +691,7 @@ const defineSync = (definition) => {
|
|
|
487
691
|
submit,
|
|
488
692
|
synchronize,
|
|
489
693
|
start,
|
|
694
|
+
dismiss,
|
|
490
695
|
close: Effect.fn("Sync.close")(function* () {
|
|
491
696
|
yield* Ref.set(closed, true);
|
|
492
697
|
yield* Queue.offer(wake, void 0);
|
|
@@ -497,9 +702,13 @@ const defineSync = (definition) => {
|
|
|
497
702
|
return {
|
|
498
703
|
documentId,
|
|
499
704
|
durable: definition.durable,
|
|
705
|
+
serverOnly,
|
|
500
706
|
codec: {
|
|
707
|
+
messageVersion: current,
|
|
708
|
+
message: messages.current,
|
|
501
709
|
normalizeOperation,
|
|
502
710
|
operationFrom,
|
|
711
|
+
writtenFrom,
|
|
503
712
|
committedFrom,
|
|
504
713
|
decodeExchange
|
|
505
714
|
},
|
|
@@ -507,6 +716,6 @@ const defineSync = (definition) => {
|
|
|
507
716
|
openReplica
|
|
508
717
|
};
|
|
509
718
|
};
|
|
510
|
-
export { defineSync, syncMetrics };
|
|
719
|
+
export { SERVER_REPLICA_PREFIX, defineSync, syncMetrics, writtenEnvelope };
|
|
511
720
|
|
|
512
721
|
//# sourceMappingURL=sync.js.map
|