@openlfcp/client 0.1.0-rc.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +27 -0
- package/dist/apply.d.ts +299 -0
- package/dist/apply.js +741 -0
- package/dist/checkpoint.d.ts +31 -0
- package/dist/checkpoint.js +50 -0
- package/dist/connection.d.ts +87 -0
- package/dist/connection.js +210 -0
- package/dist/data-unit.d.ts +50 -0
- package/dist/data-unit.js +39 -0
- package/dist/engine-guard.d.ts +58 -0
- package/dist/engine-guard.js +148 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +15 -0
- package/dist/invite.d.ts +154 -0
- package/dist/invite.js +391 -0
- package/dist/outbound.d.ts +227 -0
- package/dist/outbound.js +508 -0
- package/dist/queue.d.ts +38 -0
- package/dist/queue.js +116 -0
- package/dist/resource-state.d.ts +34 -0
- package/dist/resource-state.js +24 -0
- package/dist/snapshot.d.ts +42 -0
- package/dist/snapshot.js +40 -0
- package/dist/storage.d.ts +96 -0
- package/dist/storage.js +289 -0
- package/dist/sync-client.d.ts +203 -0
- package/dist/sync-client.js +1032 -0
- package/package.json +50 -0
package/dist/apply.js
ADDED
|
@@ -0,0 +1,741 @@
|
|
|
1
|
+
import { actorSequence, bytesEqual, dataUnitId, fromHex, toHex, } from "@openlfcp/core";
|
|
2
|
+
import { decryptDataUnit, deriveActorDataKey } from "@openlfcp/crypto";
|
|
3
|
+
import { beyondCutoff, canonicalFrontierFromCbor, checkDataUnit, classifyDataUnit, dataUnitAad, parseDataUnit, receiveDataUnit, } from "@openlfcp/wire";
|
|
4
|
+
import { decodeDeterministic } from "@openlfcp/wire/cbor";
|
|
5
|
+
import { EngineGuard, isEngineTrap, unitItem } from "./engine-guard.js";
|
|
6
|
+
import { dataUnitRow, StoredSeenUnits } from "./storage.js";
|
|
7
|
+
/** A codec for bytes that do not parse: receiveDataUnit rejects them before any decode. */
|
|
8
|
+
const unreachableCodec = (dataProfile) => ({
|
|
9
|
+
dataProfile,
|
|
10
|
+
encode: () => {
|
|
11
|
+
throw new Error("no codec for a malformed unit");
|
|
12
|
+
},
|
|
13
|
+
decode: () => {
|
|
14
|
+
throw new Error("no codec for a malformed unit");
|
|
15
|
+
},
|
|
16
|
+
});
|
|
17
|
+
const LFCP_ACCEPTED = ["merged", "profile-pending"];
|
|
18
|
+
const CRASHED = "this unit crashed the profile engine twice, applied alone; it is not applied again on this device (local only)";
|
|
19
|
+
const profileUnit = (view, r) => ({
|
|
20
|
+
unitId: r.unitId,
|
|
21
|
+
resourceId: view.state.resourceId,
|
|
22
|
+
actor: r.actor,
|
|
23
|
+
seq: r.seq,
|
|
24
|
+
epoch: r.epoch,
|
|
25
|
+
});
|
|
26
|
+
export class DataUnitApplier {
|
|
27
|
+
#options;
|
|
28
|
+
#handlers;
|
|
29
|
+
#storage;
|
|
30
|
+
#seen;
|
|
31
|
+
#guard;
|
|
32
|
+
/** Units quarantined by the crash-loop breaker at this start, not yet reported. */
|
|
33
|
+
#crashed = [];
|
|
34
|
+
constructor(options) {
|
|
35
|
+
this.#options = options;
|
|
36
|
+
this.#handlers = new Map(options.handlers.map((h) => [h.dataProfile, h]));
|
|
37
|
+
this.#storage = options.storage;
|
|
38
|
+
this.#seen = new StoredSeenUnits(options.storage);
|
|
39
|
+
this.#guard = new EngineGuard(options.storage, "units");
|
|
40
|
+
}
|
|
41
|
+
/** Once per Resource: a leftover apply record becomes suspects; second-time crashers are quarantined. */
|
|
42
|
+
async #recover(resource) {
|
|
43
|
+
const crashed = await this.#guard.recover(resource);
|
|
44
|
+
const writes = [];
|
|
45
|
+
for (const item of crashed) {
|
|
46
|
+
const unitId = dataUnitId(fromHex(item.slice("unit:".length)));
|
|
47
|
+
if ((await this.#storage.dataUnits.get(unitId)) === undefined)
|
|
48
|
+
continue;
|
|
49
|
+
writes.push({
|
|
50
|
+
op: "set-data-unit-status",
|
|
51
|
+
unitId,
|
|
52
|
+
status: "local-failure",
|
|
53
|
+
detail: `INVALID_AUTOMERGE_BYTES: ${CRASHED}`,
|
|
54
|
+
});
|
|
55
|
+
this.#crashed.push(unitId);
|
|
56
|
+
}
|
|
57
|
+
await this.#write(writes);
|
|
58
|
+
}
|
|
59
|
+
/** The unit's ID from its bytes, or undefined when they do not parse (reported by the LFCP checks). */
|
|
60
|
+
#idOf(bytes) {
|
|
61
|
+
try {
|
|
62
|
+
return parseDataUnit(bytes).signed.id;
|
|
63
|
+
}
|
|
64
|
+
catch {
|
|
65
|
+
return undefined;
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
#crashOutcome(unitId) {
|
|
69
|
+
return Object.freeze({
|
|
70
|
+
kind: "engine-crash",
|
|
71
|
+
code: "INVALID_AUTOMERGE_BYTES",
|
|
72
|
+
unitId,
|
|
73
|
+
message: CRASHED,
|
|
74
|
+
});
|
|
75
|
+
}
|
|
76
|
+
async #write(writes) {
|
|
77
|
+
if (writes.length === 0)
|
|
78
|
+
return;
|
|
79
|
+
const r = await this.#storage.commit(writes);
|
|
80
|
+
if (!r.ok)
|
|
81
|
+
throw new Error(`unexpected storage precondition failure: ${r.reason}`);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Receives one Data Unit of the Resource `view` describes. Never creates
|
|
85
|
+
* a Data Unit: received content is applied, not re-sent as local work.
|
|
86
|
+
*/
|
|
87
|
+
async receive(view, bytes) {
|
|
88
|
+
const R = view.state.resourceId;
|
|
89
|
+
await this.#recover(R);
|
|
90
|
+
const id = this.#idOf(bytes);
|
|
91
|
+
if (id !== undefined && this.#guard.suspicion(R, unitItem(id)) === 2)
|
|
92
|
+
return this.#crashOutcome(id);
|
|
93
|
+
return this.#guard.run(R, id === undefined ? [] : [unitItem(id)], async () => {
|
|
94
|
+
const v = await this.#verify(view, bytes);
|
|
95
|
+
if ("outcome" in v)
|
|
96
|
+
return v.outcome;
|
|
97
|
+
return v.r.kind === "accepted"
|
|
98
|
+
? this.#apply(view, v.handler, v.r)
|
|
99
|
+
: this.#settle(view, v.handler, v.r);
|
|
100
|
+
});
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Receives the units of one DATA_BATCH (or any run of units) of the
|
|
104
|
+
* Resource `view` describes: each passes the LFCP checks of receive in
|
|
105
|
+
* order, unchanged, and the accepted ones reach the profile together
|
|
106
|
+
* through the handler's applyBatch, when it has one. Outcomes come in the
|
|
107
|
+
* order of `units`. An applied unit's objects and diagnostics are those
|
|
108
|
+
* of the whole batch, reported on the last unit merged; the units
|
|
109
|
+
* buffered before that the batch unblocked count as merged by the first.
|
|
110
|
+
* Before an equivocation is handled, the units accepted so far are
|
|
111
|
+
* applied, so its exclusion sees them.
|
|
112
|
+
*/
|
|
113
|
+
async receiveBatch(view, units) {
|
|
114
|
+
const handler = this.#handlers.get(view.state.dataProfile);
|
|
115
|
+
const outcomes = [];
|
|
116
|
+
if (handler?.applyBatch === undefined) {
|
|
117
|
+
for (const bytes of units)
|
|
118
|
+
outcomes.push(await this.receive(view, bytes));
|
|
119
|
+
return outcomes;
|
|
120
|
+
}
|
|
121
|
+
// Units that crashed the engine before go alone (receive), after the batch.
|
|
122
|
+
const R = view.state.resourceId;
|
|
123
|
+
await this.#recover(R);
|
|
124
|
+
const batch = [];
|
|
125
|
+
const alone = [];
|
|
126
|
+
for (const [index, bytes] of units.entries()) {
|
|
127
|
+
const id = this.#idOf(bytes);
|
|
128
|
+
if (id !== undefined && this.#guard.suspicion(R, unitItem(id)) > 0)
|
|
129
|
+
alone.push({ index, bytes });
|
|
130
|
+
else
|
|
131
|
+
batch.push({ index, bytes });
|
|
132
|
+
}
|
|
133
|
+
const items = batch.flatMap(({ bytes }) => {
|
|
134
|
+
const id = this.#idOf(bytes);
|
|
135
|
+
return id === undefined ? [] : [unitItem(id)];
|
|
136
|
+
});
|
|
137
|
+
const done = await this.#guard.run(R, items, () => this.#receiveBatch(view, handler, batch.map((b) => b.bytes)));
|
|
138
|
+
for (const [k, { index }] of batch.entries())
|
|
139
|
+
outcomes[index] = done[k];
|
|
140
|
+
for (const { index, bytes } of alone)
|
|
141
|
+
outcomes[index] = await this.receive(view, bytes);
|
|
142
|
+
return outcomes;
|
|
143
|
+
}
|
|
144
|
+
async #receiveBatch(view, handler, units) {
|
|
145
|
+
const outcomes = [];
|
|
146
|
+
let staged = [];
|
|
147
|
+
const flush = async () => {
|
|
148
|
+
const batch = staged;
|
|
149
|
+
staged = [];
|
|
150
|
+
for (const [index, outcome] of await this.#applyStaged(view, handler, batch))
|
|
151
|
+
outcomes[index] = outcome;
|
|
152
|
+
};
|
|
153
|
+
for (const [index, bytes] of units.entries()) {
|
|
154
|
+
const v = await this.#verify(view, bytes);
|
|
155
|
+
if ("outcome" in v)
|
|
156
|
+
outcomes[index] = v.outcome;
|
|
157
|
+
else if (v.r.kind === "accepted")
|
|
158
|
+
staged.push({ index, r: v.r });
|
|
159
|
+
else {
|
|
160
|
+
if (v.r.kind === "equivocation")
|
|
161
|
+
await flush();
|
|
162
|
+
outcomes[index] = await this.#settle(view, v.handler, v.r);
|
|
163
|
+
}
|
|
164
|
+
}
|
|
165
|
+
await flush();
|
|
166
|
+
return outcomes;
|
|
167
|
+
}
|
|
168
|
+
/** The LFCP checks of one unit (receiveDataUnit), or the outcome when there is no handler. */
|
|
169
|
+
async #verify(view, bytes) {
|
|
170
|
+
const dataProfile = view.state.dataProfile;
|
|
171
|
+
let row;
|
|
172
|
+
try {
|
|
173
|
+
row = dataUnitRow(bytes);
|
|
174
|
+
this.#seen.expect(row);
|
|
175
|
+
}
|
|
176
|
+
catch {
|
|
177
|
+
row = undefined; // receiveDataUnit reports it as MALFORMED_MESSAGE
|
|
178
|
+
}
|
|
179
|
+
const handler = this.#handlers.get(dataProfile);
|
|
180
|
+
if (handler === undefined)
|
|
181
|
+
return { outcome: await this.#unsupported(view, bytes, dataProfile) };
|
|
182
|
+
const codec = row === undefined
|
|
183
|
+
? unreachableCodec(dataProfile)
|
|
184
|
+
: handler.codecFor({ resourceId: row.resourceId, actor: row.actor });
|
|
185
|
+
const r = await receiveDataUnit(view, bytes, {
|
|
186
|
+
...this.#options,
|
|
187
|
+
seen: this.#seen,
|
|
188
|
+
profile: codec,
|
|
189
|
+
});
|
|
190
|
+
// A decode that trapped the engine is not this unit's local failure.
|
|
191
|
+
if (r.kind === "local-failure" && isEngineTrap(r.error))
|
|
192
|
+
throw r.error;
|
|
193
|
+
return { handler, r };
|
|
194
|
+
}
|
|
195
|
+
/** A unit LFCP did not accept for merging: held, quarantined, failed locally, equivocating, … */
|
|
196
|
+
async #settle(view, handler, r) {
|
|
197
|
+
const status = (unitId, s, detail) => ({
|
|
198
|
+
op: "set-data-unit-status",
|
|
199
|
+
unitId,
|
|
200
|
+
status: s,
|
|
201
|
+
...(detail === undefined ? {} : { detail }),
|
|
202
|
+
});
|
|
203
|
+
switch (r.kind) {
|
|
204
|
+
case "held":
|
|
205
|
+
await this.#write([status(r.unitId, "held", r.reason)]);
|
|
206
|
+
return r;
|
|
207
|
+
case "quarantined":
|
|
208
|
+
await this.#write([status(r.unitId, "quarantined", r.reason)]);
|
|
209
|
+
return r;
|
|
210
|
+
case "local-failure":
|
|
211
|
+
await this.#write([status(r.unitId, "local-failure", `${r.reason}: ${r.message}`)]);
|
|
212
|
+
return r;
|
|
213
|
+
case "equivocation":
|
|
214
|
+
return this.#equivocation(view, handler, r);
|
|
215
|
+
default:
|
|
216
|
+
return r; // duplicate (harmless replay) or rejected (never stored)
|
|
217
|
+
}
|
|
218
|
+
}
|
|
219
|
+
async #apply(view, handler, r) {
|
|
220
|
+
let result;
|
|
221
|
+
try {
|
|
222
|
+
result = handler.apply(profileUnit(view, r), r.value);
|
|
223
|
+
}
|
|
224
|
+
catch (e) {
|
|
225
|
+
if (isEngineTrap(e))
|
|
226
|
+
throw e; // the engine is gone: not the unit's refusal
|
|
227
|
+
const code = e.code;
|
|
228
|
+
return this.#rejected(r, handler.dataProfile, typeof code === "string" ? code : "PROFILE_REJECTED", e instanceof Error ? e.message : String(e));
|
|
229
|
+
}
|
|
230
|
+
return this.#merged(view, handler, r, result);
|
|
231
|
+
}
|
|
232
|
+
/** The staged accepted units through the handler's applyBatch, as #apply would one by one. */
|
|
233
|
+
async #applyStaged(view, handler, staged) {
|
|
234
|
+
if (staged.length === 0 || handler.applyBatch === undefined)
|
|
235
|
+
return [];
|
|
236
|
+
let batch;
|
|
237
|
+
try {
|
|
238
|
+
batch = handler.applyBatch(staged.map(({ r }) => ({ unit: profileUnit(view, r), value: r.value })));
|
|
239
|
+
}
|
|
240
|
+
catch (e) {
|
|
241
|
+
if (isEngineTrap(e))
|
|
242
|
+
throw e;
|
|
243
|
+
// The handler failed as a whole: apply one by one, which isolates the failing unit.
|
|
244
|
+
const out = [];
|
|
245
|
+
for (const { index, r } of staged)
|
|
246
|
+
out.push([index, await this.#apply(view, handler, r)]);
|
|
247
|
+
return out;
|
|
248
|
+
}
|
|
249
|
+
const merged = new Set(batch.merged.map((id) => toHex(id)));
|
|
250
|
+
const rejected = new Map(batch.rejected.map((x) => [toHex(x.unitId), x]));
|
|
251
|
+
const inBatch = new Set(staged.map(({ r }) => toHex(r.unitId)));
|
|
252
|
+
const unblocked = batch.merged.filter((id) => !inBatch.has(toHex(id)));
|
|
253
|
+
let last = -1;
|
|
254
|
+
staged.forEach(({ r }, k) => {
|
|
255
|
+
if (merged.has(toHex(r.unitId)))
|
|
256
|
+
last = k;
|
|
257
|
+
});
|
|
258
|
+
const out = [];
|
|
259
|
+
for (const [k, { index, r }] of staged.entries()) {
|
|
260
|
+
const key = toHex(r.unitId);
|
|
261
|
+
const refused = rejected.get(key);
|
|
262
|
+
if (refused !== undefined) {
|
|
263
|
+
out.push([
|
|
264
|
+
index,
|
|
265
|
+
await this.#rejected(r, handler.dataProfile, refused.code, refused.message),
|
|
266
|
+
]);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
const now = merged.has(key);
|
|
270
|
+
out.push([
|
|
271
|
+
index,
|
|
272
|
+
await this.#merged(view, handler, r, {
|
|
273
|
+
merged: [...(now ? [r.unitId] : []), ...(k === 0 ? unblocked : [])],
|
|
274
|
+
objects: k === last ? batch.objects : [],
|
|
275
|
+
diagnostics: k === last ? batch.diagnostics : [],
|
|
276
|
+
...(now ? {} : { pending: "buffered by the profile" }),
|
|
277
|
+
}, false),
|
|
278
|
+
]);
|
|
279
|
+
}
|
|
280
|
+
// §26.2 (G-DP1-GAP): held units whose previous names a unit of the batch
|
|
281
|
+
// may link now. One scan for the whole batch keeps it linear; each
|
|
282
|
+
// released unit is reported with the unit it links to.
|
|
283
|
+
const byPrevious = new Map(staged.map(({ index, r }) => [toHex(r.unitId), index]));
|
|
284
|
+
const held = await this.#storage.dataUnits.withStatus(view.state.resourceId, "held");
|
|
285
|
+
const releasedFor = new Map();
|
|
286
|
+
for (const h of held) {
|
|
287
|
+
const previous = parseDataUnit(h.bytes).payload.prevDataUnitId;
|
|
288
|
+
const owner = previous === null ? undefined : byPrevious.get(toHex(previous));
|
|
289
|
+
if (owner === undefined)
|
|
290
|
+
continue;
|
|
291
|
+
if ((await this.#storage.dataUnits.get(h.unitId))?.status !== "held")
|
|
292
|
+
continue;
|
|
293
|
+
releasedFor.set(owner, [
|
|
294
|
+
...(releasedFor.get(owner) ?? []),
|
|
295
|
+
await this.receive(view, h.bytes),
|
|
296
|
+
]);
|
|
297
|
+
}
|
|
298
|
+
return out.map(([index, outcome]) => {
|
|
299
|
+
const extra = releasedFor.get(index);
|
|
300
|
+
if (extra === undefined || !("released" in outcome))
|
|
301
|
+
return [index, outcome];
|
|
302
|
+
return [
|
|
303
|
+
index,
|
|
304
|
+
Object.freeze({ ...outcome, released: Object.freeze([...outcome.released, ...extra]) }),
|
|
305
|
+
];
|
|
306
|
+
});
|
|
307
|
+
}
|
|
308
|
+
/** An accepted unit the profile refused: never merged. */
|
|
309
|
+
async #rejected(r, dataProfile, code, message) {
|
|
310
|
+
await this.#write([
|
|
311
|
+
{ op: "set-data-unit-status", unitId: r.unitId, status: "profile-rejected", detail: message },
|
|
312
|
+
]);
|
|
313
|
+
return Object.freeze({
|
|
314
|
+
kind: "profile-rejected",
|
|
315
|
+
unitId: r.unitId,
|
|
316
|
+
dataProfile,
|
|
317
|
+
code,
|
|
318
|
+
message,
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
/** An accepted unit the profile merged or buffered: its statuses, then held units it releases. */
|
|
322
|
+
async #merged(view, handler, r, result, releaseHeld = true) {
|
|
323
|
+
const dataProfile = handler.dataProfile;
|
|
324
|
+
const mergedNow = result.merged.some((id) => bytesEqual(id, r.unitId));
|
|
325
|
+
const alsoMerged = result.merged.filter((id) => !bytesEqual(id, r.unitId));
|
|
326
|
+
await this.#write([
|
|
327
|
+
mergedNow
|
|
328
|
+
? { op: "set-data-unit-status", unitId: r.unitId, status: "merged" }
|
|
329
|
+
: {
|
|
330
|
+
op: "set-data-unit-status",
|
|
331
|
+
unitId: r.unitId,
|
|
332
|
+
status: "profile-pending",
|
|
333
|
+
detail: result.pending ?? "buffered by the profile",
|
|
334
|
+
},
|
|
335
|
+
...alsoMerged.map((unitId) => ({ op: "set-data-unit-status", unitId, status: "merged" })),
|
|
336
|
+
]);
|
|
337
|
+
// §26.2 (G-DP1, G-DP1-GAP): a held unit of this actor above it may link now, across a gap.
|
|
338
|
+
// A batch releases its held units once, at its end (#applyStaged).
|
|
339
|
+
const next = releaseHeld
|
|
340
|
+
? await this.#storage.dataUnits.range(view.state.resourceId, r.actor, actorSequence(r.seq + 1n), actorSequence(2n ** 64n - 1n))
|
|
341
|
+
: [];
|
|
342
|
+
const released = [];
|
|
343
|
+
// Only a held unit whose previous names the unit just accepted can link now.
|
|
344
|
+
for (const held of next.filter((u) => u.status === "held")) {
|
|
345
|
+
const previous = parseDataUnit(held.bytes).payload.prevDataUnitId;
|
|
346
|
+
if (previous === null || !bytesEqual(previous, r.unitId))
|
|
347
|
+
continue;
|
|
348
|
+
// A retry above may already have released it.
|
|
349
|
+
if ((await this.#storage.dataUnits.get(held.unitId))?.status !== "held")
|
|
350
|
+
continue;
|
|
351
|
+
released.push(await this.receive(view, held.bytes));
|
|
352
|
+
}
|
|
353
|
+
const base = {
|
|
354
|
+
unitId: r.unitId,
|
|
355
|
+
dataProfile,
|
|
356
|
+
actor: r.actor,
|
|
357
|
+
seq: r.seq,
|
|
358
|
+
epoch: r.epoch,
|
|
359
|
+
haveEligible: false,
|
|
360
|
+
alsoMerged: Object.freeze(alsoMerged),
|
|
361
|
+
released: Object.freeze(released),
|
|
362
|
+
};
|
|
363
|
+
return Object.freeze(mergedNow
|
|
364
|
+
? {
|
|
365
|
+
...base,
|
|
366
|
+
kind: "applied",
|
|
367
|
+
objects: result.objects,
|
|
368
|
+
diagnostics: result.diagnostics,
|
|
369
|
+
}
|
|
370
|
+
: {
|
|
371
|
+
...base,
|
|
372
|
+
kind: "profile-pending",
|
|
373
|
+
detail: result.pending ?? "buffered by the profile",
|
|
374
|
+
});
|
|
375
|
+
}
|
|
376
|
+
// §26.2 (G-DP5): exclude every merged unit of an equivocating set.
|
|
377
|
+
async #equivocation(view, handler, r) {
|
|
378
|
+
const stored = (await Promise.all(r.unitIds.map((id) => this.#storage.dataUnits.get(id)))).filter((u) => u !== undefined);
|
|
379
|
+
const merged = stored.filter((u) => LFCP_ACCEPTED.includes(u.status)).map((u) => u.unitId);
|
|
380
|
+
const result = merged.length > 0 ? handler.exclude(merged) : { objects: [], pending: [] };
|
|
381
|
+
await this.#write([
|
|
382
|
+
...stored.flatMap((u) => [
|
|
383
|
+
{
|
|
384
|
+
op: "set-data-unit-status",
|
|
385
|
+
unitId: u.unitId,
|
|
386
|
+
status: "equivocation",
|
|
387
|
+
detail: "ACTOR_EQUIVOCATION (G-DP5)",
|
|
388
|
+
},
|
|
389
|
+
{ op: "set-accepted", unitId: u.unitId, accepted: false },
|
|
390
|
+
]),
|
|
391
|
+
...result.pending.map((unitId) => ({
|
|
392
|
+
op: "set-data-unit-status",
|
|
393
|
+
unitId,
|
|
394
|
+
status: "profile-pending",
|
|
395
|
+
detail: "builds on an equivocating unit",
|
|
396
|
+
})),
|
|
397
|
+
]);
|
|
398
|
+
// The actor's latest accepted unit may now be lower: held units can link (G-DP1-GAP).
|
|
399
|
+
const released = merged.length > 0 ? await this.#retryHeld(view, [r.actor]) : [];
|
|
400
|
+
return Object.freeze({
|
|
401
|
+
...r,
|
|
402
|
+
excluded: Object.freeze(merged),
|
|
403
|
+
objects: result.objects,
|
|
404
|
+
pending: result.pending,
|
|
405
|
+
released,
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
/**
|
|
409
|
+
* After a restart: the stored, LFCP-accepted units (merged or buffered by
|
|
410
|
+
* the profile) whose content the restored profile state does not hold
|
|
411
|
+
* (merged after its last checkpoint), decrypted again from their exact
|
|
412
|
+
* stored bytes and applied, in (actor, seq) order. Units covered by a
|
|
413
|
+
* Snapshot are not replayed (their content came with it). Needs the
|
|
414
|
+
* handler's has(); returns what was replayed and what could not be (no
|
|
415
|
+
* DEK, refused). LFCP checks are not repeated: the units passed them when
|
|
416
|
+
* they were accepted.
|
|
417
|
+
*/
|
|
418
|
+
async replayStored(view) {
|
|
419
|
+
const handler = this.#handlers.get(view.state.dataProfile);
|
|
420
|
+
const replayed = [];
|
|
421
|
+
const skipped = [];
|
|
422
|
+
if (handler?.has === undefined) {
|
|
423
|
+
await this.#recover(view.state.resourceId);
|
|
424
|
+
const crashed = this.#crashed;
|
|
425
|
+
this.#crashed = [];
|
|
426
|
+
return { replayed, skipped, crashed };
|
|
427
|
+
}
|
|
428
|
+
const resource = view.state.resourceId;
|
|
429
|
+
// A crash during an earlier apply: suspects, or units quarantined now.
|
|
430
|
+
await this.#recover(resource);
|
|
431
|
+
const crashed = this.#crashed;
|
|
432
|
+
this.#crashed = [];
|
|
433
|
+
// Also units accepted by LFCP whose merge was never recorded (a crash
|
|
434
|
+
// between the accepted mark and the status write leaves them "seen" or "held").
|
|
435
|
+
const stored = [
|
|
436
|
+
...(await this.#storage.dataUnits.withStatus(resource, "merged")),
|
|
437
|
+
...(await this.#storage.dataUnits.withStatus(resource, "profile-pending")),
|
|
438
|
+
...(await this.#storage.dataUnits.withStatus(resource, "seen")),
|
|
439
|
+
...(await this.#storage.dataUnits.withStatus(resource, "held")),
|
|
440
|
+
]
|
|
441
|
+
.filter((u) => u.accepted && u.detail !== "covered by a Snapshot" && !handler.has?.(u.unitId))
|
|
442
|
+
.sort((a, b) => (a.actorSeq < b.actorSeq ? -1 : a.actorSeq > b.actorSeq ? 1 : 0));
|
|
443
|
+
const merged = [];
|
|
444
|
+
const pendingNow = [];
|
|
445
|
+
// Suspects of an earlier crash replay alone, each under its own record.
|
|
446
|
+
const suspect = (u) => this.#guard.suspicion(resource, unitItem(u.unitId)) > 0;
|
|
447
|
+
const groups = [...stored.filter(suspect).map((u) => [u]), stored.filter((u) => !suspect(u))];
|
|
448
|
+
for (const group of groups)
|
|
449
|
+
if (group.length > 0)
|
|
450
|
+
await this.#guard.run(resource, group.map((u) => unitItem(u.unitId)), () => this.#replayGroup(handler, group, { merged, pendingNow, replayed, skipped }));
|
|
451
|
+
await this.#write([
|
|
452
|
+
...pendingNow
|
|
453
|
+
.filter((id) => !merged.some((m) => bytesEqual(m, id)))
|
|
454
|
+
.map((unitId) => ({
|
|
455
|
+
op: "set-data-unit-status",
|
|
456
|
+
unitId,
|
|
457
|
+
status: "profile-pending",
|
|
458
|
+
detail: "replayed; waiting for content it builds on",
|
|
459
|
+
})),
|
|
460
|
+
...merged.map((unitId) => ({ op: "set-data-unit-status", unitId, status: "merged" })),
|
|
461
|
+
]);
|
|
462
|
+
return {
|
|
463
|
+
replayed: Object.freeze(replayed),
|
|
464
|
+
skipped: Object.freeze(skipped),
|
|
465
|
+
crashed: Object.freeze(crashed),
|
|
466
|
+
};
|
|
467
|
+
}
|
|
468
|
+
/** Decrypts, decodes and applies replayed units (replayStored), adding to `acc`. */
|
|
469
|
+
async #replayGroup(handler, group, acc) {
|
|
470
|
+
const { merged, pendingNow, replayed, skipped } = acc;
|
|
471
|
+
const decoded = [];
|
|
472
|
+
for (const u of group) {
|
|
473
|
+
const dek = await this.#options.dek(u.dataEpoch);
|
|
474
|
+
if (dek === undefined) {
|
|
475
|
+
skipped.push({ unitId: u.unitId, reason: `no DEK for epoch ${u.dataEpoch}` });
|
|
476
|
+
continue;
|
|
477
|
+
}
|
|
478
|
+
try {
|
|
479
|
+
const p = parseDataUnit(u.bytes).payload;
|
|
480
|
+
const key = deriveActorDataKey(dek, p.resourceId, p.dataEpoch, p.actor);
|
|
481
|
+
const plaintext = decryptDataUnit(key, p.actorSeq, dataUnitAad(p), p.ciphertext);
|
|
482
|
+
const value = handler
|
|
483
|
+
.codecFor({ resourceId: p.resourceId, actor: p.actor })
|
|
484
|
+
.decode(plaintext);
|
|
485
|
+
decoded.push({
|
|
486
|
+
unit: {
|
|
487
|
+
unitId: u.unitId,
|
|
488
|
+
resourceId: p.resourceId,
|
|
489
|
+
actor: p.actor,
|
|
490
|
+
seq: p.actorSeq,
|
|
491
|
+
epoch: p.dataEpoch,
|
|
492
|
+
},
|
|
493
|
+
value,
|
|
494
|
+
});
|
|
495
|
+
}
|
|
496
|
+
catch (e) {
|
|
497
|
+
if (isEngineTrap(e))
|
|
498
|
+
throw e;
|
|
499
|
+
skipped.push({ unitId: u.unitId, reason: e instanceof Error ? e.message : String(e) });
|
|
500
|
+
}
|
|
501
|
+
}
|
|
502
|
+
if (handler.applyBatch !== undefined && decoded.length > 0) {
|
|
503
|
+
// One profile call for the whole replay (a restart after many changes).
|
|
504
|
+
const batch = handler.applyBatch(decoded);
|
|
505
|
+
const refused = new Map(batch.rejected.map((x) => [toHex(x.unitId), x]));
|
|
506
|
+
merged.push(...batch.merged);
|
|
507
|
+
for (const { unit } of decoded) {
|
|
508
|
+
const no = refused.get(toHex(unit.unitId));
|
|
509
|
+
if (no !== undefined) {
|
|
510
|
+
skipped.push({ unitId: unit.unitId, reason: `${no.code}: ${no.message}` });
|
|
511
|
+
continue;
|
|
512
|
+
}
|
|
513
|
+
if (!batch.merged.some((id) => bytesEqual(id, unit.unitId)))
|
|
514
|
+
pendingNow.push(unit.unitId);
|
|
515
|
+
replayed.push(unit.unitId);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
else
|
|
519
|
+
for (const { unit, value } of decoded) {
|
|
520
|
+
try {
|
|
521
|
+
const r = handler.apply(unit, value);
|
|
522
|
+
merged.push(...r.merged);
|
|
523
|
+
if (!r.merged.some((id) => bytesEqual(id, unit.unitId)))
|
|
524
|
+
pendingNow.push(unit.unitId);
|
|
525
|
+
replayed.push(unit.unitId);
|
|
526
|
+
}
|
|
527
|
+
catch (e) {
|
|
528
|
+
if (isEngineTrap(e))
|
|
529
|
+
throw e;
|
|
530
|
+
skipped.push({ unitId: unit.unitId, reason: e instanceof Error ? e.message : String(e) });
|
|
531
|
+
}
|
|
532
|
+
}
|
|
533
|
+
}
|
|
534
|
+
/**
|
|
535
|
+
* A unit inside the frontier of a Snapshot this client loaded (§29, §66
|
|
536
|
+
* step 3): its content is in the Snapshot, so it is not decrypted or
|
|
537
|
+
* merged again. It gets the DEK-free LFCP checks (structure, signature,
|
|
538
|
+
* equivocation, data/write at its head, epoch cutoff) and is then marked
|
|
539
|
+
* accepted, so the actor's next unit links to it (§26.2). Its own link to
|
|
540
|
+
* the previous unit is not checked: that unit is attested by the signed,
|
|
541
|
+
* authorized Snapshot and was never received. A replay is a duplicate.
|
|
542
|
+
*/
|
|
543
|
+
async acceptCovered(view, bytes) {
|
|
544
|
+
const dataProfile = view.state.dataProfile;
|
|
545
|
+
try {
|
|
546
|
+
this.#seen.expect(dataUnitRow(bytes));
|
|
547
|
+
}
|
|
548
|
+
catch {
|
|
549
|
+
// checkDataUnit reports it as MALFORMED_MESSAGE
|
|
550
|
+
}
|
|
551
|
+
const c = await checkDataUnit(view, bytes, this.#seen, this.#options);
|
|
552
|
+
if (c.kind === "equivocation") {
|
|
553
|
+
const handler = this.#handlers.get(dataProfile);
|
|
554
|
+
return handler === undefined
|
|
555
|
+
? Object.freeze({ ...c, excluded: [], objects: [], pending: [], released: [] })
|
|
556
|
+
: this.#equivocation(view, handler, c);
|
|
557
|
+
}
|
|
558
|
+
if (c.kind !== "valid") {
|
|
559
|
+
if (c.kind === "quarantined")
|
|
560
|
+
await this.#write([
|
|
561
|
+
{ op: "set-data-unit-status", unitId: c.unitId, status: "quarantined", detail: c.reason },
|
|
562
|
+
]);
|
|
563
|
+
return c;
|
|
564
|
+
}
|
|
565
|
+
const p = c.parsed.payload;
|
|
566
|
+
if ((await this.#storage.dataUnits.acceptedAt(p.resourceId, p.actor, p.actorSeq)) !== undefined)
|
|
567
|
+
return Object.freeze({ kind: "duplicate", unitId: c.unitId });
|
|
568
|
+
await this.#write([
|
|
569
|
+
{
|
|
570
|
+
op: "set-data-unit-status",
|
|
571
|
+
unitId: c.unitId,
|
|
572
|
+
status: "merged",
|
|
573
|
+
detail: "covered by a Snapshot",
|
|
574
|
+
},
|
|
575
|
+
{ op: "set-accepted", unitId: c.unitId, accepted: true },
|
|
576
|
+
]);
|
|
577
|
+
// §26.2 (G-DP1, G-DP1-GAP): a held unit of this actor above it may link now, across a gap.
|
|
578
|
+
const next = await this.#storage.dataUnits.range(p.resourceId, p.actor, actorSequence(p.actorSeq + 1n), actorSequence(2n ** 64n - 1n));
|
|
579
|
+
const released = [];
|
|
580
|
+
// Only a held unit whose previous names the unit just accepted can link now.
|
|
581
|
+
for (const held of next.filter((u) => u.status === "held")) {
|
|
582
|
+
const previous = parseDataUnit(held.bytes).payload.prevDataUnitId;
|
|
583
|
+
if (previous === null || !bytesEqual(previous, c.unitId))
|
|
584
|
+
continue;
|
|
585
|
+
// A retry above may already have released it.
|
|
586
|
+
if ((await this.#storage.dataUnits.get(held.unitId))?.status !== "held")
|
|
587
|
+
continue;
|
|
588
|
+
released.push(await this.receive(view, held.bytes));
|
|
589
|
+
}
|
|
590
|
+
return Object.freeze({
|
|
591
|
+
kind: "covered",
|
|
592
|
+
unitId: c.unitId,
|
|
593
|
+
dataProfile,
|
|
594
|
+
released: Object.freeze(released),
|
|
595
|
+
});
|
|
596
|
+
}
|
|
597
|
+
async #unsupported(view, bytes, dataProfile) {
|
|
598
|
+
// DEK-free checks only: the plaintext of an unknown profile is never decrypted.
|
|
599
|
+
const c = await checkDataUnit(view, bytes, this.#seen, this.#options);
|
|
600
|
+
if (c.kind === "equivocation")
|
|
601
|
+
return Object.freeze({ ...c, excluded: [], objects: [], pending: [], released: [] });
|
|
602
|
+
if (c.kind !== "valid")
|
|
603
|
+
return c;
|
|
604
|
+
await this.#write([
|
|
605
|
+
{ op: "set-data-unit-status", unitId: c.unitId, status: "profile-unsupported" },
|
|
606
|
+
]);
|
|
607
|
+
return Object.freeze({
|
|
608
|
+
kind: "profile-unsupported",
|
|
609
|
+
code: "PROFILE_UNSUPPORTED",
|
|
610
|
+
unitId: c.unitId,
|
|
611
|
+
dataProfile,
|
|
612
|
+
});
|
|
613
|
+
}
|
|
614
|
+
/**
|
|
615
|
+
* LFCP-WIRE-01 §19.1 (G-EP7): the single entry point the sync engine calls with
|
|
616
|
+
* every newly validated Control view that may carry a new Key Epoch.
|
|
617
|
+
* Merged (or profile-buffered) units that the view now puts beyond an
|
|
618
|
+
* epoch cutoff are taken out of the profile state, which is rebuilt from
|
|
619
|
+
* the remaining accepted units; they are quarantined (STALE_DATA_EPOCH)
|
|
620
|
+
* and un-accepted.
|
|
621
|
+
*/
|
|
622
|
+
// PROVISIONAL (SNAP-EP): a Key Epoch learned after a Snapshot was loaded
|
|
623
|
+
// that puts any unit the Snapshot covers beyond its cutoff drops the
|
|
624
|
+
// Snapshot-derived state: the profile is rebuilt from accepted units only,
|
|
625
|
+
// the covered units are fetched again, and G-EP7 then applies normally.
|
|
626
|
+
async #dropCutSnapshots(view, handler) {
|
|
627
|
+
const resource = view.state.resourceId;
|
|
628
|
+
const snapshots = await this.#storage.snapshots.list(resource);
|
|
629
|
+
if (snapshots.length === 0 || handler.reset === undefined)
|
|
630
|
+
return false;
|
|
631
|
+
const covered = (await this.#storage.dataUnits.withStatus(resource, "merged")).filter((u) => u.accepted && u.detail === "covered by a Snapshot");
|
|
632
|
+
const cut = snapshots.some((x) => beyondCutoff(view, x.dataEpoch, canonicalFrontierFromCbor(decodeDeterministic(x.frontier))) !== undefined) || covered.some((u) => classifyDataUnit(view, u).kind === "quarantine");
|
|
633
|
+
if (!cut)
|
|
634
|
+
return false;
|
|
635
|
+
await this.#write([
|
|
636
|
+
...snapshots.map((x) => ({ op: "delete-snapshot", snapshotId: x.snapshotId })),
|
|
637
|
+
...covered.flatMap((u) => [
|
|
638
|
+
{ op: "set-accepted", unitId: u.unitId, accepted: false },
|
|
639
|
+
{
|
|
640
|
+
op: "set-data-unit-status",
|
|
641
|
+
unitId: u.unitId,
|
|
642
|
+
status: "seen",
|
|
643
|
+
detail: "Snapshot dropped (SNAP-EP)",
|
|
644
|
+
},
|
|
645
|
+
]),
|
|
646
|
+
]);
|
|
647
|
+
handler.reset();
|
|
648
|
+
await this.replayStored(view);
|
|
649
|
+
return true;
|
|
650
|
+
}
|
|
651
|
+
/** The distinct actors of stored units. */
|
|
652
|
+
async #actorsOf(unitIds) {
|
|
653
|
+
const byHex = new Map();
|
|
654
|
+
for (const id of unitIds) {
|
|
655
|
+
const u = await this.#storage.dataUnits.get(id);
|
|
656
|
+
if (u !== undefined)
|
|
657
|
+
byHex.set(toHex(u.actor), u.actor);
|
|
658
|
+
}
|
|
659
|
+
return [...byHex.values()];
|
|
660
|
+
}
|
|
661
|
+
/**
|
|
662
|
+
* §26.2 (G-DP1-GAP): after units of `actors` stopped being accepted, a
|
|
663
|
+
* held unit may now name the actor's latest accepted unit. Every held
|
|
664
|
+
* unit of those actors is received again, lowest sequence first.
|
|
665
|
+
*/
|
|
666
|
+
async #retryHeld(view, actors) {
|
|
667
|
+
const released = [];
|
|
668
|
+
for (const actor of actors) {
|
|
669
|
+
const held = (await this.#storage.dataUnits.range(view.state.resourceId, actor, actorSequence(1n), actorSequence(2n ** 64n - 1n))).filter((u) => u.status === "held");
|
|
670
|
+
for (const u of held) {
|
|
671
|
+
if ((await this.#storage.dataUnits.get(u.unitId))?.status !== "held")
|
|
672
|
+
continue;
|
|
673
|
+
released.push(await this.receive(view, u.bytes));
|
|
674
|
+
}
|
|
675
|
+
}
|
|
676
|
+
return released;
|
|
677
|
+
}
|
|
678
|
+
async reconcileEpochs(view) {
|
|
679
|
+
const handler = this.#handlers.get(view.state.dataProfile);
|
|
680
|
+
if (handler === undefined)
|
|
681
|
+
return Object.freeze({
|
|
682
|
+
excluded: [],
|
|
683
|
+
objects: [],
|
|
684
|
+
pending: [],
|
|
685
|
+
snapshotDropped: false,
|
|
686
|
+
released: [],
|
|
687
|
+
});
|
|
688
|
+
const resource = view.state.resourceId;
|
|
689
|
+
const snapshotDropped = await this.#dropCutSnapshots(view, handler);
|
|
690
|
+
const empty = Object.freeze({
|
|
691
|
+
excluded: [],
|
|
692
|
+
objects: [],
|
|
693
|
+
pending: [],
|
|
694
|
+
snapshotDropped,
|
|
695
|
+
released: [],
|
|
696
|
+
});
|
|
697
|
+
const candidates = [
|
|
698
|
+
...(await this.#storage.dataUnits.withStatus(resource, "merged")),
|
|
699
|
+
...(await this.#storage.dataUnits.withStatus(resource, "profile-pending")),
|
|
700
|
+
].filter((u) => u.accepted);
|
|
701
|
+
const excluded = [];
|
|
702
|
+
for (const u of candidates) {
|
|
703
|
+
const c = classifyDataUnit(view, u);
|
|
704
|
+
if (c.kind === "quarantine")
|
|
705
|
+
excluded.push({
|
|
706
|
+
unitId: u.unitId,
|
|
707
|
+
quarantine: { code: c.code, reason: c.reason, epoch: c.epoch, closedBy: c.closedBy },
|
|
708
|
+
});
|
|
709
|
+
}
|
|
710
|
+
if (excluded.length === 0)
|
|
711
|
+
return empty;
|
|
712
|
+
const result = handler.exclude(excluded.map((e) => e.unitId));
|
|
713
|
+
await this.#write([
|
|
714
|
+
...excluded.flatMap((e) => [
|
|
715
|
+
{
|
|
716
|
+
op: "set-data-unit-status",
|
|
717
|
+
unitId: e.unitId,
|
|
718
|
+
status: "quarantined",
|
|
719
|
+
detail: e.quarantine.reason,
|
|
720
|
+
},
|
|
721
|
+
{ op: "set-accepted", unitId: e.unitId, accepted: false },
|
|
722
|
+
]),
|
|
723
|
+
...result.pending.map((unitId) => ({
|
|
724
|
+
op: "set-data-unit-status",
|
|
725
|
+
unitId,
|
|
726
|
+
status: "profile-pending",
|
|
727
|
+
detail: "builds on an excluded unit",
|
|
728
|
+
})),
|
|
729
|
+
]);
|
|
730
|
+
const actors = await this.#actorsOf(excluded.map((e) => e.unitId));
|
|
731
|
+
const released = await this.#retryHeld(view, actors);
|
|
732
|
+
return Object.freeze({
|
|
733
|
+
snapshotDropped,
|
|
734
|
+
excluded: Object.freeze(excluded),
|
|
735
|
+
objects: result.objects,
|
|
736
|
+
pending: result.pending,
|
|
737
|
+
released,
|
|
738
|
+
});
|
|
739
|
+
}
|
|
740
|
+
}
|
|
741
|
+
//# sourceMappingURL=apply.js.map
|