edytor 0.1.0-next.21 → 0.1.0-next.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cloudflare/DocumentRoom.d.ts +302 -10
- package/dist/cloudflare/DocumentRoom.js +965 -88
- package/dist/cloudflare/index.d.ts +1 -1
- package/dist/cloudflare/index.js +1 -1
- package/dist/cloudflare/routeDocumentSocket.js +30 -6
- package/dist/collaboration/awarenessSelection.d.ts +7 -1
- package/dist/collaboration/awarenessSelection.js +7 -1
- package/dist/collaboration/index.d.ts +2 -2
- package/dist/collaboration/index.js +2 -2
- package/dist/collaboration/providers.d.ts +4 -0
- package/dist/collaboration/providers.js +4 -0
- package/dist/components/Edytor.svelte +14 -2
- package/dist/components/Edytor.svelte.d.ts +8 -1
- package/dist/crdt/admission.d.ts +6 -1
- package/dist/crdt/admission.js +9 -2
- package/dist/crdt/data.d.ts +3 -1
- package/dist/crdt/data.js +42 -32
- package/dist/crdt/document.d.ts +31 -8
- package/dist/crdt/document.js +53 -18
- package/dist/crdt/edytor-doc.d.ts +31 -3
- package/dist/crdt/edytor-doc.js +131 -14
- package/dist/crdt/engine.d.ts +6 -0
- package/dist/crdt/engine.js +7 -0
- package/dist/crdt/flow.d.ts +2 -0
- package/dist/crdt/flow.js +2 -2
- package/dist/crdt/index.d.ts +20 -9
- package/dist/crdt/index.js +3 -3
- package/dist/crdt/migration/migrate.js +9 -3
- package/dist/crdt/placement/model.d.ts +9 -1
- package/dist/crdt/placement/model.js +29 -10
- package/dist/crdt/placement/rank.d.ts +20 -0
- package/dist/crdt/placement/rank.js +79 -0
- package/dist/crdt/protocols/envelope.d.ts +20 -1
- package/dist/crdt/protocols/envelope.js +14 -2
- package/dist/crdt/providers/container.d.ts +15 -0
- package/dist/crdt/providers/container.js +17 -3
- package/dist/crdt/providers/index.d.ts +21 -1
- package/dist/crdt/providers/index.js +86 -3
- package/dist/crdt/providers/indexeddb.js +60 -40
- package/dist/crdt/providers/room.d.ts +18 -2
- package/dist/crdt/providers/room.js +29 -5
- package/dist/crdt/providers/websocket.d.ts +9 -1
- package/dist/crdt/providers/websocket.js +21 -6
- package/dist/crdt/storage.d.ts +21 -0
- package/dist/crdt/storage.js +37 -0
- package/dist/crdt/structs.d.ts +17 -0
- package/dist/crdt/structs.js +41 -0
- package/dist/crdt/text/model.d.ts +9 -1
- package/dist/crdt/text/model.js +24 -14
- package/dist/crdt/text/rows.d.ts +94 -0
- package/dist/crdt/text/rows.js +193 -0
- package/dist/crdt/text/runs.d.ts +30 -10
- package/dist/crdt/text/runs.js +1235 -264
- package/dist/edytor.svelte.d.ts +3 -1
- package/dist/edytor.svelte.js +9 -6
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/plugins/image/ImageEmpty.svelte +19 -6
- package/dist/plugins/image/ImagePlugin.svelte +9 -3
- package/dist/plugins/image/ImagePlugin.svelte.d.ts +2 -2
- package/dist/plugins/image/image.d.ts +16 -0
- package/dist/plugins/image/image.js +19 -0
- package/dist/plugins/index.d.ts +1 -1
- package/dist/plugins/index.js +1 -1
- package/dist/plugins.d.ts +11 -0
- package/dist/session/commands.d.ts +1 -0
- package/dist/session/commands.js +4 -1
- package/package.json +1 -1
|
@@ -34,8 +34,9 @@
|
|
|
34
34
|
* what storage does not.
|
|
35
35
|
* - every INTEGRATED update is appended to SQLite as one record split into
|
|
36
36
|
* rows ≤ `maxRowBytes` (2 MB row cap) in one `transactionSync`, then
|
|
37
|
-
* broadcast; after `compactAfter` update records the rows are
|
|
38
|
-
*
|
|
37
|
+
* broadcast; after `compactAfter` update records the rows are replaced
|
|
38
|
+
* by one snapshot record of the live document's state (its healed
|
|
39
|
+
* state, deleted content collected: P2), after the message is
|
|
39
40
|
* acknowledged — never inside the engine's `update` observer, where one
|
|
40
41
|
* throw would silence every later emit. Memory never runs ahead of
|
|
41
42
|
* storage: when an append fails, nothing is relayed or acknowledged, the
|
|
@@ -51,6 +52,11 @@
|
|
|
51
52
|
* left out of the ack), the rest applied. Compaction reclaims the
|
|
52
53
|
* waiting deletes of client ids no user registered and no socket holds;
|
|
53
54
|
* `dropWaitingDeletes()` every one.
|
|
55
|
+
* - storage format (P5): the generation record says `storage: 'v2'` —
|
|
56
|
+
* snapshots in the v2 encoding, compressed in place after they are
|
|
57
|
+
* stored (`compressLater`), inflated at start (`inflate`); update and
|
|
58
|
+
* pending records stay v1. A container without `storage` (0.1.0-next.22)
|
|
59
|
+
* is all v1, and its next compaction rewrites it.
|
|
54
60
|
* - a Step2 serves the STORED state: the engine's pending structs (waiting
|
|
55
61
|
* for a dependency, never stored) are not served.
|
|
56
62
|
* - a frame larger than `maxFrameBytes` (32 MiB) goes out as chunks the
|
|
@@ -92,8 +98,10 @@ import * as encoding from 'lib0-v14/encoding';
|
|
|
92
98
|
import { Y } from '../crdt/engine.js';
|
|
93
99
|
import * as E from '../crdt/index.js';
|
|
94
100
|
import { READ_ONLY_DENIAL } from '../crdt/protocols/auth.js';
|
|
95
|
-
import { isGenerationRecord } from '../crdt/protocols/envelope.js';
|
|
96
|
-
import {
|
|
101
|
+
import { isGenerationRecord, STORED_GENERATION_RECORD, storageOf } from '../crdt/protocols/envelope.js';
|
|
102
|
+
import { gunzip, isGzip, packed } from '../crdt/storage.js';
|
|
103
|
+
import { DEL_PREFIX, REGISTRY_KEY, WITHDRAW_PREFIX } from '../crdt/schema.js';
|
|
104
|
+
import { ChunkLimitError, ChunkSequenceError, CLOSE } from '../crdt/providers/room.js';
|
|
97
105
|
const crdt = E.bindCrdt(Y);
|
|
98
106
|
const sync = crdt.sync;
|
|
99
107
|
/** Under SQLite-backed Durable Objects' 2 MB row cap, with room for the other columns. */
|
|
@@ -102,6 +110,26 @@ export const DEFAULT_MAX_ROW_BYTES = 2_000_000 - 4096;
|
|
|
102
110
|
export const DEFAULT_COMPACT_AFTER = 500;
|
|
103
111
|
/** ms between the first unsaved change and `onSave`. */
|
|
104
112
|
export const DEFAULT_SAVE_AFTER = 2000;
|
|
113
|
+
/**
|
|
114
|
+
* Room quotas (H3), each refused with `4413` (`quota: <name>`). The
|
|
115
|
+
* document's size: what its records hold (uncompressed) and the updates the
|
|
116
|
+
* engine holds waiting — the live document holds about that, several
|
|
117
|
+
* times over in memory for text (an isolate has 128 MB). Raise or lower
|
|
118
|
+
* it for your documents.
|
|
119
|
+
*/
|
|
120
|
+
export const DEFAULT_MAX_DOCUMENT_BYTES = 64 * 1024 * 1024;
|
|
121
|
+
/** The largest frame a socket may send, reassembled from chunks (the platform caps one message at 32 MiB). */
|
|
122
|
+
export const DEFAULT_MAX_INBOUND_FRAME_BYTES = 64 * 1024 * 1024;
|
|
123
|
+
/**
|
|
124
|
+
* Sync messages a socket may send per second, sustained; a burst of ten
|
|
125
|
+
* seconds' worth is allowed (a reconnect, a paste). Far above typing
|
|
126
|
+
* speed: a script should batch its edits into transactions.
|
|
127
|
+
*/
|
|
128
|
+
export const DEFAULT_MAX_UPDATES_PER_SECOND = 50;
|
|
129
|
+
/** Seconds of the update rate a socket may spend at once. */
|
|
130
|
+
const BURST_SECONDS = 10;
|
|
131
|
+
/** The ceiling of a quota knob (a host may raise the defaults up to it). */
|
|
132
|
+
const QUOTA_CEILING = 2 ** 40;
|
|
105
133
|
/** Longest wait between the save alarms of a room that cannot read its rows. */
|
|
106
134
|
const MAX_SAVE_RETRY = 5 * 60_000;
|
|
107
135
|
/**
|
|
@@ -112,6 +140,8 @@ const MAX_SAVE_RETRY = 5 * 60_000;
|
|
|
112
140
|
export const MAX_WAITING_DELETES = 1024;
|
|
113
141
|
/** Refusals kept in `refusals` (the newest); `refusalCounts` counts every one. */
|
|
114
142
|
export const MAX_REFUSALS = 100;
|
|
143
|
+
/** The header `routeDocumentSocket` sets on an authorized `lastUpdated` probe (H12). */
|
|
144
|
+
export const PROBE_HEADER = 'X-Edytor-Probe';
|
|
115
145
|
/** The transaction origin of the room's own edits (`transact`, `onLoad` seeds). */
|
|
116
146
|
export const ROOM_ORIGIN = Symbol('edytor-room');
|
|
117
147
|
/**
|
|
@@ -125,6 +155,16 @@ export const IDENTITY_HEADERS = {
|
|
|
125
155
|
replica: 'X-Edytor-Replica',
|
|
126
156
|
access: 'X-Edytor-Access'
|
|
127
157
|
};
|
|
158
|
+
const now = () => typeof performance !== 'undefined' && typeof performance.now === 'function'
|
|
159
|
+
? performance.now()
|
|
160
|
+
: Date.now();
|
|
161
|
+
const timing = () => ({ count: 0, totalMs: 0, maxMs: 0, lastMs: 0 });
|
|
162
|
+
const tally = (t, ms) => {
|
|
163
|
+
t.count++;
|
|
164
|
+
t.totalMs += ms;
|
|
165
|
+
t.maxMs = Math.max(t.maxMs, ms);
|
|
166
|
+
t.lastMs = ms;
|
|
167
|
+
};
|
|
128
168
|
/**
|
|
129
169
|
* An upgrade accepted and closed at once, so the client reads `code`: a
|
|
130
170
|
* browser sees an HTTP error at the upgrade only as `1006`, which a
|
|
@@ -145,10 +185,16 @@ class MalformedFrame extends Error {
|
|
|
145
185
|
/** A SQLite fault outside the append path (the replica registry). */
|
|
146
186
|
class StorageFault extends Error {
|
|
147
187
|
}
|
|
148
|
-
/** A stored record missing some of its rows: the container is corrupt, not the read. */
|
|
188
|
+
/** A stored record missing some of its rows, or one that does not inflate: the container is corrupt, not the read. */
|
|
149
189
|
class TornRecord extends Error {
|
|
190
|
+
constructor(detail = 'torn record') {
|
|
191
|
+
super(detail);
|
|
192
|
+
}
|
|
193
|
+
}
|
|
194
|
+
/** A compressed record this instance has not inflated (only `start` can): the next dial reads it again. */
|
|
195
|
+
class CompressedRecord extends Error {
|
|
150
196
|
constructor() {
|
|
151
|
-
super('
|
|
197
|
+
super('compressed record not inflated');
|
|
152
198
|
}
|
|
153
199
|
}
|
|
154
200
|
/** Decode the client's bytes: a throw is theirs (`malformed`), not the room's. */
|
|
@@ -417,6 +463,28 @@ const unheldDeletes = ({ structs, ds }, stripped, doc) => {
|
|
|
417
463
|
}
|
|
418
464
|
return unheld;
|
|
419
465
|
};
|
|
466
|
+
/** The content `ds` deletes of what `doc` holds live (an item's length: characters, atoms). */
|
|
467
|
+
const freedBy = (doc, ds) => {
|
|
468
|
+
let freed = 0;
|
|
469
|
+
for (const [client, ranges] of ds.clients) {
|
|
470
|
+
const structs = doc.store.clients.get(client) ?? [];
|
|
471
|
+
const stored = heldClock(doc, client);
|
|
472
|
+
for (const { clock, len } of ranges.getIds()) {
|
|
473
|
+
if (clock >= stored)
|
|
474
|
+
continue;
|
|
475
|
+
for (let i = Y.findIndexSS(structs, clock); i < structs.length; i++) {
|
|
476
|
+
const struct = structs[i];
|
|
477
|
+
if (struct.id.clock >= clock + len)
|
|
478
|
+
break;
|
|
479
|
+
if (struct.deleted || struct instanceof Y.Skip)
|
|
480
|
+
continue;
|
|
481
|
+
freed +=
|
|
482
|
+
Math.min(clock + len, struct.id.clock + struct.length) - Math.max(clock, struct.id.clock);
|
|
483
|
+
}
|
|
484
|
+
}
|
|
485
|
+
}
|
|
486
|
+
return freed;
|
|
487
|
+
};
|
|
420
488
|
/** How many ranges `ids` holds. */
|
|
421
489
|
const rangeCount = (ids) => {
|
|
422
490
|
let count = 0;
|
|
@@ -449,6 +517,27 @@ const addIds = (into, from, keep = () => true) => {
|
|
|
449
517
|
}
|
|
450
518
|
return into;
|
|
451
519
|
};
|
|
520
|
+
/** Rows grouped into their records, in write order; a record missing rows throws. */
|
|
521
|
+
const reassemble = (rows) => {
|
|
522
|
+
const byRecord = new Map();
|
|
523
|
+
for (const row of rows) {
|
|
524
|
+
const parts = byRecord.get(row.record) ?? [];
|
|
525
|
+
parts.push(row);
|
|
526
|
+
byRecord.set(row.record, parts);
|
|
527
|
+
}
|
|
528
|
+
return [...byRecord.values()].map((parts) => {
|
|
529
|
+
if (parts.length !== parts[0].parts)
|
|
530
|
+
throw new TornRecord();
|
|
531
|
+
parts.sort((a, b) => a.part - b.part);
|
|
532
|
+
const bytes = new Uint8Array(parts.reduce((n, p) => n + p.bytes.byteLength, 0));
|
|
533
|
+
let at = 0;
|
|
534
|
+
for (const part of parts) {
|
|
535
|
+
bytes.set(new Uint8Array(part.bytes), at);
|
|
536
|
+
at += part.bytes.byteLength;
|
|
537
|
+
}
|
|
538
|
+
return { kind: parts[0].kind, record: parts[0].record, bytes };
|
|
539
|
+
});
|
|
540
|
+
};
|
|
452
541
|
/** The deletes of the `pending` records among `records`. */
|
|
453
542
|
const storedPending = (records) => {
|
|
454
543
|
const pending = records.filter((record) => record.kind === 'pending');
|
|
@@ -504,19 +593,195 @@ const heldIds = (doc, ds, deleted = false) => {
|
|
|
504
593
|
* that replaces it).
|
|
505
594
|
*/
|
|
506
595
|
const heldDeletes = (doc, ds) => heldIds(doc, ds, true);
|
|
507
|
-
/**
|
|
508
|
-
|
|
596
|
+
/**
|
|
597
|
+
* `read` with the engine's pending store set aside: what waits for a
|
|
598
|
+
* dependency (structs, deletes of items it lacks) is never part of the
|
|
599
|
+
* stored state — waiting deletes are stored apart, as `pending` records.
|
|
600
|
+
*/
|
|
601
|
+
const withoutPending = (doc, read) => {
|
|
509
602
|
const { store } = doc;
|
|
510
603
|
const held = [store.pendingStructs, store.pendingDs];
|
|
511
604
|
store.pendingStructs = null;
|
|
512
605
|
store.pendingDs = null;
|
|
513
606
|
try {
|
|
514
|
-
return
|
|
607
|
+
return read();
|
|
515
608
|
}
|
|
516
609
|
finally {
|
|
517
610
|
[store.pendingStructs, store.pendingDs] = held;
|
|
518
611
|
}
|
|
519
612
|
};
|
|
613
|
+
/** A Step2 of what the room STORED: the engine's pending store (never stored) is left out. */
|
|
614
|
+
const storedStep2 = (doc, sv) => withoutPending(doc, () => E.frame(E.messageSync, (e) => sync.writeSyncStep2(e, doc, sv)));
|
|
615
|
+
/**
|
|
616
|
+
* The live document as one update (P2): its healed state — the engine
|
|
617
|
+
* merged what each keystroke wrote and collected deleted content, sparing
|
|
618
|
+
* the text a replica may copy again (`keepCopies`, P11) —, without what
|
|
619
|
+
* waits (`withoutPending`). Memory never runs ahead of storage, so it
|
|
620
|
+
* holds exactly what the stored records hold, collected.
|
|
621
|
+
*/
|
|
622
|
+
const liveState = (doc) => withoutPending(doc, () => Y.encodeStateAsUpdateV2(doc));
|
|
623
|
+
/** A fresh room document: it keeps what an editing replica keeps (P11). */
|
|
624
|
+
const roomDoc = () => {
|
|
625
|
+
const doc = crdt.createDoc();
|
|
626
|
+
crdt.doc.keepCopies(doc);
|
|
627
|
+
return doc;
|
|
628
|
+
};
|
|
629
|
+
/** Admit stored or loaded updates into a room document (`roomDoc`'s collection rules). */
|
|
630
|
+
const admit = (updates, name) => crdt.admission.admitUpdate(updates, name, {
|
|
631
|
+
prepare: (doc) => crdt.doc.keepCopies(doc)
|
|
632
|
+
});
|
|
633
|
+
// ── Per-writer marks (H2) ──────────────────────────────────────────
|
|
634
|
+
/** The writer `n` of a per-writer block mark key (`del.<n>`, `wd.<n>`), or `null`. */
|
|
635
|
+
const markWriter = (key) => {
|
|
636
|
+
if (typeof key !== 'string')
|
|
637
|
+
return null;
|
|
638
|
+
const prefix = key.startsWith(DEL_PREFIX)
|
|
639
|
+
? DEL_PREFIX
|
|
640
|
+
: key.startsWith(WITHDRAW_PREFIX)
|
|
641
|
+
? WITHDRAW_PREFIX
|
|
642
|
+
: null;
|
|
643
|
+
const digits = prefix === null ? '' : key.slice(prefix.length);
|
|
644
|
+
return /^\d{1,16}$/.test(digits) ? Number(digits) : null;
|
|
645
|
+
};
|
|
646
|
+
/** A frame's structs by client, each run in clock order. */
|
|
647
|
+
const runsOf = (structs) => {
|
|
648
|
+
const runs = new Map();
|
|
649
|
+
for (const struct of structs) {
|
|
650
|
+
const run = runs.get(struct.id.client) ?? [];
|
|
651
|
+
run.push(struct);
|
|
652
|
+
runs.set(struct.id.client, run);
|
|
653
|
+
}
|
|
654
|
+
return runs;
|
|
655
|
+
};
|
|
656
|
+
const isId = (value) => typeof value === 'object' &&
|
|
657
|
+
value !== null &&
|
|
658
|
+
typeof value.client === 'number' &&
|
|
659
|
+
typeof value.clock === 'number' &&
|
|
660
|
+
!('_item' in value);
|
|
661
|
+
/**
|
|
662
|
+
* Where the item at `id` sits, held by the room or carried by the frame
|
|
663
|
+
* (`frame`): an encoded map entry with a left origin names neither key nor
|
|
664
|
+
* parent, so it is followed origin to origin to the one that does.
|
|
665
|
+
*/
|
|
666
|
+
const placeOf = (doc, frame, id) => {
|
|
667
|
+
const seen = new Set();
|
|
668
|
+
for (let at = id; at !== null;) {
|
|
669
|
+
const held = storedStruct(doc, at.client, at.clock);
|
|
670
|
+
if (held)
|
|
671
|
+
return held instanceof Y.Item ? { key: held.parentSub, parent: held.parent } : null;
|
|
672
|
+
const struct = structAt(frame.get(at.client) ?? [], at.clock);
|
|
673
|
+
if (!(struct instanceof Y.Item) || seen.has(struct))
|
|
674
|
+
return null;
|
|
675
|
+
seen.add(struct);
|
|
676
|
+
if (struct.parent !== null || struct.parentSub !== null) {
|
|
677
|
+
return { key: struct.parentSub, parent: struct.parent };
|
|
678
|
+
}
|
|
679
|
+
if (struct.rightOrigin !== null || struct.origin === null)
|
|
680
|
+
return null;
|
|
681
|
+
at = struct.origin;
|
|
682
|
+
}
|
|
683
|
+
return null;
|
|
684
|
+
};
|
|
685
|
+
/** Is `parent` (a {@link Place}'s) a block node: an entry of the block registry? */
|
|
686
|
+
const isBlockNode = (doc, frame, parent) => {
|
|
687
|
+
const registry = doc.share.get(REGISTRY_KEY);
|
|
688
|
+
if (isId(parent)) {
|
|
689
|
+
const node = placeOf(doc, frame, parent);
|
|
690
|
+
return (node !== null &&
|
|
691
|
+
node.key !== null &&
|
|
692
|
+
(node.parent === registry || node.parent === REGISTRY_KEY));
|
|
693
|
+
}
|
|
694
|
+
const item = parent?._item;
|
|
695
|
+
return item?.parent === registry && registry !== undefined;
|
|
696
|
+
};
|
|
697
|
+
/**
|
|
698
|
+
* The clients of a frame (but `skip`) that write a per-writer block mark
|
|
699
|
+
* of another writer: a struct whose key is `del.<n>` or `wd.<n>` on a
|
|
700
|
+
* block node, from a client other than `n` (H2). Only `n` writes its mark.
|
|
701
|
+
*/
|
|
702
|
+
const forgedWriters = (doc, structs, skip) => {
|
|
703
|
+
const frame = runsOf(structs);
|
|
704
|
+
const forged = new Set();
|
|
705
|
+
for (const struct of structs) {
|
|
706
|
+
const { client, clock } = struct.id;
|
|
707
|
+
if (!(struct instanceof Y.Item) || skip.has(client) || forged.has(client))
|
|
708
|
+
continue;
|
|
709
|
+
if (storedStruct(doc, client, clock) !== null)
|
|
710
|
+
continue;
|
|
711
|
+
const place = placeOf(doc, frame, struct.id);
|
|
712
|
+
const writer = markWriter(place?.key);
|
|
713
|
+
if (writer !== null && writer !== client && isBlockNode(doc, frame, place.parent)) {
|
|
714
|
+
forged.add(client);
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
return forged;
|
|
718
|
+
};
|
|
719
|
+
/**
|
|
720
|
+
* Of a frame's deletes, those of live per-writer block marks whose writer
|
|
721
|
+
* the sender may not delete for (`mayDelete`): only `n`'s replicas delete
|
|
722
|
+
* `del.<n>`/`wd.<n>` (H2). A mark deleted with its block node (deleted, or
|
|
723
|
+
* deleted by the same frame) is not one.
|
|
724
|
+
*/
|
|
725
|
+
const forgedDeletes = (doc, ds, mayDelete) => {
|
|
726
|
+
const forged = Y.createIdSet();
|
|
727
|
+
const registry = doc.share.get(REGISTRY_KEY);
|
|
728
|
+
if (registry === undefined)
|
|
729
|
+
return forged;
|
|
730
|
+
for (const [client, ranges] of ds.clients) {
|
|
731
|
+
const structs = doc.store.clients.get(client) ?? [];
|
|
732
|
+
const held = heldClock(doc, client);
|
|
733
|
+
for (const { clock, len } of ranges.getIds()) {
|
|
734
|
+
if (clock >= held)
|
|
735
|
+
continue;
|
|
736
|
+
for (let i = Y.findIndexSS(structs, clock); i < structs.length; i++) {
|
|
737
|
+
const struct = structs[i];
|
|
738
|
+
if (struct.id.clock >= clock + len)
|
|
739
|
+
break;
|
|
740
|
+
if (!(struct instanceof Y.Item) || struct.deleted)
|
|
741
|
+
continue;
|
|
742
|
+
const writer = markWriter(struct.parentSub);
|
|
743
|
+
if (writer === null || mayDelete(writer))
|
|
744
|
+
continue;
|
|
745
|
+
const node = struct.parent?._item;
|
|
746
|
+
if (!node || node.parent !== registry)
|
|
747
|
+
continue;
|
|
748
|
+
if (node.deleted || ds.has(node.id.client, node.id.clock))
|
|
749
|
+
continue;
|
|
750
|
+
const from = Math.max(clock, struct.id.clock);
|
|
751
|
+
forged.add(client, from, Math.min(clock + len, struct.id.clock + struct.length) - from);
|
|
752
|
+
}
|
|
753
|
+
}
|
|
754
|
+
}
|
|
755
|
+
return forged;
|
|
756
|
+
};
|
|
757
|
+
/** The ids of `next` that left their order relative to the others both lists hold (a longest increasing run kept). */
|
|
758
|
+
const reordered = (previous, next) => {
|
|
759
|
+
const at = new Map(previous.map((id, i) => [id, i]));
|
|
760
|
+
const common = next.filter((id) => at.has(id));
|
|
761
|
+
// Longest increasing run of previous positions (patience sorting).
|
|
762
|
+
const tails = [];
|
|
763
|
+
const links = new Array(common.length).fill(-1);
|
|
764
|
+
const tailAt = [];
|
|
765
|
+
common.forEach((id, i) => {
|
|
766
|
+
const position = at.get(id);
|
|
767
|
+
let lo = 0;
|
|
768
|
+
let hi = tails.length;
|
|
769
|
+
while (lo < hi) {
|
|
770
|
+
const mid = (lo + hi) >> 1;
|
|
771
|
+
if (tails[mid] < position)
|
|
772
|
+
lo = mid + 1;
|
|
773
|
+
else
|
|
774
|
+
hi = mid;
|
|
775
|
+
}
|
|
776
|
+
tails[lo] = position;
|
|
777
|
+
tailAt[lo] = i;
|
|
778
|
+
links[i] = lo > 0 ? tailAt[lo - 1] : -1;
|
|
779
|
+
});
|
|
780
|
+
const kept = new Set();
|
|
781
|
+
for (let i = tails.length ? tailAt[tails.length - 1] : -1; i !== -1; i = links[i])
|
|
782
|
+
kept.add(i);
|
|
783
|
+
return common.filter((_, i) => !kept.has(i));
|
|
784
|
+
};
|
|
520
785
|
/** The tag of the document's sockets: other sockets of the object are left to you. */
|
|
521
786
|
export const SOCKET_TAG = 'edytor';
|
|
522
787
|
/** The 1011 close of a storage fault: the provider redials and resends. */
|
|
@@ -537,6 +802,9 @@ export class AttachedDocument {
|
|
|
537
802
|
maxFrameBytes;
|
|
538
803
|
compactAfter;
|
|
539
804
|
saveAfter;
|
|
805
|
+
maxDocumentBytes;
|
|
806
|
+
maxInboundFrameBytes;
|
|
807
|
+
maxUpdatesPerSecond;
|
|
540
808
|
live = null;
|
|
541
809
|
/** Why the stored container was refused (another generation, a torn record, `onLoad`…). */
|
|
542
810
|
failure = null;
|
|
@@ -550,6 +818,21 @@ export class AttachedDocument {
|
|
|
550
818
|
origin = { kind: 'fresh' };
|
|
551
819
|
nextRecord = 0;
|
|
552
820
|
updates = 0;
|
|
821
|
+
/** What the records hold, uncompressed (the generation record aside): the document quota's measure. */
|
|
822
|
+
documentBytes = 0;
|
|
823
|
+
/** The counters `metrics()` reads (H14). */
|
|
824
|
+
counters = {
|
|
825
|
+
compaction: { ...timing(), lastBytes: 0 },
|
|
826
|
+
fold: timing(),
|
|
827
|
+
fanOut: { messages: 0, bytes: 0 },
|
|
828
|
+
since: Date.now()
|
|
829
|
+
};
|
|
830
|
+
/** The index `validate` reads (built at its first frame, for the live document). */
|
|
831
|
+
validation = null;
|
|
832
|
+
/** Each socket's update allowance (a token bucket; memory: a wake refills it). */
|
|
833
|
+
allowances = new WeakMap();
|
|
834
|
+
/** Each socket's chunked frame in flight (memory: a wake loses it, and the socket is faulted). */
|
|
835
|
+
chunkReaders = new WeakMap();
|
|
553
836
|
/** A failed append: the live doc holds what storage does not, until it is rebuilt. */
|
|
554
837
|
unstored = null;
|
|
555
838
|
/** Inside `transact`, its commit included (a nested call is refused but while `fn` runs). */
|
|
@@ -558,6 +841,12 @@ export class AttachedDocument {
|
|
|
558
841
|
running = false;
|
|
559
842
|
/** Inside a client frame's apply, its events included: the frame's handler settles it. */
|
|
560
843
|
handling = false;
|
|
844
|
+
/** How the container stores its snapshots (its generation record says; a fresh one: this build's). */
|
|
845
|
+
format = 'v2';
|
|
846
|
+
/** The snapshot compression in flight (`compressed()`). */
|
|
847
|
+
compressing = Promise.resolve();
|
|
848
|
+
/** The raw bytes of the compressed snapshot record (`records` reads it; only `start` can inflate). */
|
|
849
|
+
inflated = null;
|
|
561
850
|
/** The waiting deletes stored as `pending` records (the engine may hold more, in memory). */
|
|
562
851
|
storedWaiting = Y.createIdSet();
|
|
563
852
|
/** The failure is the storage's (a failed read, `onLoad`'s store down): the next dial starts again. */
|
|
@@ -570,6 +859,7 @@ export class AttachedDocument {
|
|
|
570
859
|
options;
|
|
571
860
|
rowsTable;
|
|
572
861
|
replicasTable;
|
|
862
|
+
metaTable;
|
|
573
863
|
_lookups = null;
|
|
574
864
|
constructor(ctx, options = {}) {
|
|
575
865
|
this.ctx = ctx;
|
|
@@ -579,11 +869,15 @@ export class AttachedDocument {
|
|
|
579
869
|
this.maxFrameBytes = knob(options.maxFrameBytes, E.MAX_FRAME_BYTES);
|
|
580
870
|
this.compactAfter = knob(options.compactAfter, DEFAULT_COMPACT_AFTER, 1e9);
|
|
581
871
|
this.saveAfter = knob(options.saveAfter, DEFAULT_SAVE_AFTER, 1e9);
|
|
872
|
+
this.maxDocumentBytes = knob(options.maxDocumentBytes, DEFAULT_MAX_DOCUMENT_BYTES, QUOTA_CEILING);
|
|
873
|
+
this.maxInboundFrameBytes = knob(options.maxInboundFrameBytes, DEFAULT_MAX_INBOUND_FRAME_BYTES, QUOTA_CEILING);
|
|
874
|
+
this.maxUpdatesPerSecond = knob(options.maxUpdatesPerSecond, DEFAULT_MAX_UPDATES_PER_SECOND, QUOTA_CEILING);
|
|
582
875
|
const prefix = options.tablePrefix ?? 'edytor_';
|
|
583
876
|
if (!/^\w*$/.test(prefix))
|
|
584
877
|
throw new Error(`invalid table prefix ${prefix}`);
|
|
585
878
|
this.rowsTable = `${prefix}rows`;
|
|
586
879
|
this.replicasTable = `${prefix}replicas`;
|
|
880
|
+
this.metaTable = `${prefix}meta`;
|
|
587
881
|
// The provider pings a silent socket: answer without waking the object.
|
|
588
882
|
if (ctx.setWebSocketAutoResponse && !ctx.getWebSocketAutoResponse?.()) {
|
|
589
883
|
ctx.setWebSocketAutoResponse(new WebSocketRequestResponsePair(PING, PONG));
|
|
@@ -599,12 +893,40 @@ export class AttachedDocument {
|
|
|
599
893
|
// A wake keeps a pending save: re-arming it would push `onSave` back at every wake.
|
|
600
894
|
if (this.options.onSave)
|
|
601
895
|
this.saveScheduled = (await this.ctx.storage.getAlarm()) !== null;
|
|
896
|
+
await this.inflate();
|
|
602
897
|
noTimers(() => this.load());
|
|
603
898
|
if (this.origin.kind === 'fresh' && this.live !== null)
|
|
604
899
|
await this.seed();
|
|
605
900
|
}
|
|
606
901
|
catch (error) {
|
|
607
|
-
this.fail(error,
|
|
902
|
+
this.fail(error, !(error instanceof TornRecord));
|
|
903
|
+
}
|
|
904
|
+
}
|
|
905
|
+
/**
|
|
906
|
+
* Inflate a compressed snapshot record (P5) for `records`, which reads
|
|
907
|
+
* synchronously (a rebuild does): decompression is asynchronous, so
|
|
908
|
+
* only `start` can. One that does not inflate is a corrupt container.
|
|
909
|
+
*/
|
|
910
|
+
async inflate() {
|
|
911
|
+
this.inflated = null;
|
|
912
|
+
let rows;
|
|
913
|
+
try {
|
|
914
|
+
rows = this.sql
|
|
915
|
+
.exec(`SELECT kind, record, part, parts, bytes FROM ${this.rowsTable} WHERE kind = 'snapshot' ORDER BY seq`)
|
|
916
|
+
.toArray();
|
|
917
|
+
}
|
|
918
|
+
catch {
|
|
919
|
+
return; // no table yet, or a failed read: `load` reports it
|
|
920
|
+
}
|
|
921
|
+
for (const { record, bytes } of reassemble(rows)) {
|
|
922
|
+
if (!isGzip(bytes))
|
|
923
|
+
continue;
|
|
924
|
+
try {
|
|
925
|
+
this.inflated = { record, bytes: (await gunzip(bytes)) };
|
|
926
|
+
}
|
|
927
|
+
catch (error) {
|
|
928
|
+
throw new TornRecord(`snapshot does not inflate: ${String(error)}`);
|
|
929
|
+
}
|
|
608
930
|
}
|
|
609
931
|
}
|
|
610
932
|
/** A read of the rows or `onLoad` failed (storage was down): start again. */
|
|
@@ -777,6 +1099,7 @@ export class AttachedDocument {
|
|
|
777
1099
|
this.ctx.storage.transactionSync(() => {
|
|
778
1100
|
this.sql.exec(`DELETE FROM ${this.rowsTable}`);
|
|
779
1101
|
this.sql.exec(`DELETE FROM ${this.replicasTable}`);
|
|
1102
|
+
this.sql.exec(`DELETE FROM ${this.metaTable}`);
|
|
780
1103
|
});
|
|
781
1104
|
this.presence.clear();
|
|
782
1105
|
await this.start();
|
|
@@ -816,20 +1139,19 @@ export class AttachedDocument {
|
|
|
816
1139
|
scratch.destroy();
|
|
817
1140
|
return update;
|
|
818
1141
|
});
|
|
819
|
-
doc =
|
|
1142
|
+
doc = admit(loaded.update, `room ${this.ctx.id} onLoad`);
|
|
820
1143
|
replicas = loaded.replicas;
|
|
821
1144
|
}
|
|
822
1145
|
catch (error) {
|
|
823
1146
|
return this.fail(error, false);
|
|
824
1147
|
}
|
|
825
1148
|
const waiting = pendingDeletes(doc);
|
|
826
|
-
const
|
|
827
|
-
|
|
828
|
-
const snapshot = Y.encodeStateAsUpdate(doc);
|
|
829
|
-
doc.store.pendingDs = pendingDs;
|
|
1149
|
+
const snapshot = liveState(doc);
|
|
1150
|
+
let record = -1;
|
|
830
1151
|
try {
|
|
831
1152
|
this.ctx.storage.transactionSync(() => {
|
|
832
|
-
this.insert('snapshot', snapshot);
|
|
1153
|
+
record = this.insert('snapshot', snapshot);
|
|
1154
|
+
this.touch();
|
|
833
1155
|
if (!waiting.isEmpty())
|
|
834
1156
|
this.insert('pending', deletesUpdate(waiting));
|
|
835
1157
|
// Without a registry, every id with content is left claimable (user
|
|
@@ -845,8 +1167,10 @@ export class AttachedDocument {
|
|
|
845
1167
|
return this.fail(error, true);
|
|
846
1168
|
}
|
|
847
1169
|
this.live?.destroy();
|
|
1170
|
+
this.format = STORED_GENERATION_RECORD.storage;
|
|
848
1171
|
this.adopt(doc);
|
|
849
1172
|
this.storedWaiting = waiting;
|
|
1173
|
+
this.compressLater(record, snapshot);
|
|
850
1174
|
});
|
|
851
1175
|
}
|
|
852
1176
|
// ── Storage ──────────────────────────────────────────────────────────
|
|
@@ -871,6 +1195,7 @@ export class AttachedDocument {
|
|
|
871
1195
|
bytes BLOB NOT NULL
|
|
872
1196
|
)`);
|
|
873
1197
|
this.sql.exec(`CREATE TABLE IF NOT EXISTS ${this.replicasTable} (replica INTEGER PRIMARY KEY, user TEXT NOT NULL)`);
|
|
1198
|
+
this.sql.exec(`CREATE TABLE IF NOT EXISTS ${this.metaTable} (key TEXT PRIMARY KEY, value INTEGER NOT NULL)`);
|
|
874
1199
|
this.nextRecord = 0;
|
|
875
1200
|
records = this.records();
|
|
876
1201
|
}
|
|
@@ -881,8 +1206,10 @@ export class AttachedDocument {
|
|
|
881
1206
|
if (records.length === 0) {
|
|
882
1207
|
// A fresh room: the generation record is written with the first stored record.
|
|
883
1208
|
this.origin = { kind: 'fresh' };
|
|
1209
|
+
this.documentBytes = 0;
|
|
1210
|
+
this.format = STORED_GENERATION_RECORD.storage;
|
|
884
1211
|
this.storedWaiting = Y.createIdSet();
|
|
885
|
-
this.adopt(
|
|
1212
|
+
this.adopt(roomDoc());
|
|
886
1213
|
return;
|
|
887
1214
|
}
|
|
888
1215
|
const [generation, ...rest] = records;
|
|
@@ -890,8 +1217,11 @@ export class AttachedDocument {
|
|
|
890
1217
|
if (generation.kind !== 'generation' || !isGenerationRecord(found)) {
|
|
891
1218
|
throw new E.GenerationMismatchError(`room ${this.ctx.id}`, found);
|
|
892
1219
|
}
|
|
893
|
-
|
|
894
|
-
|
|
1220
|
+
// Applied in one transaction, never merged first: a merge of the
|
|
1221
|
+
// records keeps every keystroke's struct and the deleted content.
|
|
1222
|
+
const doc = admit(rest.map((record) => (record.v2 ? { v2: record.bytes } : record.bytes)), `room ${this.ctx.id}`);
|
|
1223
|
+
this.format = storageOf(found);
|
|
1224
|
+
this.documentBytes = rest.reduce((n, record) => n + record.bytes.length, 0);
|
|
895
1225
|
this.adopt(doc);
|
|
896
1226
|
// Every delete the engine holds waiting came from the rows.
|
|
897
1227
|
this.storedWaiting = pendingDeletes(doc);
|
|
@@ -902,26 +1232,43 @@ export class AttachedDocument {
|
|
|
902
1232
|
this.fail(error, false);
|
|
903
1233
|
}
|
|
904
1234
|
}
|
|
905
|
-
/**
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
1235
|
+
/** The bytes of the stored `pending` records (uncompressed, as every record but a snapshot). */
|
|
1236
|
+
pendingBytes() {
|
|
1237
|
+
return (this.sql
|
|
1238
|
+
.exec(`SELECT SUM(length(bytes)) AS n FROM ${this.rowsTable} WHERE kind = 'pending'`)
|
|
1239
|
+
.one().n ?? 0);
|
|
1240
|
+
}
|
|
1241
|
+
/**
|
|
1242
|
+
* Reassembled logical records in write order (of one kind, with
|
|
1243
|
+
* `only`); a torn record throws. A v2 container's snapshot is v2
|
|
1244
|
+
* (`v2`), inflated when it is stored compressed (P5).
|
|
1245
|
+
*/
|
|
1246
|
+
records(only) {
|
|
1247
|
+
const rows = this.sql
|
|
1248
|
+
.exec(`SELECT kind, record, part, parts, bytes FROM ${this.rowsTable}${only ? ' WHERE kind = ?' : ''} ORDER BY seq`, ...(only ? [only] : []))
|
|
1249
|
+
.toArray();
|
|
1250
|
+
for (const row of rows)
|
|
912
1251
|
this.nextRecord = Math.max(this.nextRecord, row.record + 1);
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
1252
|
+
const records = reassemble(rows);
|
|
1253
|
+
// The container's storage format, from its generation record.
|
|
1254
|
+
let format = 'v1';
|
|
1255
|
+
if (records[0]?.kind === 'generation') {
|
|
1256
|
+
try {
|
|
1257
|
+
const found = JSON.parse(new TextDecoder().decode(records[0].bytes));
|
|
1258
|
+
if (isGenerationRecord(found))
|
|
1259
|
+
format = storageOf(found);
|
|
1260
|
+
}
|
|
1261
|
+
catch {
|
|
1262
|
+
// `load` refuses it
|
|
923
1263
|
}
|
|
924
|
-
|
|
1264
|
+
}
|
|
1265
|
+
return records.map((record) => {
|
|
1266
|
+
const v2 = format === 'v2' && record.kind === 'snapshot';
|
|
1267
|
+
if (!v2 || !isGzip(record.bytes))
|
|
1268
|
+
return { ...record, v2 };
|
|
1269
|
+
if (this.inflated?.record !== record.record)
|
|
1270
|
+
throw new CompressedRecord();
|
|
1271
|
+
return { ...record, bytes: this.inflated.bytes, v2 };
|
|
925
1272
|
});
|
|
926
1273
|
}
|
|
927
1274
|
/**
|
|
@@ -931,17 +1278,70 @@ export class AttachedDocument {
|
|
|
931
1278
|
*/
|
|
932
1279
|
insert(kind, bytes) {
|
|
933
1280
|
if (this.nextRecord === 0 && kind !== 'generation') {
|
|
934
|
-
this.insert('generation', encodeJSON(
|
|
1281
|
+
this.insert('generation', encodeJSON(STORED_GENERATION_RECORD));
|
|
935
1282
|
}
|
|
936
1283
|
const record = this.nextRecord++;
|
|
1284
|
+
this.writeRecord(kind, record, bytes);
|
|
1285
|
+
if (kind !== 'generation')
|
|
1286
|
+
this.documentBytes += bytes.length;
|
|
1287
|
+
return record;
|
|
1288
|
+
}
|
|
1289
|
+
/** Record `record`'s rows (at the row positions `seqs`, when given). */
|
|
1290
|
+
writeRecord(kind, record, bytes, seqs) {
|
|
937
1291
|
const parts = Math.max(1, Math.ceil(bytes.length / this.maxRowBytes));
|
|
938
1292
|
for (let part = 0; part < parts; part++) {
|
|
939
|
-
|
|
1293
|
+
const row = bytes.slice(part * this.maxRowBytes, (part + 1) * this.maxRowBytes);
|
|
1294
|
+
if (seqs) {
|
|
1295
|
+
this.sql.exec(`INSERT INTO ${this.rowsTable} (seq, kind, record, part, parts, bytes) VALUES (?, ?, ?, ?, ?, ?)`, seqs[part], kind, record, part, parts, row);
|
|
1296
|
+
}
|
|
1297
|
+
else {
|
|
1298
|
+
this.sql.exec(`INSERT INTO ${this.rowsTable} (kind, record, part, parts, bytes) VALUES (?, ?, ?, ?, ?)`, kind, record, part, parts, row);
|
|
1299
|
+
}
|
|
940
1300
|
}
|
|
941
1301
|
}
|
|
1302
|
+
/**
|
|
1303
|
+
* Compress snapshot `record` in place once it is stored (P5): gzip is
|
|
1304
|
+
* asynchronous, and the store-before-ack path is not. Its rows are
|
|
1305
|
+
* rewritten (at their positions) only while it is still the stored
|
|
1306
|
+
* snapshot and only when that shrinks it; the raw bytes stay in memory
|
|
1307
|
+
* (`inflated`) for a rebuild's synchronous read. A platform without
|
|
1308
|
+
* `CompressionStream` keeps it raw.
|
|
1309
|
+
*/
|
|
1310
|
+
compressLater(record, raw) {
|
|
1311
|
+
const job = packed(raw)
|
|
1312
|
+
.then((bytes) => noTimers(() => {
|
|
1313
|
+
if (bytes === raw)
|
|
1314
|
+
return;
|
|
1315
|
+
const rows = this.sql
|
|
1316
|
+
.exec(`SELECT seq FROM ${this.rowsTable} WHERE record = ? AND kind = 'snapshot' ORDER BY part`, record)
|
|
1317
|
+
.toArray();
|
|
1318
|
+
// Replaced since (a compaction, a reset): nothing to do.
|
|
1319
|
+
if (rows.length !== Math.max(1, Math.ceil(raw.length / this.maxRowBytes)))
|
|
1320
|
+
return;
|
|
1321
|
+
this.ctx.storage.transactionSync(() => {
|
|
1322
|
+
this.sql.exec(`DELETE FROM ${this.rowsTable} WHERE record = ?`, record);
|
|
1323
|
+
this.writeRecord('snapshot', record, bytes, rows.map((row) => row.seq));
|
|
1324
|
+
});
|
|
1325
|
+
this.inflated = { record, bytes: raw.slice() };
|
|
1326
|
+
}))
|
|
1327
|
+
.catch((error) => this.note({ reason: 'storage', detail: `compression: ${String(error)}` }));
|
|
1328
|
+
this.compressing = job;
|
|
1329
|
+
this.ctx.waitUntil?.(job);
|
|
1330
|
+
}
|
|
1331
|
+
/**
|
|
1332
|
+
* Resolves once the snapshot compression in flight, if any, is stored
|
|
1333
|
+
* (P5): a compaction stores its snapshot raw, then compresses it in
|
|
1334
|
+
* place. Wait for it before reading the rows' sizes.
|
|
1335
|
+
*/
|
|
1336
|
+
async compressed() {
|
|
1337
|
+
await this.compressing;
|
|
1338
|
+
}
|
|
942
1339
|
/**
|
|
943
1340
|
* Compaction: the rows become the generation record + one chunked
|
|
944
|
-
* snapshot
|
|
1341
|
+
* snapshot of the live document's state (P2: the healed state, deleted
|
|
1342
|
+
* content collected but the text a replica may copy again, P11 —
|
|
1343
|
+
* never a merge of the records, which keeps every keystroke's struct
|
|
1344
|
+
* and every deleted character), atomically, + one
|
|
945
1345
|
* `pending` record of the stored deletes still waiting. A waiting delete
|
|
946
1346
|
* of a client id no user registered and no socket holds is reclaimed,
|
|
947
1347
|
* in memory too: nothing can deliver its item (a made-up id) — the
|
|
@@ -955,8 +1355,12 @@ export class AttachedDocument {
|
|
|
955
1355
|
return noTimers(() => {
|
|
956
1356
|
this.heal();
|
|
957
1357
|
const doc = this.requireDoc();
|
|
958
|
-
const
|
|
959
|
-
|
|
1358
|
+
const started = now();
|
|
1359
|
+
// Only the waiting deletes are read back: the snapshot is the live doc.
|
|
1360
|
+
const pending = this.records('pending');
|
|
1361
|
+
// The live state, never a merge of the records (P2): memory never
|
|
1362
|
+
// runs ahead of storage, so it is what they hold, collected.
|
|
1363
|
+
const snapshot = liveState(doc);
|
|
960
1364
|
const sockets = new Set();
|
|
961
1365
|
for (const ws of this.ctx.getWebSockets(SOCKET_TAG)) {
|
|
962
1366
|
const replica = ws.deserializeAttachment()?.replica;
|
|
@@ -971,32 +1375,118 @@ export class AttachedDocument {
|
|
|
971
1375
|
// Waiting deletes stay apart (a discarded forgery drops them,
|
|
972
1376
|
// `settleDeletes`): the stored ones the engine still holds waiting.
|
|
973
1377
|
const all = pendingDeletes(doc);
|
|
974
|
-
const stored = storedPending(
|
|
1378
|
+
const stored = storedPending(pending);
|
|
975
1379
|
const waiting = Y.diffIdSet(stored, Y.diffIdSet(stored, all));
|
|
976
1380
|
const still = addIds(Y.createIdSet(), waiting, (client) => known.has(client));
|
|
977
1381
|
// Memory never runs ahead of storage: the live state vector is the stored one.
|
|
978
1382
|
const kept = new Set([...stateVector(doc).keys(), ...sockets, ...still.clients.keys()]);
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
this.
|
|
983
|
-
|
|
984
|
-
this.
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
1383
|
+
let record = -1;
|
|
1384
|
+
const measured = this.documentBytes;
|
|
1385
|
+
try {
|
|
1386
|
+
this.ctx.storage.transactionSync(() => {
|
|
1387
|
+
this.sql.exec(`DELETE FROM ${this.rowsTable}`);
|
|
1388
|
+
this.documentBytes = 0;
|
|
1389
|
+
// The container is now this build's format (a v1 one migrates here).
|
|
1390
|
+
this.insert('generation', encodeJSON(STORED_GENERATION_RECORD));
|
|
1391
|
+
record = this.insert('snapshot', snapshot);
|
|
1392
|
+
if (!still.isEmpty())
|
|
1393
|
+
this.insert('pending', deletesUpdate(still));
|
|
1394
|
+
for (const replica of registered) {
|
|
1395
|
+
if (!kept.has(replica)) {
|
|
1396
|
+
this.sql.exec(`DELETE FROM ${this.replicasTable} WHERE replica = ?`, replica);
|
|
1397
|
+
}
|
|
988
1398
|
}
|
|
989
|
-
}
|
|
990
|
-
}
|
|
1399
|
+
});
|
|
1400
|
+
}
|
|
1401
|
+
catch (error) {
|
|
1402
|
+
this.documentBytes = measured;
|
|
1403
|
+
throw error;
|
|
1404
|
+
}
|
|
991
1405
|
this.updates = 0;
|
|
1406
|
+
this.format = STORED_GENERATION_RECORD.storage;
|
|
992
1407
|
this.storedWaiting = still;
|
|
1408
|
+
this.compressLater(record, snapshot);
|
|
993
1409
|
// Unknown ids' deletes go, the ones waiting in memory with a rewrite too.
|
|
994
1410
|
forgetWaiting(doc, addIds(Y.createIdSet(), all, (client) => !known.has(client)));
|
|
1411
|
+
const rows = this.sql
|
|
1412
|
+
.exec(`SELECT COUNT(*) AS n FROM ${this.rowsTable}`)
|
|
1413
|
+
.one().n;
|
|
1414
|
+
const ms = now() - started;
|
|
1415
|
+
tally(this.counters.compaction, ms);
|
|
1416
|
+
this.counters.compaction.lastBytes = snapshot.length;
|
|
1417
|
+
this.log({
|
|
1418
|
+
edytor: 'compaction',
|
|
1419
|
+
ms,
|
|
1420
|
+
bytes: snapshot.length,
|
|
1421
|
+
records: still.isEmpty() ? 2 : 3,
|
|
1422
|
+
rows
|
|
1423
|
+
});
|
|
1424
|
+
return { rows };
|
|
1425
|
+
});
|
|
1426
|
+
}
|
|
1427
|
+
/** Record now as the last stored change (callers run it in the change's transaction). */
|
|
1428
|
+
touch() {
|
|
1429
|
+
this.sql.exec(`INSERT OR REPLACE INTO ${this.metaTable} (key, value) VALUES ('updated', ?)`, Date.now());
|
|
1430
|
+
}
|
|
1431
|
+
/**
|
|
1432
|
+
* When the room last stored a change (ms since the epoch), or `null`
|
|
1433
|
+
* for a room that stored none since 0.1.0-next.23 (H12). Cheap: one
|
|
1434
|
+
* row, no document. `routeDocumentSocket` answers it over HTTP
|
|
1435
|
+
* (`GET <room>?lastUpdated`), also over RPC.
|
|
1436
|
+
*/
|
|
1437
|
+
lastUpdated() {
|
|
1438
|
+
try {
|
|
1439
|
+
return (this.sql
|
|
1440
|
+
.exec(`SELECT value FROM ${this.metaTable} WHERE key = 'updated'`)
|
|
1441
|
+
.toArray()[0]?.value ?? null);
|
|
1442
|
+
}
|
|
1443
|
+
catch {
|
|
1444
|
+
return null; // no table yet
|
|
1445
|
+
}
|
|
1446
|
+
}
|
|
1447
|
+
/** The room's counters (H14), also over RPC — see {@link RoomMetrics}. */
|
|
1448
|
+
metrics() {
|
|
1449
|
+
return noTimers(() => {
|
|
1450
|
+
const stored = this.live === null ? null : this.storage();
|
|
995
1451
|
return {
|
|
996
|
-
|
|
1452
|
+
documentBytes: this.documentBytes,
|
|
1453
|
+
storedBytes: stored?.bytes ?? 0,
|
|
1454
|
+
records: stored?.records ?? 0,
|
|
1455
|
+
rows: stored?.rows ?? 0,
|
|
1456
|
+
updateRecords: this.updates,
|
|
1457
|
+
waitingBytes: this.live?.store.pendingStructs?.update.length ?? 0,
|
|
1458
|
+
sockets: this.ctx.getWebSockets(SOCKET_TAG).length,
|
|
1459
|
+
compaction: { ...this.counters.compaction },
|
|
1460
|
+
fold: { ...this.counters.fold },
|
|
1461
|
+
fanOut: { ...this.counters.fanOut },
|
|
1462
|
+
quotaHits: this.refusalCounts.quota ?? 0,
|
|
1463
|
+
validationDenials: this.refusalCounts.denied ?? 0,
|
|
1464
|
+
refusals: { ...this.refusalCounts },
|
|
1465
|
+
since: this.counters.since
|
|
997
1466
|
};
|
|
998
1467
|
});
|
|
999
1468
|
}
|
|
1469
|
+
/** What the rows take: bytes, logical records, rows. */
|
|
1470
|
+
storage() {
|
|
1471
|
+
return this.sql
|
|
1472
|
+
.exec(`SELECT SUM(length(bytes)) AS bytes, COUNT(DISTINCT record) AS records, COUNT(*) AS rows FROM ${this.rowsTable}`)
|
|
1473
|
+
.one();
|
|
1474
|
+
}
|
|
1475
|
+
/** Write one log entry (H14) — the `log` option's, `console.log` by default. */
|
|
1476
|
+
log(entry) {
|
|
1477
|
+
const log = this.options.log;
|
|
1478
|
+
if (log === false)
|
|
1479
|
+
return;
|
|
1480
|
+
try {
|
|
1481
|
+
if (log)
|
|
1482
|
+
log(entry);
|
|
1483
|
+
else
|
|
1484
|
+
console.log(JSON.stringify(entry));
|
|
1485
|
+
}
|
|
1486
|
+
catch {
|
|
1487
|
+
// a log never fails the room
|
|
1488
|
+
}
|
|
1489
|
+
}
|
|
1000
1490
|
/**
|
|
1001
1491
|
* Drop every waiting delete, stored or in memory (callable over RPC):
|
|
1002
1492
|
* the way out when a writer filled {@link MAX_WAITING_DELETES} with
|
|
@@ -1010,9 +1500,11 @@ export class AttachedDocument {
|
|
|
1010
1500
|
this.heal();
|
|
1011
1501
|
const doc = this.requireDoc();
|
|
1012
1502
|
const dropped = pendingDeletes(doc);
|
|
1503
|
+
const pending = this.pendingBytes();
|
|
1013
1504
|
this.ctx.storage.transactionSync(() => {
|
|
1014
1505
|
this.sql.exec(`DELETE FROM ${this.rowsTable} WHERE kind = 'pending'`);
|
|
1015
1506
|
});
|
|
1507
|
+
this.documentBytes -= pending;
|
|
1016
1508
|
this.storedWaiting = Y.createIdSet();
|
|
1017
1509
|
forgetWaiting(doc, dropped);
|
|
1018
1510
|
this.scheduleSave();
|
|
@@ -1038,6 +1530,11 @@ export class AttachedDocument {
|
|
|
1038
1530
|
reason: storage ? 'storage' : 'internal',
|
|
1039
1531
|
detail: String(append ? this.unstored : error)
|
|
1040
1532
|
});
|
|
1533
|
+
this.log({
|
|
1534
|
+
edytor: 'fault',
|
|
1535
|
+
reason: storage ? 'storage' : 'internal',
|
|
1536
|
+
detail: String(append ? this.unstored : error)
|
|
1537
|
+
});
|
|
1041
1538
|
// Only a registry fault leaves the live doc as stored.
|
|
1042
1539
|
if (!registry && this.live !== null)
|
|
1043
1540
|
this.rebuild();
|
|
@@ -1112,6 +1609,8 @@ export class AttachedDocument {
|
|
|
1112
1609
|
return this.live;
|
|
1113
1610
|
}
|
|
1114
1611
|
adopt(doc) {
|
|
1612
|
+
this.validation?.off();
|
|
1613
|
+
this.validation = null;
|
|
1115
1614
|
this._facade?.dispose();
|
|
1116
1615
|
this._facade = null;
|
|
1117
1616
|
this.live = doc;
|
|
@@ -1124,7 +1623,10 @@ export class AttachedDocument {
|
|
|
1124
1623
|
if (this.unstored !== null)
|
|
1125
1624
|
return;
|
|
1126
1625
|
try {
|
|
1127
|
-
this.ctx.storage.transactionSync(() =>
|
|
1626
|
+
this.ctx.storage.transactionSync(() => {
|
|
1627
|
+
this.insert('update', update);
|
|
1628
|
+
this.touch();
|
|
1629
|
+
});
|
|
1128
1630
|
this.updates++;
|
|
1129
1631
|
this.broadcast(updateFrame(update), origin);
|
|
1130
1632
|
this.scheduleSave();
|
|
@@ -1230,6 +1732,14 @@ export class AttachedDocument {
|
|
|
1230
1732
|
}
|
|
1231
1733
|
// ── Hibernation WebSocket API ────────────────────────────────────────
|
|
1232
1734
|
async fetch(request) {
|
|
1735
|
+
// The `lastUpdated` probe (H12), forwarded by `routeDocumentSocket` once authorized.
|
|
1736
|
+
if (request.headers.get(PROBE_HEADER) === 'lastUpdated') {
|
|
1737
|
+
if (readIdentity(request.headers) === null) {
|
|
1738
|
+
return new Response('verified identity required', { status: 401 });
|
|
1739
|
+
}
|
|
1740
|
+
await this.retryStart();
|
|
1741
|
+
return Response.json({ lastUpdated: this.lastUpdated() });
|
|
1742
|
+
}
|
|
1233
1743
|
if (request.headers.get('Upgrade')?.toLowerCase() !== 'websocket') {
|
|
1234
1744
|
return new Response('expected a websocket upgrade', { status: 426 });
|
|
1235
1745
|
}
|
|
@@ -1297,36 +1807,139 @@ export class AttachedDocument {
|
|
|
1297
1807
|
const attachment = ws.deserializeAttachment();
|
|
1298
1808
|
if (!attachment?.user)
|
|
1299
1809
|
return this.refuse(ws, { reason: 'identity', detail: null });
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1810
|
+
this.onFrame(ws, attachment, doc, new Uint8Array(message));
|
|
1811
|
+
});
|
|
1812
|
+
return true;
|
|
1813
|
+
}
|
|
1814
|
+
/** One frame: whole as it came, or reassembled from its chunks. */
|
|
1815
|
+
onFrame(ws, attachment, doc, bytes) {
|
|
1816
|
+
// The frame quota (H3): never decoded past it.
|
|
1817
|
+
if (bytes.length > this.maxInboundFrameBytes) {
|
|
1818
|
+
return this.overQuota(ws, attachment.user, 'frame', {
|
|
1819
|
+
bytes: bytes.length,
|
|
1820
|
+
limit: this.maxInboundFrameBytes
|
|
1821
|
+
});
|
|
1822
|
+
}
|
|
1823
|
+
const decoder = E.createDecoder(bytes);
|
|
1824
|
+
// 1 · Admission: the generation word, before anything is decoded.
|
|
1825
|
+
if (!E.readProtocolVersion(decoder)) {
|
|
1826
|
+
return this.refuse(ws, { reason: 'generation', detail: bytes[0] });
|
|
1827
|
+
}
|
|
1828
|
+
try {
|
|
1829
|
+
const type = decode(() => E.readVarUint(decoder));
|
|
1830
|
+
if (type === E.messageChunk)
|
|
1831
|
+
return this.onChunk(ws, attachment, doc, decoder);
|
|
1832
|
+
if (type === E.messageSync)
|
|
1833
|
+
return this.onSync(ws, attachment, doc, decoder);
|
|
1834
|
+
if (type === E.messageAwareness) {
|
|
1835
|
+
const entries = decode(() => E.readAwarenessEntries(E.readVarUint8Array(decoder)));
|
|
1836
|
+
return this.onPresence(ws, attachment, doc, entries);
|
|
1837
|
+
}
|
|
1838
|
+
if (type === E.messageQueryAwareness) {
|
|
1839
|
+
return this.send(ws, presenceFrame([...this.presence.values()]));
|
|
1840
|
+
}
|
|
1841
|
+
this.refuse(ws, { reason: 'malformed', detail: `message type ${type}` });
|
|
1842
|
+
}
|
|
1843
|
+
catch (error) {
|
|
1844
|
+
// Only the client's bytes are its fault: the room's own faults close 1011.
|
|
1845
|
+
if (error instanceof MalformedFrame) {
|
|
1846
|
+
this.refuse(ws, { reason: 'malformed', detail: error.message });
|
|
1847
|
+
}
|
|
1848
|
+
else {
|
|
1849
|
+
this.fault(ws, error);
|
|
1305
1850
|
}
|
|
1851
|
+
}
|
|
1852
|
+
}
|
|
1853
|
+
/**
|
|
1854
|
+
* One chunk of a frame too large to send whole (H6: a provider's
|
|
1855
|
+
* reconnect diff): buffered per socket, the whole frame handled once its
|
|
1856
|
+
* sequence ends. A sequence announcing more than `maxInboundFrameBytes`
|
|
1857
|
+
* is a frame quota refusal, before anything is buffered. A part with no
|
|
1858
|
+
* sequence started — the room woke between two chunks and lost the
|
|
1859
|
+
* buffer — faults the socket (1011): its provider redials and resends.
|
|
1860
|
+
*/
|
|
1861
|
+
onChunk(ws, attachment, doc, decoder) {
|
|
1862
|
+
let read = this.chunkReaders.get(ws);
|
|
1863
|
+
if (read === undefined) {
|
|
1864
|
+
read = E.createChunkReader(this.maxInboundFrameBytes);
|
|
1865
|
+
this.chunkReaders.set(ws, read);
|
|
1866
|
+
}
|
|
1867
|
+
let whole;
|
|
1868
|
+
try {
|
|
1869
|
+
whole = read(decoder);
|
|
1870
|
+
}
|
|
1871
|
+
catch (error) {
|
|
1872
|
+
this.chunkReaders.delete(ws);
|
|
1873
|
+
if (error instanceof ChunkLimitError) {
|
|
1874
|
+
return this.overQuota(ws, attachment.user, 'frame', {
|
|
1875
|
+
bytes: error.total,
|
|
1876
|
+
limit: error.limit
|
|
1877
|
+
});
|
|
1878
|
+
}
|
|
1879
|
+
if (error instanceof ChunkSequenceError) {
|
|
1880
|
+
this.note({ reason: 'internal', detail: `chunks: ${error.message}` });
|
|
1881
|
+
this.depart(ws);
|
|
1882
|
+
return this.close(ws, CLOSE.fault, 'chunk sequence lost');
|
|
1883
|
+
}
|
|
1884
|
+
throw new MalformedFrame(String(error));
|
|
1885
|
+
}
|
|
1886
|
+
if (whole !== null)
|
|
1887
|
+
this.onFrame(ws, attachment, doc, whole);
|
|
1888
|
+
}
|
|
1889
|
+
/**
|
|
1890
|
+
* A quota refused the socket's frame (H3): it is not applied, and the
|
|
1891
|
+
* socket is closed `4413` (`quota: <name>`), a refusal its provider
|
|
1892
|
+
* reports (`onSyncRefused`) and does not redial. Never a dropped frame
|
|
1893
|
+
* on a live socket: the sender's later frames would build on it and
|
|
1894
|
+
* wait in the room's memory for good, unstored and unacknowledged; and
|
|
1895
|
+
* a redial would resend it and meet the same quota.
|
|
1896
|
+
*/
|
|
1897
|
+
overQuota(ws, user, quota, detail) {
|
|
1898
|
+
this.note({ reason: 'quota', detail: { user, quota, ...detail } });
|
|
1899
|
+
this.log({ edytor: 'quota', user, quota });
|
|
1900
|
+
this.chunkReaders.delete(ws);
|
|
1901
|
+
this.depart(ws);
|
|
1902
|
+
this.close(ws, CLOSE.quota, `quota: ${quota}`);
|
|
1903
|
+
}
|
|
1904
|
+
/**
|
|
1905
|
+
* The update rate quota (H3): a token bucket per socket, refilled at
|
|
1906
|
+
* `maxUpdatesPerSecond` up to ten seconds' worth. `false`: over it.
|
|
1907
|
+
*/
|
|
1908
|
+
allow(ws) {
|
|
1909
|
+
const now = Date.now();
|
|
1910
|
+
const burst = this.maxUpdatesPerSecond * BURST_SECONDS;
|
|
1911
|
+
const held = this.allowances.get(ws) ?? { tokens: burst, at: now };
|
|
1912
|
+
const tokens = Math.min(burst, held.tokens + ((now - held.at) / 1000) * this.maxUpdatesPerSecond);
|
|
1913
|
+
if (tokens < 1) {
|
|
1914
|
+
this.allowances.set(ws, { tokens, at: now });
|
|
1915
|
+
return false;
|
|
1916
|
+
}
|
|
1917
|
+
this.allowances.set(ws, { tokens: tokens - 1, at: now });
|
|
1918
|
+
return true;
|
|
1919
|
+
}
|
|
1920
|
+
/**
|
|
1921
|
+
* The document quota (H3): would `incoming` bytes take the document
|
|
1922
|
+
* past `maxDocumentBytes`? Its measure is what its records hold and what
|
|
1923
|
+
* the engine holds waiting; update records count until compaction
|
|
1924
|
+
* collapses them, so it compacts first when they would.
|
|
1925
|
+
*/
|
|
1926
|
+
overDocument(incoming, freed) {
|
|
1927
|
+
// A frame that deletes at least what it adds always applies: a full
|
|
1928
|
+
// document can still be trimmed.
|
|
1929
|
+
if (incoming <= freed)
|
|
1930
|
+
return false;
|
|
1931
|
+
const usage = () => this.documentBytes + (this.live?.store.pendingStructs?.update.length ?? 0) + incoming - freed;
|
|
1932
|
+
if (usage() <= this.maxDocumentBytes)
|
|
1933
|
+
return false;
|
|
1934
|
+
if (this.updates > 0) {
|
|
1306
1935
|
try {
|
|
1307
|
-
|
|
1308
|
-
if (type === E.messageSync)
|
|
1309
|
-
return this.onSync(ws, attachment, doc, decoder);
|
|
1310
|
-
if (type === E.messageAwareness) {
|
|
1311
|
-
const entries = decode(() => E.readAwarenessEntries(E.readVarUint8Array(decoder)));
|
|
1312
|
-
return this.onPresence(ws, attachment, doc, entries);
|
|
1313
|
-
}
|
|
1314
|
-
if (type === E.messageQueryAwareness) {
|
|
1315
|
-
return this.send(ws, presenceFrame([...this.presence.values()]));
|
|
1316
|
-
}
|
|
1317
|
-
this.refuse(ws, { reason: 'malformed', detail: `message type ${type}` });
|
|
1936
|
+
this.compact();
|
|
1318
1937
|
}
|
|
1319
1938
|
catch (error) {
|
|
1320
|
-
|
|
1321
|
-
if (error instanceof MalformedFrame) {
|
|
1322
|
-
this.refuse(ws, { reason: 'malformed', detail: error.message });
|
|
1323
|
-
}
|
|
1324
|
-
else {
|
|
1325
|
-
this.fault(ws, error);
|
|
1326
|
-
}
|
|
1939
|
+
this.note({ reason: 'storage', detail: `compaction: ${String(error)}` });
|
|
1327
1940
|
}
|
|
1328
|
-
}
|
|
1329
|
-
return
|
|
1941
|
+
}
|
|
1942
|
+
return usage() > this.maxDocumentBytes;
|
|
1330
1943
|
}
|
|
1331
1944
|
webSocketClose(ws, code, reason) {
|
|
1332
1945
|
if (!this.owns(ws))
|
|
@@ -1360,6 +1973,12 @@ export class AttachedDocument {
|
|
|
1360
1973
|
}
|
|
1361
1974
|
onSync(ws, attachment, doc, decoder) {
|
|
1362
1975
|
const syncType = decode(() => E.readVarUint(decoder));
|
|
1976
|
+
// The rate quota (H3) counts every sync message, a Step1 too (it costs a Step2).
|
|
1977
|
+
if (!this.allow(ws)) {
|
|
1978
|
+
return this.overQuota(ws, attachment.user, 'rate', {
|
|
1979
|
+
perSecond: this.maxUpdatesPerSecond
|
|
1980
|
+
});
|
|
1981
|
+
}
|
|
1363
1982
|
if (syncType === E.messageYjsSyncStep1) {
|
|
1364
1983
|
const sv = decode(() => {
|
|
1365
1984
|
const sv = E.readVarUint8Array(decoder);
|
|
@@ -1375,6 +1994,16 @@ export class AttachedDocument {
|
|
|
1375
1994
|
return this.refuse(ws, { reason: 'malformed', detail: `sync type ${syncType}` });
|
|
1376
1995
|
}
|
|
1377
1996
|
const update = decode(() => E.readVarUint8Array(decoder));
|
|
1997
|
+
const started = now();
|
|
1998
|
+
try {
|
|
1999
|
+
this.onUpdate(ws, attachment, doc, update);
|
|
2000
|
+
}
|
|
2001
|
+
finally {
|
|
2002
|
+
tally(this.counters.fold, now() - started);
|
|
2003
|
+
}
|
|
2004
|
+
}
|
|
2005
|
+
/** A client's Step2 or Update: admitted, applied, stored, relayed, validated, acknowledged. */
|
|
2006
|
+
onUpdate(ws, attachment, doc, update) {
|
|
1378
2007
|
// 2 · Access: a read-only socket writes nothing (it stays, and is told).
|
|
1379
2008
|
if (attachment.readOnly) {
|
|
1380
2009
|
this.note({ reason: 'read-only', detail: attachment.user });
|
|
@@ -1383,28 +2012,73 @@ export class AttachedDocument {
|
|
|
1383
2012
|
// 3 · Attribution: new structs only under client ids this user may
|
|
1384
2013
|
// write under; another user's are stripped, the rest applied.
|
|
1385
2014
|
const decoded = decode(() => Y.decodeUpdate(update));
|
|
2015
|
+
// The document quota (H3), net of what the frame deletes.
|
|
2016
|
+
if (decoded.structs.some((struct) => !(struct instanceof Y.Skip)) &&
|
|
2017
|
+
this.overDocument(update.length, freedBy(doc, decoded.ds))) {
|
|
2018
|
+
return this.overQuota(ws, attachment.user, 'document', {
|
|
2019
|
+
bytes: this.documentBytes,
|
|
2020
|
+
incoming: update.length,
|
|
2021
|
+
limit: this.maxDocumentBytes
|
|
2022
|
+
});
|
|
2023
|
+
}
|
|
1386
2024
|
const sv = stateVector(doc);
|
|
1387
2025
|
const stripped = this.attribute(attachment, newWriters(decoded, sv), sv);
|
|
2026
|
+
// Per-writer block marks (H2): only `n` writes or deletes `del.<n>` /
|
|
2027
|
+
// `wd.<n>`. A client writing another's is stripped whole, as a client
|
|
2028
|
+
// under another user's id is; a delete of another's mark is dropped.
|
|
2029
|
+
const forgers = forgedWriters(doc, decoded.structs, stripped);
|
|
2030
|
+
const owners = new Map();
|
|
2031
|
+
const mayDelete = (writer) => {
|
|
2032
|
+
if (writer === attachment.replica)
|
|
2033
|
+
return true;
|
|
2034
|
+
if (!owners.has(writer))
|
|
2035
|
+
owners.set(writer, this.ownerOf(writer) === attachment.user);
|
|
2036
|
+
return owners.get(writer);
|
|
2037
|
+
};
|
|
2038
|
+
const forgedMarks = decoded.ds.isEmpty()
|
|
2039
|
+
? Y.createIdSet()
|
|
2040
|
+
: forgedDeletes(doc, decoded.ds, mayDelete);
|
|
2041
|
+
if (forgers.size > 0 || !forgedMarks.isEmpty()) {
|
|
2042
|
+
for (const client of forgers)
|
|
2043
|
+
stripped.add(client);
|
|
2044
|
+
this.note({
|
|
2045
|
+
reason: 'mark',
|
|
2046
|
+
detail: { user: attachment.user, writers: [...forgers], ranges: rangeCount(forgedMarks) }
|
|
2047
|
+
});
|
|
2048
|
+
}
|
|
1388
2049
|
// Waiting deletes are capped: a frame that would pass the cap has them dropped.
|
|
1389
2050
|
const waiting = pendingDeletes(doc);
|
|
1390
2051
|
const unheld = decoded.ds.isEmpty() ? null : unheldDeletes(decoded, stripped, doc);
|
|
1391
|
-
const
|
|
1392
|
-
if (
|
|
2052
|
+
const overflow = unheld && rangeCount(waiting) + rangeCount(unheld) > MAX_WAITING_DELETES ? unheld : null;
|
|
2053
|
+
if (overflow !== null) {
|
|
1393
2054
|
this.note({
|
|
1394
2055
|
reason: 'waiting',
|
|
1395
|
-
detail: { user: attachment.user, ranges: rangeCount(
|
|
2056
|
+
detail: { user: attachment.user, ranges: rangeCount(overflow) }
|
|
1396
2057
|
});
|
|
1397
2058
|
}
|
|
2059
|
+
const dropped = overflow === null && forgedMarks.isEmpty()
|
|
2060
|
+
? null
|
|
2061
|
+
: addIds(addIds(Y.createIdSet(), overflow ?? Y.createIdSet()), forgedMarks);
|
|
1398
2062
|
const admitted = stripped.size === 0 && dropped === null
|
|
1399
2063
|
? update
|
|
1400
2064
|
: withoutClients(decoded, stripped, doc, dropped ?? undefined);
|
|
2065
|
+
// Validation (H2) reads the document as it was, and records the frame.
|
|
2066
|
+
const validation = this.options.validate ? this.validating(doc, ws) : null;
|
|
1401
2067
|
// 4 · Schema: the inbound refusal of a foreign stamp (the update's,
|
|
1402
2068
|
// or a pending one it would release — discarded, the sender kept).
|
|
1403
2069
|
// Integrating persists (the doc's update handler) before the ack.
|
|
1404
2070
|
let failure = null;
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
2071
|
+
validation?.history?.trackedOrigins.add(ws);
|
|
2072
|
+
let outcome;
|
|
2073
|
+
try {
|
|
2074
|
+
outcome = this.handle(() => sync.applyRemote(doc, admitted, ws, (error) => {
|
|
2075
|
+
failure = error;
|
|
2076
|
+
}));
|
|
2077
|
+
}
|
|
2078
|
+
finally {
|
|
2079
|
+
validation?.history?.trackedOrigins.delete(ws);
|
|
2080
|
+
}
|
|
2081
|
+
const { applied, problem, discarded } = outcome;
|
|
1408
2082
|
if (this.unstored !== null)
|
|
1409
2083
|
return this.fault(ws, this.unstored);
|
|
1410
2084
|
if (problem !== null)
|
|
@@ -1421,6 +2095,11 @@ export class AttachedDocument {
|
|
|
1421
2095
|
return this.fault(ws, this.unstored);
|
|
1422
2096
|
if (discarded)
|
|
1423
2097
|
this.note({ reason: 'schema', detail: { discarded } });
|
|
2098
|
+
if (validation !== null) {
|
|
2099
|
+
this.validateFrame(validation, attachment);
|
|
2100
|
+
if (this.unstored !== null)
|
|
2101
|
+
return this.fault(ws, this.unstored);
|
|
2102
|
+
}
|
|
1424
2103
|
// A dropped delete is not acknowledged: its sender stays unsaved.
|
|
1425
2104
|
this.send(ws, savedFrame(doc, addIds(this.storedDeletes(doc, decoded.ds), released)));
|
|
1426
2105
|
// Waiting writes the frame released are other sockets' (a relayer's
|
|
@@ -1431,6 +2110,164 @@ export class AttachedDocument {
|
|
|
1431
2110
|
}
|
|
1432
2111
|
this.compactIfDue();
|
|
1433
2112
|
}
|
|
2113
|
+
// ── Validation (H2) ──────────────────────────────────────────────
|
|
2114
|
+
/** A shown block's state, read from the facade (`null`: it does not show). */
|
|
2115
|
+
blockState(id) {
|
|
2116
|
+
const facade = this.facade;
|
|
2117
|
+
if (!facade.isVisibleBlock(id))
|
|
2118
|
+
return null;
|
|
2119
|
+
return {
|
|
2120
|
+
id,
|
|
2121
|
+
type: facade.blockTypeOf(id) ?? '',
|
|
2122
|
+
data: (facade.blockDataOf(id) ?? {}),
|
|
2123
|
+
content: facade.contentJSON(id),
|
|
2124
|
+
parent: facade.parentOf(id)
|
|
2125
|
+
};
|
|
2126
|
+
}
|
|
2127
|
+
/**
|
|
2128
|
+
* The index `validate` reads, for `doc`, set to record the frame of
|
|
2129
|
+
* `origin`: every shown block's state and each parent's children, built
|
|
2130
|
+
* once and kept from every change report. Its history (once the
|
|
2131
|
+
* document is initialized: a history first initializes its document)
|
|
2132
|
+
* records the frame for its compensation.
|
|
2133
|
+
*/
|
|
2134
|
+
validating(doc, origin) {
|
|
2135
|
+
let v = this.validation;
|
|
2136
|
+
if (v === null || v.doc !== doc) {
|
|
2137
|
+
v?.off();
|
|
2138
|
+
const facade = this.facade;
|
|
2139
|
+
const states = new Map();
|
|
2140
|
+
const children = new Map();
|
|
2141
|
+
const walk = (parent) => {
|
|
2142
|
+
const ids = facade.childrenIds(parent);
|
|
2143
|
+
children.set(parent, ids);
|
|
2144
|
+
for (const id of ids) {
|
|
2145
|
+
const state = this.blockState(id);
|
|
2146
|
+
if (state !== null)
|
|
2147
|
+
states.set(id, state);
|
|
2148
|
+
walk(id);
|
|
2149
|
+
}
|
|
2150
|
+
};
|
|
2151
|
+
walk(null);
|
|
2152
|
+
const created = {
|
|
2153
|
+
doc,
|
|
2154
|
+
states,
|
|
2155
|
+
children,
|
|
2156
|
+
history: null,
|
|
2157
|
+
frame: null,
|
|
2158
|
+
off: () => { }
|
|
2159
|
+
};
|
|
2160
|
+
created.off = facade.onChange((change) => this.indexChange(created, change));
|
|
2161
|
+
v = this.validation = created;
|
|
2162
|
+
}
|
|
2163
|
+
if (v.history === null && crdt.doc.isInitialized(doc)) {
|
|
2164
|
+
v.history = this.facade.createUndoManager({ captureTimeout: 0, trackedOrigins: new Set() });
|
|
2165
|
+
}
|
|
2166
|
+
v.frame = { origin, before: new Map(), touched: new Set(), data: false };
|
|
2167
|
+
return v;
|
|
2168
|
+
}
|
|
2169
|
+
/** Keep the index from one change report, recording what the frame being validated touched. */
|
|
2170
|
+
indexChange(v, change) {
|
|
2171
|
+
const frame = v.frame?.origin === change.origin ? v.frame : null;
|
|
2172
|
+
const touch = (id) => {
|
|
2173
|
+
if (frame === null)
|
|
2174
|
+
return;
|
|
2175
|
+
if (!frame.before.has(id))
|
|
2176
|
+
frame.before.set(id, v.states.get(id) ?? null);
|
|
2177
|
+
frame.touched.add(id);
|
|
2178
|
+
};
|
|
2179
|
+
const drop = (id) => {
|
|
2180
|
+
touch(id);
|
|
2181
|
+
for (const child of v.children.get(id) ?? [])
|
|
2182
|
+
drop(child);
|
|
2183
|
+
v.children.delete(id);
|
|
2184
|
+
v.states.delete(id);
|
|
2185
|
+
};
|
|
2186
|
+
const read = (id, moved) => {
|
|
2187
|
+
const was = v.states.get(id);
|
|
2188
|
+
const state = this.blockState(id);
|
|
2189
|
+
if (state === null)
|
|
2190
|
+
return drop(id);
|
|
2191
|
+
if (!moved || was === undefined || was.parent !== state.parent)
|
|
2192
|
+
touch(id);
|
|
2193
|
+
v.states.set(id, state);
|
|
2194
|
+
};
|
|
2195
|
+
for (const id of change.removed)
|
|
2196
|
+
drop(id);
|
|
2197
|
+
const add = (block) => {
|
|
2198
|
+
touch(block.id);
|
|
2199
|
+
read(block.id, false);
|
|
2200
|
+
v.children.set(block.id, this.facade.childrenIds(block.id));
|
|
2201
|
+
block.children.forEach(add);
|
|
2202
|
+
};
|
|
2203
|
+
for (const block of change.added.values())
|
|
2204
|
+
add(block);
|
|
2205
|
+
for (const id of change.moved)
|
|
2206
|
+
read(id, true);
|
|
2207
|
+
for (const id of change.meta.keys())
|
|
2208
|
+
read(id, false);
|
|
2209
|
+
for (const id of change.content.keys())
|
|
2210
|
+
read(id, false);
|
|
2211
|
+
// A block moved among its siblings, not shifted by another's move.
|
|
2212
|
+
for (const [parent, ids] of change.order) {
|
|
2213
|
+
for (const id of reordered(v.children.get(parent) ?? [], ids))
|
|
2214
|
+
touch(id);
|
|
2215
|
+
v.children.set(parent, ids);
|
|
2216
|
+
}
|
|
2217
|
+
if (frame !== null && change.data !== undefined)
|
|
2218
|
+
frame.data = true;
|
|
2219
|
+
}
|
|
2220
|
+
/**
|
|
2221
|
+
* Ask `validate` about the frame just applied (H2); a denial is
|
|
2222
|
+
* compensated by the room's own transaction: the history undo of that
|
|
2223
|
+
* frame (`room.validate.inverse`), or, for the frame that initialized
|
|
2224
|
+
* the document (no history recorded it), a delete of the blocks it
|
|
2225
|
+
* added. Stored and sent to every socket, the sender's included.
|
|
2226
|
+
*/
|
|
2227
|
+
validateFrame(v, attachment) {
|
|
2228
|
+
const frame = v.frame;
|
|
2229
|
+
v.frame = null;
|
|
2230
|
+
if (frame === null || (frame.touched.size === 0 && !frame.data)) {
|
|
2231
|
+
if (v.history)
|
|
2232
|
+
this.facade.releaseHistory(v.history);
|
|
2233
|
+
return;
|
|
2234
|
+
}
|
|
2235
|
+
const touched = [...frame.touched];
|
|
2236
|
+
let allowed;
|
|
2237
|
+
try {
|
|
2238
|
+
allowed =
|
|
2239
|
+
this.options.validate({
|
|
2240
|
+
user: attachment.user,
|
|
2241
|
+
replica: attachment.replica,
|
|
2242
|
+
touched,
|
|
2243
|
+
dataChanged: frame.data,
|
|
2244
|
+
before: (id) => frame.before.has(id) ? frame.before.get(id) : (v.states.get(id) ?? null),
|
|
2245
|
+
after: (id) => v.states.get(id) ?? null,
|
|
2246
|
+
facade: this.facade
|
|
2247
|
+
}) !== false;
|
|
2248
|
+
}
|
|
2249
|
+
catch (error) {
|
|
2250
|
+
this.note({ reason: 'internal', detail: `validate: ${String(error)}` });
|
|
2251
|
+
allowed = false;
|
|
2252
|
+
}
|
|
2253
|
+
if (!allowed) {
|
|
2254
|
+
this.note({ reason: 'denied', detail: { user: attachment.user, touched } });
|
|
2255
|
+
this.log({ edytor: 'denied', user: attachment.user, touched: touched.length });
|
|
2256
|
+
const history = v.history;
|
|
2257
|
+
if (history !== null && history.undoStack.length > 0) {
|
|
2258
|
+
history.undo();
|
|
2259
|
+
}
|
|
2260
|
+
else {
|
|
2261
|
+
const added = touched.filter((id) => frame.before.get(id) === null && v.states.has(id));
|
|
2262
|
+
const facade = this.facade;
|
|
2263
|
+
if (added.length > 0) {
|
|
2264
|
+
facade.transact(() => facade.apply(facade.prepare.deleteBlocks(added)), ROOM_ORIGIN);
|
|
2265
|
+
}
|
|
2266
|
+
}
|
|
2267
|
+
}
|
|
2268
|
+
if (v.history)
|
|
2269
|
+
this.facade.releaseHistory(v.history);
|
|
2270
|
+
}
|
|
1434
2271
|
/**
|
|
1435
2272
|
* A delete of items the room does not hold (a relayer's delete of an
|
|
1436
2273
|
* author's edit a restore lost) waits in the engine for them: it is
|
|
@@ -1460,16 +2297,21 @@ export class AttachedDocument {
|
|
|
1460
2297
|
const added = Y.diffIdSet(discarded ? now : Y.diffIdSet(now, waiting), stay);
|
|
1461
2298
|
const released = heldDeletes(doc, Y.diffIdSet(waiting, now));
|
|
1462
2299
|
if (discarded || !added.isEmpty()) {
|
|
2300
|
+
const before = this.documentBytes;
|
|
2301
|
+
const pending = discarded ? this.pendingBytes() : 0;
|
|
1463
2302
|
try {
|
|
1464
2303
|
this.ctx.storage.transactionSync(() => {
|
|
1465
2304
|
if (discarded)
|
|
1466
2305
|
this.sql.exec(`DELETE FROM ${this.rowsTable} WHERE kind = 'pending'`);
|
|
1467
2306
|
if (!added.isEmpty())
|
|
1468
2307
|
this.insert('pending', deletesUpdate(added));
|
|
2308
|
+
this.touch();
|
|
1469
2309
|
});
|
|
2310
|
+
this.documentBytes -= pending;
|
|
1470
2311
|
this.updates++;
|
|
1471
2312
|
}
|
|
1472
2313
|
catch (error) {
|
|
2314
|
+
this.documentBytes = before;
|
|
1473
2315
|
this.unstored = error;
|
|
1474
2316
|
return released;
|
|
1475
2317
|
}
|
|
@@ -1569,6 +2411,8 @@ export class AttachedDocument {
|
|
|
1569
2411
|
try {
|
|
1570
2412
|
for (const piece of pieces)
|
|
1571
2413
|
ws.send(piece);
|
|
2414
|
+
this.counters.fanOut.messages += pieces.length;
|
|
2415
|
+
this.counters.fanOut.bytes += bytes.length;
|
|
1572
2416
|
}
|
|
1573
2417
|
catch {
|
|
1574
2418
|
// a socket that died mid-broadcast gets its close event
|
|
@@ -1615,19 +2459,25 @@ export class DocumentRoom extends DurableObject {
|
|
|
1615
2459
|
super(ctx, env);
|
|
1616
2460
|
const knobs = env;
|
|
1617
2461
|
const saves = this.onSave !== DocumentRoom.prototype.onSave;
|
|
2462
|
+
const validates = this.validate !== DocumentRoom.prototype.validate;
|
|
1618
2463
|
const semantics = () => this.semantics();
|
|
1619
2464
|
this.room = new AttachedDocument(ctx, {
|
|
1620
2465
|
maxRowBytes: Number(knobs.EDYTOR_MAX_ROW_BYTES),
|
|
1621
2466
|
maxFrameBytes: Number(knobs.EDYTOR_MAX_FRAME_BYTES),
|
|
1622
2467
|
compactAfter: Number(knobs.EDYTOR_COMPACT_AFTER),
|
|
1623
2468
|
saveAfter: Number(knobs.EDYTOR_SAVE_AFTER),
|
|
2469
|
+
maxDocumentBytes: Number(knobs.EDYTOR_MAX_DOCUMENT_BYTES),
|
|
2470
|
+
maxInboundFrameBytes: Number(knobs.EDYTOR_MAX_INBOUND_FRAME_BYTES),
|
|
2471
|
+
maxUpdatesPerSecond: Number(knobs.EDYTOR_MAX_UPDATES_PER_SECOND),
|
|
1624
2472
|
tablePrefix: '',
|
|
1625
2473
|
// A getter: a subclass's fields do not exist yet in this constructor.
|
|
1626
2474
|
get semantics() {
|
|
1627
2475
|
return semantics();
|
|
1628
2476
|
},
|
|
1629
2477
|
onLoad: () => this.onLoad(),
|
|
1630
|
-
onSave: saves ? (document) => this.onSave(document) : undefined
|
|
2478
|
+
onSave: saves ? (document) => this.onSave(document) : undefined,
|
|
2479
|
+
validate: validates ? (frame) => this.validate(frame) : undefined,
|
|
2480
|
+
log: (entry) => this.log(entry)
|
|
1631
2481
|
});
|
|
1632
2482
|
}
|
|
1633
2483
|
/** Retrieve — see {@link AttachDocumentOptions.onLoad}. */
|
|
@@ -1636,6 +2486,21 @@ export class DocumentRoom extends DurableObject {
|
|
|
1636
2486
|
}
|
|
1637
2487
|
/** Save — see {@link AttachDocumentOptions.onSave}. Not overridden: no alarm is ever set. */
|
|
1638
2488
|
async onSave(_document) { }
|
|
2489
|
+
/**
|
|
2490
|
+
* Validate a client frame — see {@link AttachDocumentOptions.validate}:
|
|
2491
|
+
* `false` denies it, and the room writes its inverse. Not overridden: no
|
|
2492
|
+
* frame is validated, and no block index is kept.
|
|
2493
|
+
*/
|
|
2494
|
+
validate(_frame) { }
|
|
2495
|
+
/**
|
|
2496
|
+
* One log entry (H14): `console.log` of it as JSON, unless the
|
|
2497
|
+
* `EDYTOR_LOG` var is `off`. Override it to send entries elsewhere.
|
|
2498
|
+
*/
|
|
2499
|
+
log(entry) {
|
|
2500
|
+
if (String(this.env.EDYTOR_LOG ?? '') === 'off')
|
|
2501
|
+
return;
|
|
2502
|
+
console.log(JSON.stringify(entry));
|
|
2503
|
+
}
|
|
1639
2504
|
/**
|
|
1640
2505
|
* Block roles — see {@link AttachDocumentOptions.semantics}. Read once,
|
|
1641
2506
|
* at first use (after construction: it may return a subclass field).
|
|
@@ -1663,6 +2528,18 @@ export class DocumentRoom extends DurableObject {
|
|
|
1663
2528
|
compact() {
|
|
1664
2529
|
return this.room.compact();
|
|
1665
2530
|
}
|
|
2531
|
+
/** The room's counters, also over RPC — see {@link RoomMetrics}. */
|
|
2532
|
+
metrics() {
|
|
2533
|
+
return this.room.metrics();
|
|
2534
|
+
}
|
|
2535
|
+
/** When the room last stored a change, also over RPC — see {@link AttachedDocument.lastUpdated}. */
|
|
2536
|
+
lastUpdated() {
|
|
2537
|
+
return this.room.lastUpdated();
|
|
2538
|
+
}
|
|
2539
|
+
/** The snapshot compression in flight, also over RPC — see {@link AttachedDocument.compressed}. */
|
|
2540
|
+
compressed() {
|
|
2541
|
+
return this.room.compressed();
|
|
2542
|
+
}
|
|
1666
2543
|
/** Drop the stored waiting deletes, also over RPC — see {@link AttachedDocument.dropWaitingDeletes}. */
|
|
1667
2544
|
dropWaitingDeletes() {
|
|
1668
2545
|
return this.room.dropWaitingDeletes();
|