edytor 0.1.0-next.21 → 0.1.0-next.23

package/README.md

@@ -15,7 +15,7 @@
 
 Edytor aims to be for Svelte what Slate.js is for React: a heavily customizable editor with an API to build any kind of collaborative rich text editor.
 
-> **Work in progress.** Edytor is a pre-release (`0.1.0-next.21`, `edytor@next` on npm) and not ready for production; the API changes between releases without a compatibility layer. The untagged `edytor@0.0.11` on npm predates the current API. Issues and PRs are welcome: when you report a bug, include the document's JSON value.
+> **Work in progress.** Edytor is a pre-release (`0.1.0-next.23`, `edytor@next` on npm) and not ready for production; the API changes between releases without a compatibility layer. The untagged `edytor@0.0.11` on npm predates the current API. Issues and PRs are welcome: when you report a bug, include the document's JSON value.
 
 ## Features
 

package/dist/cloudflare/DocumentRoom.d.ts

@@ -34,8 +34,9 @@
  *   what storage does not.
  * - every INTEGRATED update is appended to SQLite as one record split into
  *   rows ≤ `maxRowBytes` (2 MB row cap) in one `transactionSync`, then
- *   broadcast; after `compactAfter` update records the rows are merged
- *   (`mergeUpdates`) into one snapshot record, after the message is
+ *   broadcast; after `compactAfter` update records the rows are replaced
+ *   by one snapshot record of the live document's state (its healed
+ *   state, deleted content collected: P2), after the message is
  *   acknowledged — never inside the engine's `update` observer, where one
  *   throw would silence every later emit. Memory never runs ahead of
  *   storage: when an append fails, nothing is relayed or acknowledged, the
@@ -51,6 +52,11 @@
  *   left out of the ack), the rest applied. Compaction reclaims the
  *   waiting deletes of client ids no user registered and no socket holds;
  *   `dropWaitingDeletes()` every one.
+ * - storage format (P5): the generation record says `storage: 'v2'` —
+ *   snapshots in the v2 encoding, compressed in place after they are
+ *   stored (`compressLater`), inflated at start (`inflate`); update and
+ *   pending records stay v1. A container without `storage` (0.1.0-next.22)
+ *   is all v1, and its next compaction rewrites it.
  * - a Step2 serves the STORED state: the engine's pending structs (waiting
  *   for a dependency, never stored) are not served.
  * - a frame larger than `maxFrameBytes` (32 MiB) goes out as chunks the
@@ -89,13 +95,29 @@
  */
 import { DurableObject } from 'cloudflare:workers';
 import * as E from '../crdt/index.js';
-import type { AwarenessEntry, DocumentSemanticsConfig, EdytorDoc, JSONDoc, YDoc } from '../crdt/index.js';
+import type { AwarenessEntry, DocumentSemanticsConfig, EdytorDoc, JSONBlock, JSONDoc, YDoc } from '../crdt/index.js';
 /** Under SQLite-backed Durable Objects' 2 MB row cap, with room for the other columns. */
 export declare const DEFAULT_MAX_ROW_BYTES: number;
 /** Update records before the rows are merged into one snapshot. */
 export declare const DEFAULT_COMPACT_AFTER = 500;
 /** ms between the first unsaved change and `onSave`. */
 export declare const DEFAULT_SAVE_AFTER = 2000;
+/**
+ * Room quotas (H3), each refused with `4413` (`quota: <name>`). The
+ * document's size: what its records hold (uncompressed) and the updates the
+ * engine holds waiting — the live document holds about that, several
+ * times over in memory for text (an isolate has 128 MB). Raise or lower
+ * it for your documents.
+ */
+export declare const DEFAULT_MAX_DOCUMENT_BYTES: number;
+/** The largest frame a socket may send, reassembled from chunks (the platform caps one message at 32 MiB). */
+export declare const DEFAULT_MAX_INBOUND_FRAME_BYTES: number;
+/**
+ * Sync messages a socket may send per second, sustained; a burst of ten
+ * seconds' worth is allowed (a reconnect, a paste). Far above typing
+ * speed: a script should batch its edits into transactions.
+ */
+export declare const DEFAULT_MAX_UPDATES_PER_SECOND = 50;
 /**
  * Delete ranges the room keeps waiting for items it does not hold (a
  * relayer's delete of an author's edit a restore lost). Every update the
@@ -128,6 +150,8 @@ export type SavedDocument = {
 };
 /** What `onLoad` may return: JSON, a bare v14 update, or `{ update, replicas }` (a `SavedDocument`). */
 export type LoadedDocument = JSONDoc | Uint8Array | Pick<SavedDocument, 'update' | 'replicas'>;
+/** The header `routeDocumentSocket` sets on an authorized `lastUpdated` probe (H12). */
+export declare const PROBE_HEADER = "X-Edytor-Probe";
 /** The transaction origin of the room's own edits (`transact`, `onLoad` seeds). */
 export declare const ROOM_ORIGIN: unique symbol;
 /**
@@ -150,6 +174,11 @@ export type DocumentRoomEnv = {
     EDYTOR_MAX_FRAME_BYTES?: string | number;
     EDYTOR_COMPACT_AFTER?: string | number;
     EDYTOR_SAVE_AFTER?: string | number;
+    EDYTOR_MAX_DOCUMENT_BYTES?: string | number;
+    EDYTOR_MAX_INBOUND_FRAME_BYTES?: string | number;
+    EDYTOR_MAX_UPDATES_PER_SECOND?: string | number;
+    /** `off`: the room logs nothing (default: one JSON line per compaction, quota hit, denial, fault). */
+    EDYTOR_LOG?: string;
 };
 /** What a socket is bound to: its verified user, its replica (Yjs client id), its access. */
 export type SocketIdentity = {
@@ -163,9 +192,76 @@ export type Attachment = SocketIdentity & {
     clock: number | null;
 };
 export type Refusal = {
-    reason: 'generation' | 'schema' | 'malformed' | 'container' | 'identity' | 'replica' | 'read-only' | 'waiting' | 'storage' | 'internal' | 'orphan' | 'relayed';
+    reason: 'generation' | 'schema' | 'malformed' | 'container' | 'identity' | 'replica' | 'read-only' | 'waiting' | 'storage' | 'internal' | 'quota' | 'mark' | 'denied' | 'orphan' | 'relayed';
     detail: unknown;
 };
+/** A duration series: how many, their total and longest (ms; see {@link RoomMetrics}). */
+export type Timing = {
+    count: number;
+    totalMs: number;
+    maxMs: number;
+    lastMs: number;
+};
+/**
+ * The room's counters (H14), since this instance started (memory: a wake
+ * starts them again), read with `metrics()` (also over RPC). Times are
+ * what the runtime's clock reports: Workers advance it only across I/O, so
+ * in production synchronous work (a compaction, a fold) often reads `0`;
+ * sizes and counts are exact.
+ */
+export type RoomMetrics = {
+    /** The document's size as the document quota measures it: its records, uncompressed. */
+    documentBytes: number;
+    /** What its rows take in storage (a snapshot compressed). */
+    storedBytes: number;
+    /** Logical records stored (the generation record included), and their rows. */
+    records: number;
+    rows: number;
+    /** Update records since the last compaction (compaction runs at `compactAfter`). */
+    updateRecords: number;
+    /** Bytes of updates held waiting for a dependency (in memory, never stored). */
+    waitingBytes: number;
+    /** The document's open sockets. */
+    sockets: number;
+    /** Compactions run, with their time. */
+    compaction: Timing & {
+        lastBytes: number;
+    };
+    /** Client sync frames folded (applied, indexed, stored and relayed), with their time. */
+    fold: Timing;
+    /** Frames sent to other sockets (relays, presence, acknowledgements to others), and their bytes. */
+    fanOut: {
+        messages: number;
+        bytes: number;
+    };
+    /** Writes refused by a quota (`quota` refusals) and frames `validate` denied. */
+    quotaHits: number;
+    validationDenials: number;
+    /** Every refusal, by reason (as `refusalCounts`). */
+    refusals: Partial<Record<Refusal['reason'], number>>;
+    /** When this instance started (ms since the epoch). */
+    since: number;
+};
+/** One line of the room's log (H14): a compaction, a quota hit, a denial, a fault. */
+export type RoomLogEntry = {
+    edytor: 'compaction';
+    ms: number;
+    bytes: number;
+    records: number;
+    rows: number;
+} | {
+    edytor: 'quota';
+    user: string;
+    quota: string;
+} | {
+    edytor: 'denied';
+    user: string;
+    touched: number;
+} | {
+    edytor: 'fault';
+    reason: 'storage' | 'internal';
+    detail: string;
+};
 /** `pending`: deletes of items the room does not hold yet (they wait for them). */
 type RowKind = 'generation' | 'update' | 'snapshot' | 'pending';
 /**
@@ -177,12 +273,50 @@ type RowKind = 'generation' | 'update' | 'snapshot' | 'pending';
  * `isRefusal` in `crdt/providers/room.ts`).
  */
 export declare const closedSocket: (code: number, reason: string) => Response;
+/** A stored record, reassembled: its kind, record number, bytes, and whether they are v2 (a v2 container's snapshot). */
+export type StoredRecord = {
+    kind: RowKind;
+    record: number;
+    bytes: Uint8Array<ArrayBuffer>;
+    v2: boolean;
+};
 /** Run an entry point with timers forbidden — a pending timer keeps a Durable Object from hibernating. */
 export declare const noTimers: <T>(fn: () => T) => T;
 /** A Yjs client id (a non-negative safe integer), or `null`. */
 export declare const parseReplica: (raw: unknown) => number | null;
 /** An awareness frame carrying `entries` — no `Awareness` instance involved. */
 export declare const presenceFrame: (entries: AwarenessEntry[]) => Uint8Array;
+/** A block as `validate` reads it, before or after a frame. */
+export type ValidatedBlock = {
+    id: string;
+    type: string;
+    data: Record<string, unknown>;
+    content: NonNullable<JSONBlock['content']>;
+    /** Its display parent (`null`: the document's root). */
+    parent: string | null;
+};
+/** One client frame the room applied, as `validate` reads it. */
+export type FrameValidation = {
+    /** The sender's verified user. */
+    user: string;
+    /** The sender's replica (client id), when bound. */
+    replica: number | null;
+    /**
+     * The blocks the frame changed, from the facade's change report: shown
+     * or hidden (with their subtrees), given another parent or moved among
+     * their siblings (not shifted by a sibling's move), retyped, given
+     * other data, or other content.
+     */
+    touched: string[];
+    /** The frame changed the document's own data. */
+    dataChanged: boolean;
+    /** A block as it was before the frame (`null`: it did not show). */
+    before: (id: string) => ValidatedBlock | null;
+    /** A block as it is now (`null`: it does not show). */
+    after: (id: string) => ValidatedBlock | null;
+    /** The live document. Read it; never write from `validate` (defer a `transact` instead). */
+    facade: EdytorDoc;
+};
 /** What {@link attachDocument} (and `DocumentRoom`) takes. */
 export type AttachDocumentOptions = {
     /**
@@ -211,6 +345,29 @@ export type AttachDocumentOptions = {
     maxFrameBytes?: number;
     /** Prefix of the document's SQL tables, beside your own (default `'edytor_'`). */
     tablePrefix?: string;
+    /** Largest document, in stored bytes (default {@link DEFAULT_MAX_DOCUMENT_BYTES}). */
+    maxDocumentBytes?: number;
+    /** Largest frame a socket may send, reassembled (default {@link DEFAULT_MAX_INBOUND_FRAME_BYTES}). */
+    maxInboundFrameBytes?: number;
+    /** Sync messages per second a socket may send (default {@link DEFAULT_MAX_UPDATES_PER_SECOND}). */
+    maxUpdatesPerSecond?: number;
+    /**
+     * Accept, then compensate (H2): after the room applied and stored a
+     * client frame that changed blocks, it is asked whether to keep it.
+     * `false` (or a throw) denies it: the room writes the frame's inverse as
+     * its own transaction — the history undo of exactly that frame
+     * (`room.validate.inverse`) — stores it and sends it to every socket,
+     * the sender's included, so every replica converges. The frame stays
+     * stored and acknowledged and the socket open. Synchronous; runs only
+     * when set (the room then keeps an index of every block's state).
+     */
+    validate?: (frame: FrameValidation) => boolean | void;
+    /**
+     * The room's log (H14): one entry per compaction, quota hit, denial or
+     * fault. Default: `console.log` of the entry as JSON (Workers Logs and
+     * `wrangler tail` collect it); `false` logs nothing.
+     */
+    log?: ((entry: RoomLogEntry) => void) | false;
     /**
      * The block roles `transact` edits obey — the document semantics your
      * clients' plugins declare (`defaultType` also names the block an empty
@@ -234,6 +391,9 @@ export declare class AttachedDocument {
     readonly maxFrameBytes: number;
     readonly compactAfter: number;
     readonly saveAfter: number;
+    readonly maxDocumentBytes: number;
+    readonly maxInboundFrameBytes: number;
+    readonly maxUpdatesPerSecond: number;
     private live;
     /** Why the stored container was refused (another generation, a torn record, `onLoad`…). */
     failure: Error | null;
@@ -252,6 +412,16 @@ export declare class AttachedDocument {
     };
     private nextRecord;
     private updates;
+    /** What the records hold, uncompressed (the generation record aside): the document quota's measure. */
+    private documentBytes;
+    /** The counters `metrics()` reads (H14). */
+    private readonly counters;
+    /** The index `validate` reads (built at its first frame, for the live document). */
+    private validation;
+    /** Each socket's update allowance (a token bucket; memory: a wake refills it). */
+    private readonly allowances;
+    /** Each socket's chunked frame in flight (memory: a wake loses it, and the socket is faulted). */
+    private readonly chunkReaders;
     /** A failed append: the live doc holds what storage does not, until it is rebuilt. */
     private unstored;
     /** Inside `transact`, its commit included (a nested call is refused but while `fn` runs). */
@@ -260,6 +430,12 @@ export declare class AttachedDocument {
     private running;
     /** Inside a client frame's apply, its events included: the frame's handler settles it. */
     private handling;
+    /** How the container stores its snapshots (its generation record says; a fresh one: this build's). */
+    private format;
+    /** The snapshot compression in flight (`compressed()`). */
+    private compressing;
+    /** The raw bytes of the compressed snapshot record (`records` reads it; only `start` can inflate). */
+    private inflated;
     /** The waiting deletes stored as `pending` records (the engine may hold more, in memory). */
     private storedWaiting;
     /** The failure is the storage's (a failed read, `onLoad`'s store down): the next dial starts again. */
@@ -272,6 +448,7 @@ export declare class AttachedDocument {
     private readonly options;
     private readonly rowsTable;
     private readonly replicasTable;
+    private readonly metaTable;
     private _lookups;
     constructor(ctx: DurableObjectState, options?: AttachDocumentOptions);
     /**
@@ -279,6 +456,12 @@ export declare class AttachedDocument {
      * out of `blockConcurrencyWhile` resets the object on every request.
      */
     private start;
+    /**
+     * Inflate a compressed snapshot record (P5) for `records`, which reads
+     * synchronously (a rebuild does): decompression is asynchronous, so
+     * only `start` can. One that does not inflate is a corrupt container.
+     */
+    private inflate;
     /** A read of the rows or `onLoad` failed (storage was down): start again. */
     private retryStart;
     private fail;
@@ -359,20 +542,43 @@ export declare class AttachedDocument {
      * engine refuses) is refused for good.
      */
     private load;
-    /** Reassembled logical records in write order; a torn record throws. */
-    records(): Array<{
-        kind: RowKind;
-        bytes: Uint8Array<ArrayBuffer>;
-    }>;
+    /** The bytes of the stored `pending` records (uncompressed, as every record but a snapshot). */
+    private pendingBytes;
+    /**
+     * Reassembled logical records in write order (of one kind, with
+     * `only`); a torn record throws. A v2 container's snapshot is v2
+     * (`v2`), inflated when it is stored compressed (P5).
+     */
+    records(only?: RowKind): StoredRecord[];
     /**
      * One logical record split into rows ≤ `maxRowBytes` (callers run it in
      * a transaction). The first record of an empty container brings the
      * generation record with it.
      */
     private insert;
+    /** Record `record`'s rows (at the row positions `seqs`, when given). */
+    private writeRecord;
+    /**
+     * Compress snapshot `record` in place once it is stored (P5): gzip is
+     * asynchronous, and the store-before-ack path is not. Its rows are
+     * rewritten (at their positions) only while it is still the stored
+     * snapshot and only when that shrinks it; the raw bytes stay in memory
+     * (`inflated`) for a rebuild's synchronous read. A platform without
+     * `CompressionStream` keeps it raw.
+     */
+    private compressLater;
+    /**
+     * Resolves once the snapshot compression in flight, if any, is stored
+     * (P5): a compaction stores its snapshot raw, then compresses it in
+     * place. Wait for it before reading the rows' sizes.
+     */
+    compressed(): Promise<void>;
     /**
      * Compaction: the rows become the generation record + one chunked
-     * snapshot, `mergeUpdates` of every stored record, atomically, + one
+     * snapshot of the live document's state (P2: the healed state, deleted
+     * content collected but the text a replica may copy again, P11 —
+     * never a merge of the records, which keeps every keystroke's struct
+     * and every deleted character), atomically, + one
      * `pending` record of the stored deletes still waiting. A waiting delete
      * of a client id no user registered and no socket holds is reclaimed,
      * in memory too: nothing can deliver its item (a made-up id) — the
@@ -385,6 +591,21 @@ export declare class AttachedDocument {
     compact(): {
         rows: number;
     };
+    /** Record now as the last stored change (callers run it in the change's transaction). */
+    private touch;
+    /**
+     * When the room last stored a change (ms since the epoch), or `null`
+     * for a room that stored none since 0.1.0-next.23 (H12). Cheap: one
+     * row, no document. `routeDocumentSocket` answers it over HTTP
+     * (`GET <room>?lastUpdated`), also over RPC.
+     */
+    lastUpdated(): number | null;
+    /** The room's counters (H14), also over RPC — see {@link RoomMetrics}. */
+    metrics(): RoomMetrics;
+    /** What the rows take: bytes, logical records, rows. */
+    private storage;
+    /** Write one log entry (H14) — the `log` option's, `console.log` by default. */
+    private log;
     /**
      * Drop every waiting delete, stored or in memory (callable over RPC):
      * the way out when a writer filled {@link MAX_WAITING_DELETES} with
@@ -469,11 +690,65 @@ export declare class AttachedDocument {
     /** Is `ws` one of this document's sockets? */
     owns(ws: WebSocket): boolean;
     webSocketMessage(ws: WebSocket, message: string | ArrayBuffer): boolean;
+    /** One frame: whole as it came, or reassembled from its chunks. */
+    private onFrame;
+    /**
+     * One chunk of a frame too large to send whole (H6: a provider's
+     * reconnect diff): buffered per socket, the whole frame handled once its
+     * sequence ends. A sequence announcing more than `maxInboundFrameBytes`
+     * is a frame quota refusal, before anything is buffered. A part with no
+     * sequence started — the room woke between two chunks and lost the
+     * buffer — faults the socket (1011): its provider redials and resends.
+     */
+    private onChunk;
+    /**
+     * A quota refused the socket's frame (H3): it is not applied, and the
+     * socket is closed `4413` (`quota: <name>`), a refusal its provider
+     * reports (`onSyncRefused`) and does not redial. Never a dropped frame
+     * on a live socket: the sender's later frames would build on it and
+     * wait in the room's memory for good, unstored and unacknowledged; and
+     * a redial would resend it and meet the same quota.
+     */
+    private overQuota;
+    /**
+     * The update rate quota (H3): a token bucket per socket, refilled at
+     * `maxUpdatesPerSecond` up to ten seconds' worth. `false`: over it.
+     */
+    private allow;
+    /**
+     * The document quota (H3): would `incoming` bytes take the document
+     * past `maxDocumentBytes`? Its measure is what its records hold and what
+     * the engine holds waiting; update records count until compaction
+     * collapses them, so it compacts first when they would.
+     */
+    private overDocument;
     webSocketClose(ws: WebSocket, code: number, reason: string): boolean;
     webSocketError(ws: WebSocket): boolean;
     /** The departure the client may not have announced — from the attachment, so it works after a wake. */
     private depart;
     private onSync;
+    /** A client's Step2 or Update: admitted, applied, stored, relayed, validated, acknowledged. */
+    private onUpdate;
+    /** A shown block's state, read from the facade (`null`: it does not show). */
+    private blockState;
+    /**
+     * The index `validate` reads, for `doc`, set to record the frame of
+     * `origin`: every shown block's state and each parent's children, built
+     * once and kept from every change report. Its history (once the
+     * document is initialized: a history first initializes its document)
+     * records the frame for its compensation.
+     */
+    private validating;
+    /** Keep the index from one change report, recording what the frame being validated touched. */
+    private indexChange;
+    /**
+     * Ask `validate` about the frame just applied (H2); a denial is
+     * compensated by the room's own transaction: the history undo of that
+     * frame (`room.validate.inverse`), or, for the frame that initialized
+     * the document (no history recorded it), a delete of the blocks it
+     * added. Stored and sent to every socket, the sender's included.
+     */
+    private validateFrame;
     /**
      * A delete of items the room does not hold (a relayer's delete of an
      * author's edit a restore lost) waits in the engine for them: it is
@@ -549,6 +824,17 @@ export declare class DocumentRoom<Env = DocumentRoomEnv> extends DurableObject<E
     protected onLoad(): Promise<LoadedDocument | null | undefined>;
     /** Save — see {@link AttachDocumentOptions.onSave}. Not overridden: no alarm is ever set. */
     protected onSave(_document: SavedDocument): Promise<void>;
+    /**
+     * Validate a client frame — see {@link AttachDocumentOptions.validate}:
+     * `false` denies it, and the room writes its inverse. Not overridden: no
+     * frame is validated, and no block index is kept.
+     */
+    protected validate(_frame: FrameValidation): boolean | void;
+    /**
+     * One log entry (H14): `console.log` of it as JSON, unless the
+     * `EDYTOR_LOG` var is `off`. Override it to send entries elsewhere.
+     */
+    protected log(entry: RoomLogEntry): void;
     /**
      * Block roles — see {@link AttachDocumentOptions.semantics}. Read once,
      * at first use (after construction: it may return a subclass field).
@@ -564,6 +850,12 @@ export declare class DocumentRoom<Env = DocumentRoomEnv> extends DurableObject<E
     compact(): {
         rows: number;
     };
+    /** The room's counters, also over RPC — see {@link RoomMetrics}. */
+    metrics(): RoomMetrics;
+    /** When the room last stored a change, also over RPC — see {@link AttachedDocument.lastUpdated}. */
+    lastUpdated(): number | null;
+    /** The snapshot compression in flight, also over RPC — see {@link AttachedDocument.compressed}. */
+    compressed(): Promise<void>;
     /** Drop the stored waiting deletes, also over RPC — see {@link AttachedDocument.dropWaitingDeletes}. */
     dropWaitingDeletes(): {
         ranges: number;