@open-artifact/whiteboard-core 0.1.0 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,1091 +1 @@
1
- /**
2
- * The live sync engine: keeps a value (a board) in step with a document
3
- * store, one document per element plus a few board records.
4
- *
5
- * Model, per document path:
6
- * - `server`: the last value the store reported (by listing, a change event,
7
- * or our own acknowledged write);
8
- * - `acked`: our writes acknowledged in the last few seconds, re-applied over
9
- * `server` so a late event about an older write does not flash old data;
10
- * - `inflight`: the change being written;
11
- * - `pending`: the change waiting to be written (debounced, merged).
12
- *
13
- * What the viewer sees is `server ⊕ acked ⊕ inflight ⊕ pending`. Changes are
14
- * JSON merge patches computed against the document as it was before them, so
15
- * two people editing different fields of one element both win; the same
16
- * field: the last write wins.
17
- *
18
- * Local edits are applied at once ({@link LiveStore.commit}), split into
19
- * documents on the next tick, and written after {@link LIVE_EDIT_DEBOUNCE_MS}
20
- * of quiet (at most {@link LIVE_EDIT_MAX_WAIT_MS} after the first edit). Failed
21
- * writes are retried with backoff; a document the store refuses (too large,
22
- * invalid) is parked and the rest keep syncing. Losing write access stops
23
- * syncing and keeps the edits on screen.
24
- *
25
- * Saved copy: with a {@link LiveCache}, the engine keeps the documents the
26
- * store confirmed on the device. When the store does not answer within
27
- * {@link LIVE_SNAPSHOT_TIMEOUT_MS}, or loading fails for a reason other than
28
- * access, the saved copy is shown (`cached`): people can keep editing, and
29
- * their edits wait until the store answers, then sync over what it holds.
30
- *
31
- * The engine is framework-agnostic: {@link LiveStore.getSnapshot} and
32
- * {@link LiveStore.subscribe} fit React's `useSyncExternalStore` or a Zustand
33
- * store.
34
- */
35
- import { applyJsonMergePatch, canonicalValuesEqual, countMergePatchTombstones, createJsonMergePatch, escapedJsonByteLength, isPlainRecord, removeNullMergePatchFields, } from "../json.js";
36
- import { SYSTEM_TIMERS, collectionOfPath, isAccessError, syncErrorCode, syncErrorMessage, } from "./transport.js";
37
- /* -------------------------------------------------------------------------- */
38
- /* Tuning */
39
- /* -------------------------------------------------------------------------- */
40
- /** Quiet time after the last edit of a document before it is written. */
41
- export const LIVE_EDIT_DEBOUNCE_MS = 350;
42
- /** Longest a document waits after its first unsent edit. */
43
- export const LIVE_EDIT_MAX_WAIT_MS = 1500;
44
- /** After this long loading, the saved copy shows (when there is one), else the status says it is still loading. */
45
- export const LIVE_SNAPSHOT_TIMEOUT_MS = 10_000;
46
- /** After this long loading, the load counts as failed and is retried. */
47
- export const LIVE_EXTENDED_LOAD_TIMEOUT_MS = 35_000;
48
- /** Delays between load retries; the last one repeats. */
49
- export const LIVE_RECONNECT_BACKOFF_MS = [2000, 4000, 8000, 15_000, 30_000, 60_000];
50
- /** How long an acknowledged write is re-applied over store events. */
51
- export const ACKNOWLEDGED_WRITE_GRACE_MS = 5000;
52
- /** A write that has not settled after this long is retried. */
53
- export const STUCK_WRITE_TIMEOUT_MS = 60_000;
54
- /** A patch deleting more fields than this is sent as a whole document instead. */
55
- export const MAX_MERGE_PATCH_TOMBSTONES = 64;
56
- /** The first retry after a failed write waits a random time in this range. */
57
- export const INITIAL_WRITE_RETRY_JITTER_MS = [300, 800];
58
- /** Later retries: `min(MAX, BASE · 2^(failures − 2))`. */
59
- export const WRITE_RETRY_BASE_MS = 2000;
60
- export const WRITE_RETRY_MAX_MS = 30_000;
61
- /** Retries when the store says it is overloaded: `min(MAX, BASE · 2^(failures − 1))`. */
62
- export const BACKPRESSURE_RETRY_BASE_MS = 1000;
63
- export const BACKPRESSURE_RETRY_MAX_MS = 8000;
64
- /** Overload lasting this long shows as an error. */
65
- export const BACKPRESSURE_ERROR_AFTER_MS = 30_000;
66
- /** Shortest timer the engine sets. */
67
- export const MIN_SYNC_TIMER_DELAY_MS = 16;
68
- /** Acknowledged writes kept per document, at most. */
69
- export const MAX_UNCOALESCED_ACKNOWLEDGED_WRITES = 8;
70
- /** Write calls in flight at once, at most. */
71
- export const MAX_CONCURRENT_LIVE_WRITES = 25;
72
- /** Documents per write call, at most (the store allows 500). */
73
- export const MAX_WRITES_PER_BATCH = 100;
74
- /** Approximate bytes per write call, at most (one larger document still goes alone). */
75
- export const MAX_BATCH_BYTES = 1_048_576;
76
- /** Quiet time before the saved copy is updated after the store's documents change. */
77
- export const LIVE_CACHE_WRITE_DELAY_MS = 2000;
78
- /* -------------------------------------------------------------------------- */
79
- /* Messages */
80
- /* -------------------------------------------------------------------------- */
81
- /** Status and notice texts (English defaults). */
82
- export const LIVE_MESSAGES = {
83
- loading: 'Loading the latest content…',
84
- loadingSlow: 'Still loading the latest content…',
85
- loadFailed: 'The latest content failed to load. Click to retry.',
86
- cached: 'Showing your saved copy. Reconnecting…',
87
- cachedEdits: 'Showing your saved copy. Your changes will sync when you’re back online.',
88
- cachedReadOnly: 'Showing a saved copy. Reconnecting…',
89
- readOnly: 'Read-only. You’re seeing live changes.',
90
- accessEnded: 'Live sync ended for this view. Reload to continue.',
91
- parkedSuffix: 'Other changes keep syncing.',
92
- refusedHint: 'Shorten it to resume syncing this item.',
93
- retryHint: 'Click to retry.',
94
- unreachableNow: 'The store is unreachable. Retrying now. Your edits are kept on this screen.',
95
- unreachableLater: 'The store is unreachable. Retrying shortly; click to retry now. Your edits are kept on this screen.',
96
- busy: 'The store is busy. Your edits are kept on this screen and will sync when it recovers.',
97
- saving: 'Syncing…',
98
- syncedHint: 'Edits sync as you go; if two people change the same thing, the last change wins.',
99
- itemGone: 'An item you were editing was deleted by someone else.',
100
- cannotStore: (why) => `This change can’t be stored (${why}).`,
101
- waiting: (count) => (count === 1 ? '1 change waiting to sync.' : `${count} changes waiting to sync.`),
102
- syncedAt: (time) => `All changes synced · ${time}.`,
103
- synced: 'Synced.',
104
- };
105
- const DELETE = { kind: 'delete' };
106
- /** Applies a change to a document; a patch on a document someone deleted stays deleted. */
107
- function applyChange(doc, change, removed) {
108
- if (change.kind === 'delete')
109
- return undefined;
110
- if (change.replace)
111
- return removeNullMergePatchFields(change.patch);
112
- if (doc === undefined)
113
- return removed ? undefined : removeNullMergePatchFields(change.patch);
114
- return applyJsonMergePatch(doc, change.patch);
115
- }
116
- /** The change that turns `base` into `target`. */
117
- function changeBetween(base, target, replace) {
118
- if (target === undefined)
119
- return base === undefined ? undefined : DELETE;
120
- if (base === undefined || replace)
121
- return { kind: 'patch', patch: target, replace: true };
122
- const patch = createJsonMergePatch(base, target);
123
- return patch === null ? undefined : { kind: 'patch', patch: patch, replace: false };
124
- }
125
- /**
126
- * Drops the parts of a patch that recurse into an object field the store now
127
- * holds as a non-object: merging them would build a half object.
128
- */
129
- export function dropShapeConflicts(patch, server) {
130
- if (server === undefined)
131
- return patch;
132
- let result;
133
- for (const key of Object.keys(patch)) {
134
- const value = patch[key];
135
- if (!isPlainRecord(value))
136
- continue;
137
- const current = server[key];
138
- let next;
139
- if (current !== undefined && !isPlainRecord(current))
140
- next = undefined;
141
- else if (isPlainRecord(current))
142
- next = dropShapeConflicts(value, current);
143
- else
144
- continue;
145
- if (next === value)
146
- continue;
147
- result ??= { ...patch };
148
- if (next === undefined)
149
- delete result[key];
150
- else
151
- result[key] = next;
152
- }
153
- return result ?? patch;
154
- }
155
- function defaultFormatTime(epochMs) {
156
- return new Date(epochMs).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' });
157
- }
158
- /* -------------------------------------------------------------------------- */
159
- /* Engine */
160
- /* -------------------------------------------------------------------------- */
161
- /** Creates a live store; loading starts at once. */
162
- export function createLiveStore(options) {
163
- const { transport, codec } = options;
164
- const timers = options.timers ?? SYSTEM_TIMERS;
165
- const formatTime = options.formatTime ?? defaultFormatTime;
166
- const collections = new Set(codec.collections);
167
- const collectionRank = new Map(codec.collections.map((name, index) => [name, index]));
168
- const entries = new Map();
169
- const listeners = new Set();
170
- let local = new Map();
171
- const bytes = new Map();
172
- let totalBytes = 0;
173
- let carry;
174
- let value = options.initial;
175
- let unmaterialized = false;
176
- let materializeTimer;
177
- let loadState = 'loading';
178
- let loadAttempt = 0;
179
- let loadFailures = 0;
180
- let loadSlow = false;
181
- let loadTimer;
182
- let slowTimer;
183
- let retryLoadTimer;
184
- let buffered = [];
185
- let cacheTimer;
186
- let readOnly = options.readOnly === true;
187
- let stopped = false;
188
- let disposed = false;
189
- let seqCounter = 0;
190
- let flights = new Set();
191
- let syncTimer;
192
- let syncTimerAt = Infinity;
193
- let urgent = false;
194
- let backpressureSince;
195
- let lastSyncedAt;
196
- let lastError;
197
- let notice;
198
- let noticeCounter = 0;
199
- let flushWaiters = [];
200
- let snapshot = buildSnapshot();
201
- /* ---------------------------------------------------------------- entries */
202
- function entryFor(path) {
203
- let entry = entries.get(path);
204
- if (entry === undefined) {
205
- entry = {
206
- path,
207
- server: undefined,
208
- removedSeen: false,
209
- inflight: undefined,
210
- pending: undefined,
211
- acked: [],
212
- firstAt: 0,
213
- lastAt: 0,
214
- failures: 0,
215
- notBefore: 0,
216
- seq: 0,
217
- parked: undefined,
218
- solo: false,
219
- goneNoticed: false,
220
- };
221
- entries.set(path, entry);
222
- }
223
- return entry;
224
- }
225
- /** Forgets entries that hold nothing. */
226
- function sweep(entry) {
227
- if (entry.server === undefined && !entry.removedSeen && entry.inflight === undefined && entry.pending === undefined && entry.acked.length === 0) {
228
- entries.delete(entry.path);
229
- }
230
- }
231
- /** The document before `pending` (or with it). */
232
- function viewOf(entry, withPending) {
233
- let doc = entry.server;
234
- for (const acked of entry.acked)
235
- doc = applyChange(doc, acked.change, entry.removedSeen);
236
- if (entry.inflight !== undefined)
237
- doc = applyChange(doc, entry.inflight, entry.removedSeen);
238
- if (withPending && entry.pending !== undefined)
239
- doc = applyChange(doc, entry.pending, entry.removedSeen);
240
- return doc;
241
- }
242
- function setBytes(path, doc) {
243
- totalBytes -= bytes.get(path) ?? 0;
244
- if (doc === undefined)
245
- bytes.delete(path);
246
- else {
247
- const size = escapedJsonByteLength(doc);
248
- bytes.set(path, size);
249
- totalBytes += size;
250
- }
251
- }
252
- function setPending(entry, change, now) {
253
- if (change === undefined) {
254
- entry.pending = undefined;
255
- entry.firstAt = 0;
256
- if (entry.parked?.refused === true)
257
- entry.parked = undefined;
258
- return;
259
- }
260
- if (entry.pending === undefined) {
261
- entry.firstAt = now;
262
- entry.seq = ++seqCounter;
263
- }
264
- entry.pending = change;
265
- entry.lastAt = now;
266
- }
267
- function showNotice(message) {
268
- notice = { id: ++noticeCounter, message };
269
- }
270
- /** The doc was deleted by someone else while we had edits for it. */
271
- function itemGone(entry) {
272
- entry.pending = undefined;
273
- entry.firstAt = 0;
274
- entry.parked = undefined;
275
- if (!entry.goneNoticed) {
276
- entry.goneNoticed = true;
277
- showNotice(LIVE_MESSAGES.itemGone);
278
- }
279
- }
280
- /* ------------------------------------------------------------ local edits */
281
- function canEdit() {
282
- return (loadState === 'live' || loadState === 'cached') && !readOnly && !stopped && !disposed;
283
- }
284
- function commit(next) {
285
- if (!canEdit())
286
- return false;
287
- if (next === value)
288
- return true;
289
- const previous = value;
290
- value = next;
291
- unmaterialized = true;
292
- if (materializeTimer === undefined)
293
- materializeTimer = timers.setTimeout(materialize, MIN_SYNC_TIMER_DELAY_MS);
294
- emit({ remote: false, valueChanged: true, previous });
295
- return true;
296
- }
297
- /** Splits the current value into documents and stages what changed. */
298
- function materialize() {
299
- if (materializeTimer !== undefined) {
300
- timers.clearTimeout(materializeTimer);
301
- materializeTimer = undefined;
302
- }
303
- if (!unmaterialized)
304
- return;
305
- unmaterialized = false;
306
- const now = timers.now();
307
- const next = codec.decompose(value, local, carry);
308
- const touched = [];
309
- for (const [path, doc] of next) {
310
- const previous = local.get(path);
311
- if (previous === doc)
312
- continue;
313
- if (previous !== undefined && canonicalValuesEqual(previous, doc)) {
314
- next.set(path, previous);
315
- continue;
316
- }
317
- stage(path, doc, now);
318
- touched.push(path);
319
- }
320
- for (const path of local.keys()) {
321
- if (next.has(path))
322
- continue;
323
- stage(path, undefined, now);
324
- touched.push(path);
325
- }
326
- local = next;
327
- if (touched.length > 0) {
328
- schedule();
329
- emitStatus();
330
- }
331
- }
332
- /** Stages the local value of one document. */
333
- function stage(path, doc, now) {
334
- const entry = entryFor(path);
335
- const base = viewOf(entry, false);
336
- let change;
337
- if (doc === undefined)
338
- change = base === undefined ? undefined : DELETE;
339
- else if (base === undefined || codec.replaceOnly(path))
340
- change = canonicalValuesEqual(base, doc) ? undefined : { kind: 'patch', patch: doc, replace: true };
341
- else
342
- change = changeBetween(base, doc, false);
343
- setPending(entry, change, now);
344
- const previousBytes = bytes.get(path) ?? 0;
345
- setBytes(path, doc);
346
- if (doc === undefined || change === undefined) {
347
- if (entry.parked?.refused === true)
348
- entry.parked = undefined;
349
- }
350
- else {
351
- const size = bytes.get(path) ?? 0;
352
- const reason = codec.refuse(path, doc, size > previousBytes ? totalBytes : 0);
353
- if (reason !== undefined) {
354
- entry.parked = { message: reason, refused: true };
355
- lastError = { code: 'refused', message: reason, path };
356
- }
357
- else if (entry.parked !== undefined) {
358
- entry.parked = undefined;
359
- entry.failures = 0;
360
- }
361
- }
362
- sweep(entry);
363
- }
364
- /* ----------------------------------------------------------- remote edits */
365
- function onRemote(event) {
366
- if (disposed)
367
- return;
368
- if (buffered !== undefined) {
369
- buffered.push(event);
370
- return;
371
- }
372
- applyRemote(event);
373
- }
374
- function applyRemote(event) {
375
- materialize();
376
- const touched = [];
377
- for (const [path, data] of Object.entries(event.changes)) {
378
- if (!collections.has(collectionOfPath(path)))
379
- continue;
380
- const entry = entryFor(path);
381
- if (data === null) {
382
- entry.server = undefined;
383
- entry.removedSeen = true;
384
- const pending = entry.pending;
385
- if (pending !== undefined && pending.kind === 'patch' && !pending.replace && entry.inflight === undefined)
386
- itemGone(entry);
387
- }
388
- else {
389
- entry.server = data;
390
- entry.removedSeen = false;
391
- entry.goneNoticed = false;
392
- const pending = entry.pending;
393
- if (pending !== undefined && pending.kind === 'patch' && !pending.replace) {
394
- const patch = dropShapeConflicts(pending.patch, viewOf(entry, false));
395
- if (patch !== pending.patch)
396
- setPending(entry, Object.keys(patch).length === 0 ? undefined : { kind: 'patch', patch, replace: false }, entry.lastAt);
397
- }
398
- }
399
- touched.push(path);
400
- }
401
- if (touched.length > 0)
402
- refresh(touched, true);
403
- }
404
- /** Recomputes the local documents of some paths and recomposes the value. */
405
- function refresh(paths, remote) {
406
- let changed = false;
407
- for (const path of paths) {
408
- const entry = entries.get(path);
409
- const doc = entry === undefined ? undefined : viewOf(entry, true);
410
- const previous = local.get(path);
411
- if (doc === undefined) {
412
- if (previous === undefined)
413
- continue;
414
- local.delete(path);
415
- }
416
- else {
417
- if (previous !== undefined && (previous === doc || canonicalValuesEqual(previous, doc)))
418
- continue;
419
- local.set(path, doc);
420
- }
421
- setBytes(path, doc);
422
- changed = true;
423
- if (entry !== undefined)
424
- sweep(entry);
425
- }
426
- if (!changed) {
427
- emitStatus();
428
- return;
429
- }
430
- recompose(remote);
431
- }
432
- function recompose(remote) {
433
- const previous = value;
434
- const composed = codec.compose(local);
435
- carry = composed.carry;
436
- value = codec.share !== undefined ? codec.share(previous, composed.value) : composed.value;
437
- emit({ remote, valueChanged: value !== previous, previous });
438
- }
439
- /* ------------------------------------------------------------------ load */
440
- const unsubscribe = transport.onChange(onRemote);
441
- /** Whether a load is still waiting for the store (with or without the saved copy shown). */
442
- function awaitingStore() {
443
- return loadState === 'loading' || loadState === 'cached';
444
- }
445
- function load() {
446
- if (disposed)
447
- return;
448
- const attempt = ++loadAttempt;
449
- // The saved copy stays up (and editable) while loading again.
450
- if (loadState !== 'cached')
451
- loadState = 'loading';
452
- loadSlow = false;
453
- buffered = [];
454
- clearLoadTimers();
455
- slowTimer = timers.setTimeout(() => {
456
- if (attempt !== loadAttempt || !awaitingStore())
457
- return;
458
- loadSlow = true;
459
- if (!showSavedCopy())
460
- emitStatus();
461
- }, LIVE_SNAPSHOT_TIMEOUT_MS);
462
- loadTimer = timers.setTimeout(() => {
463
- if (attempt !== loadAttempt || !awaitingStore())
464
- return;
465
- loadFailed(attempt, { code: 'unavailable', message: 'timed out' });
466
- }, LIVE_EXTENDED_LOAD_TIMEOUT_MS);
467
- emitStatus();
468
- Promise.all(codec.collections.map((collection) => transport.list(collection))).then((lists) => {
469
- if (attempt !== loadAttempt || disposed)
470
- return;
471
- clearLoadTimers();
472
- const seen = new Set();
473
- for (const list of lists) {
474
- for (const document of list) {
475
- if (!collections.has(collectionOfPath(document.path)))
476
- continue;
477
- const entry = entryFor(document.path);
478
- entry.server = document.data;
479
- entry.removedSeen = false;
480
- seen.add(document.path);
481
- }
482
- }
483
- for (const entry of entries.values()) {
484
- if (!seen.has(entry.path) && entry.server !== undefined) {
485
- entry.server = undefined;
486
- entry.removedSeen = true;
487
- // Edited on the saved copy, deleted in the store meanwhile: the edit goes.
488
- const pending = entry.pending;
489
- if (pending !== undefined && pending.kind === 'patch' && !pending.replace && entry.inflight === undefined)
490
- itemGone(entry);
491
- }
492
- }
493
- const events = buffered ?? [];
494
- buffered = undefined;
495
- loadState = 'live';
496
- loadFailures = 0;
497
- lastError = undefined;
498
- for (const event of events) {
499
- for (const [path, data] of Object.entries(event.changes)) {
500
- if (!collections.has(collectionOfPath(path)))
501
- continue;
502
- const entry = entryFor(path);
503
- entry.server = data ?? undefined;
504
- entry.removedSeen = data === null;
505
- }
506
- }
507
- local = new Map();
508
- bytes.clear();
509
- totalBytes = 0;
510
- for (const entry of entries.values()) {
511
- const doc = viewOf(entry, true);
512
- if (doc === undefined)
513
- continue;
514
- local.set(entry.path, doc);
515
- setBytes(entry.path, doc);
516
- }
517
- recompose(true);
518
- schedule();
519
- saveCopySoon();
520
- }, (error) => {
521
- if (attempt !== loadAttempt || disposed)
522
- return;
523
- loadFailed(attempt, { code: syncErrorCode(error), message: syncErrorMessage(error) });
524
- });
525
- }
526
- function loadFailed(attempt, error) {
527
- if (attempt !== loadAttempt)
528
- return;
529
- loadAttempt += 1;
530
- clearLoadTimers();
531
- buffered = [];
532
- lastError = error;
533
- const code = error.code;
534
- const retrying = code !== 'refused' && !isAccessError(code);
535
- // Out of reach (not refused): show the saved copy, if there is one, while retrying.
536
- if (!retrying || !showSavedCopy())
537
- loadState = 'failed';
538
- if (retrying) {
539
- const delay = LIVE_RECONNECT_BACKOFF_MS[Math.min(loadFailures, LIVE_RECONNECT_BACKOFF_MS.length - 1)] ?? 60_000;
540
- loadFailures += 1;
541
- retryLoadTimer = timers.setTimeout(() => {
542
- retryLoadTimer = undefined;
543
- if (loadState === 'failed' || loadState === 'cached')
544
- load();
545
- }, delay);
546
- }
547
- emitStatus();
548
- }
549
- /**
550
- * Shows the saved copy while the store is out of reach. `true` when it is
551
- * shown (now or already); `false` without one.
552
- */
553
- function showSavedCopy() {
554
- if (loadState === 'cached')
555
- return true;
556
- if (loadState === 'live' || disposed)
557
- return false;
558
- const saved = readSavedCopy();
559
- if (saved === undefined)
560
- return false;
561
- loadState = 'cached';
562
- for (const document of saved) {
563
- if (!collections.has(collectionOfPath(document.path)))
564
- continue;
565
- const entry = entryFor(document.path);
566
- entry.server = document.data;
567
- entry.removedSeen = false;
568
- }
569
- local = new Map();
570
- bytes.clear();
571
- totalBytes = 0;
572
- for (const entry of entries.values()) {
573
- const doc = viewOf(entry, true);
574
- if (doc === undefined)
575
- continue;
576
- local.set(entry.path, doc);
577
- setBytes(entry.path, doc);
578
- }
579
- recompose(true);
580
- return true;
581
- }
582
- function readSavedCopy() {
583
- try {
584
- return options.cache?.read();
585
- }
586
- catch {
587
- return undefined;
588
- }
589
- }
590
- /** The store's documents with our acknowledged writes: what the saved copy holds. */
591
- function confirmedDocuments() {
592
- const documents = [];
593
- for (const entry of entries.values()) {
594
- let doc = entry.server;
595
- for (const acked of entry.acked)
596
- doc = applyChange(doc, acked.change, entry.removedSeen);
597
- if (doc !== undefined)
598
- documents.push({ path: entry.path, data: doc });
599
- }
600
- return documents;
601
- }
602
- /** Updates the saved copy after {@link LIVE_CACHE_WRITE_DELAY_MS} (live only). */
603
- function saveCopySoon() {
604
- if (options.cache === undefined || loadState !== 'live' || disposed || cacheTimer !== undefined)
605
- return;
606
- cacheTimer = timers.setTimeout(() => {
607
- cacheTimer = undefined;
608
- saveCopyNow();
609
- }, LIVE_CACHE_WRITE_DELAY_MS);
610
- }
611
- function saveCopyNow() {
612
- if (options.cache === undefined || loadState !== 'live')
613
- return;
614
- try {
615
- options.cache.write(confirmedDocuments());
616
- }
617
- catch {
618
- // A full or disabled storage only loses the saved copy.
619
- }
620
- }
621
- function clearLoadTimers() {
622
- for (const handle of [loadTimer, slowTimer, retryLoadTimer])
623
- if (handle !== undefined)
624
- timers.clearTimeout(handle);
625
- loadTimer = slowTimer = retryLoadTimer = undefined;
626
- }
627
- /* ---------------------------------------------------------------- writes */
628
- function sendable(entry) {
629
- return entry.pending !== undefined && entry.inflight === undefined && entry.parked === undefined;
630
- }
631
- function dueAt(entry) {
632
- const debounced = urgent ? 0 : Math.min(entry.lastAt + LIVE_EDIT_DEBOUNCE_MS, entry.firstAt + LIVE_EDIT_MAX_WAIT_MS);
633
- return Math.max(debounced, entry.notBefore);
634
- }
635
- function writing() {
636
- return loadState === 'live' && !readOnly && !stopped && !disposed;
637
- }
638
- /** Sets the one sync timer for the soonest due work. */
639
- function schedule() {
640
- if (disposed)
641
- return;
642
- let soonest = Infinity;
643
- if (writing())
644
- for (const entry of entries.values())
645
- if (sendable(entry))
646
- soonest = Math.min(soonest, dueAt(entry));
647
- for (const entry of entries.values()) {
648
- const oldest = entry.acked[0];
649
- if (oldest !== undefined)
650
- soonest = Math.min(soonest, oldest.at + ACKNOWLEDGED_WRITE_GRACE_MS);
651
- }
652
- if (soonest === Infinity) {
653
- if (syncTimer !== undefined)
654
- timers.clearTimeout(syncTimer);
655
- syncTimer = undefined;
656
- syncTimerAt = Infinity;
657
- return;
658
- }
659
- const now = timers.now();
660
- const at = Math.max(soonest, now + (urgent ? 0 : MIN_SYNC_TIMER_DELAY_MS));
661
- if (syncTimer !== undefined && syncTimerAt <= at)
662
- return;
663
- if (syncTimer !== undefined)
664
- timers.clearTimeout(syncTimer);
665
- syncTimerAt = at;
666
- syncTimer = timers.setTimeout(() => {
667
- syncTimer = undefined;
668
- syncTimerAt = Infinity;
669
- pump();
670
- }, at - now);
671
- }
672
- /** Drops acknowledged writes past their grace period. */
673
- function pruneAcked(now) {
674
- const touched = [];
675
- for (const entry of entries.values()) {
676
- const before = entry.acked.length;
677
- if (before === 0)
678
- continue;
679
- entry.acked = entry.acked.filter((acked) => now - acked.at < ACKNOWLEDGED_WRITE_GRACE_MS).slice(-MAX_UNCOALESCED_ACKNOWLEDGED_WRITES);
680
- if (entry.acked.length !== before)
681
- touched.push(entry.path);
682
- }
683
- if (touched.length > 0)
684
- refresh(touched, true);
685
- }
686
- /** Sends what is due. */
687
- function pump() {
688
- if (disposed)
689
- return;
690
- materialize();
691
- const now = timers.now();
692
- pruneAcked(now);
693
- if (writing()) {
694
- const ready = [...entries.values()].filter((entry) => sendable(entry) && dueAt(entry) <= now);
695
- ready.sort(compareEntries);
696
- while (ready.length > 0 && flights.size < MAX_CONCURRENT_LIVE_WRITES) {
697
- const batch = [];
698
- let batchBytes = 0;
699
- while (ready.length > 0 && batch.length < MAX_WRITES_PER_BATCH) {
700
- const entry = ready[0];
701
- const size = bytes.get(entry.path) ?? 256;
702
- if (entry.solo && batch.length > 0)
703
- break;
704
- if (batch.length > 0 && batchBytes + size > MAX_BATCH_BYTES)
705
- break;
706
- ready.shift();
707
- batch.push(entry);
708
- batchBytes += size;
709
- if (entry.solo)
710
- break;
711
- }
712
- send(batch);
713
- }
714
- }
715
- schedule();
716
- emitStatus();
717
- }
718
- /** Write order: changes before deletes, collections in order (deletes reversed), meta last, then age. */
719
- function compareEntries(a, b) {
720
- const rank = (entry) => {
721
- const deleting = entry.pending?.kind === 'delete';
722
- const collection = collectionRank.get(collectionOfPath(entry.path)) ?? codec.collections.length;
723
- const ordered = deleting ? codec.collections.length - collection : collection;
724
- return (deleting ? 1000 : 0) + ordered + (entry.path === codec.metaPath ? 0.5 : 0);
725
- };
726
- return rank(a) - rank(b) || a.seq - b.seq;
727
- }
728
- function send(batch) {
729
- const writes = [];
730
- const sent = [];
731
- for (const entry of batch) {
732
- const change = entry.pending;
733
- if (change === undefined)
734
- continue;
735
- let write;
736
- let inflight = change;
737
- if (change.kind === 'delete')
738
- write = { op: 'delete', path: entry.path };
739
- else if (change.replace)
740
- write = { op: 'set', path: entry.path, data: removeNullMergePatchFields(change.patch) };
741
- else {
742
- const base = viewOf(entry, false);
743
- if (base === undefined || (entry.server === undefined && entry.removedSeen)) {
744
- itemGone(entry);
745
- refresh([entry.path], true);
746
- continue;
747
- }
748
- if (countMergePatchTombstones(change.patch) > MAX_MERGE_PATCH_TOMBSTONES) {
749
- const data = applyJsonMergePatch(base, change.patch);
750
- write = { op: 'set', path: entry.path, data };
751
- inflight = { kind: 'patch', patch: data, replace: true };
752
- }
753
- else
754
- write = { op: 'update', path: entry.path, patch: change.patch };
755
- }
756
- entry.inflight = inflight;
757
- entry.pending = undefined;
758
- entry.firstAt = 0;
759
- writes.push(write);
760
- sent.push(entry);
761
- }
762
- if (writes.length === 0)
763
- return;
764
- const flight = { entries: sent, writes, settled: false, timer: undefined };
765
- flights.add(flight);
766
- flight.timer = timers.setTimeout(() => settle(flight, { code: 'unavailable', error: new Error('the write timed out') }), STUCK_WRITE_TIMEOUT_MS);
767
- let request;
768
- try {
769
- request = transport.write(writes);
770
- }
771
- catch (error) {
772
- request = Promise.reject(error);
773
- }
774
- request.then(() => settle(flight, undefined), (error) => settle(flight, { code: syncErrorCode(error), error }));
775
- }
776
- /** Puts an in-flight change back in front of the pending one. */
777
- function restorePending(entry, now) {
778
- const inflight = entry.inflight;
779
- entry.inflight = undefined;
780
- if (inflight === undefined)
781
- return;
782
- if (entry.pending === undefined) {
783
- entry.pending = inflight;
784
- entry.firstAt = now;
785
- entry.lastAt = now;
786
- entry.seq = ++seqCounter;
787
- return;
788
- }
789
- const base = viewOf(entry, false);
790
- const target = applyChange(applyChange(base, inflight, entry.removedSeen), entry.pending, entry.removedSeen);
791
- const replace = (inflight.kind === 'patch' && inflight.replace) || (entry.pending.kind === 'patch' && entry.pending.replace) || inflight.kind === 'delete';
792
- entry.pending = changeBetween(base, target, replace);
793
- if (entry.pending === undefined)
794
- entry.firstAt = 0;
795
- }
796
- function settle(flight, failure) {
797
- if (flight.settled)
798
- return;
799
- flight.settled = true;
800
- if (flight.timer !== undefined)
801
- timers.clearTimeout(flight.timer);
802
- flights.delete(flight);
803
- if (disposed)
804
- return;
805
- const now = timers.now();
806
- if (failure === undefined) {
807
- backpressureSince = undefined;
808
- lastSyncedAt = now;
809
- if (lastError !== undefined && lastError.code !== 'refused')
810
- lastError = undefined;
811
- flight.writes.forEach((write, index) => {
812
- const entry = flight.entries[index];
813
- const change = entry.inflight;
814
- entry.inflight = undefined;
815
- entry.failures = 0;
816
- entry.solo = false;
817
- if (change === undefined)
818
- return;
819
- if (write.op === 'delete') {
820
- entry.server = undefined;
821
- entry.removedSeen = false;
822
- }
823
- else if (write.op === 'set') {
824
- entry.server = write.data;
825
- entry.removedSeen = false;
826
- }
827
- else if (entry.server === undefined && entry.removedSeen) {
828
- // Deleted while the update flew: the store may now hold a stub; remove it.
829
- itemGone(entry);
830
- setPending(entry, DELETE, now);
831
- entry.notBefore = 0;
832
- }
833
- else
834
- entry.server = applyJsonMergePatch(entry.server, write.patch);
835
- entry.acked.push({ change, at: now });
836
- if (entry.acked.length > MAX_UNCOALESCED_ACKNOWLEDGED_WRITES)
837
- entry.acked.shift();
838
- });
839
- refresh(flight.entries.map((entry) => entry.path), false);
840
- }
841
- else
842
- handleFailure(flight, failure.code, failure.error, now);
843
- if (urgent && flushIdle())
844
- urgent = false;
845
- pump();
846
- resolveFlushWaiters();
847
- }
848
- function handleFailure(flight, code, error, now) {
849
- const why = syncErrorMessage(error);
850
- lastError = { code, message: why, path: flight.entries.length === 1 ? flight.entries[0].path : undefined };
851
- if (isAccessError(code)) {
852
- for (const entry of flight.entries)
853
- restorePending(entry, now);
854
- stopped = true;
855
- return;
856
- }
857
- if (code === 'invalid' || code === 'not_found') {
858
- if (flight.entries.length > 1) {
859
- for (const entry of flight.entries) {
860
- restorePending(entry, now);
861
- entry.solo = true;
862
- entry.notBefore = 0;
863
- entry.firstAt = entry.lastAt = 0;
864
- }
865
- return;
866
- }
867
- const entry = flight.entries[0];
868
- const write = flight.writes[0];
869
- if (write.op === 'update' && (code === 'not_found' || (entry.server === undefined && entry.removedSeen))) {
870
- entry.inflight = undefined;
871
- itemGone(entry);
872
- refresh([entry.path], true);
873
- return;
874
- }
875
- restorePending(entry, now);
876
- entry.parked = { message: LIVE_MESSAGES.cannotStore(why), refused: false };
877
- return;
878
- }
879
- const busy = code === 'resource_exhausted';
880
- if (busy)
881
- backpressureSince ??= now;
882
- for (const entry of flight.entries) {
883
- restorePending(entry, now);
884
- entry.failures += 1;
885
- const failures = entry.failures;
886
- const delay = busy
887
- ? Math.min(BACKPRESSURE_RETRY_MAX_MS, BACKPRESSURE_RETRY_BASE_MS * 2 ** (failures - 1))
888
- : failures === 1
889
- ? INITIAL_WRITE_RETRY_JITTER_MS[0] + timers.random() * (INITIAL_WRITE_RETRY_JITTER_MS[1] - INITIAL_WRITE_RETRY_JITTER_MS[0])
890
- : Math.min(WRITE_RETRY_MAX_MS, WRITE_RETRY_BASE_MS * 2 ** Math.max(0, failures - 2));
891
- entry.notBefore = now + delay;
892
- }
893
- }
894
- /* ----------------------------------------------------------------- flush */
895
- function flushIdle() {
896
- if (flights.size > 0)
897
- return false;
898
- if (!writing())
899
- return true;
900
- const now = timers.now();
901
- for (const entry of entries.values())
902
- if (sendable(entry) && entry.failures === 0 && entry.notBefore <= now)
903
- return false;
904
- return !unmaterialized;
905
- }
906
- function resolveFlushWaiters() {
907
- if (flushWaiters.length === 0 || !flushIdle())
908
- return;
909
- const waiters = flushWaiters;
910
- flushWaiters = [];
911
- for (const resolve of waiters)
912
- resolve();
913
- }
914
- function flush() {
915
- materialize();
916
- if (flushIdle())
917
- return Promise.resolve();
918
- urgent = true;
919
- for (const entry of entries.values())
920
- if (sendable(entry))
921
- entry.notBefore = 0;
922
- const done = new Promise((resolve) => flushWaiters.push(resolve));
923
- pump();
924
- resolveFlushWaiters();
925
- return done;
926
- }
927
- function retry() {
928
- if (disposed)
929
- return;
930
- if (loadState === 'failed' || loadState === 'cached') {
931
- loadFailures = 0;
932
- load();
933
- return;
934
- }
935
- for (const entry of entries.values()) {
936
- if (entry.parked !== undefined && !entry.parked.refused)
937
- entry.parked = undefined;
938
- entry.notBefore = 0;
939
- }
940
- pump();
941
- }
942
- /* --------------------------------------------------------------- status */
943
- function pendingCount() {
944
- let count = 0;
945
- for (const entry of entries.values())
946
- if (entry.pending !== undefined || entry.inflight !== undefined)
947
- count += 1;
948
- return count + (unmaterialized ? 1 : 0);
949
- }
950
- function buildSnapshot() {
951
- const pending = pendingCount();
952
- let refusal;
953
- for (const entry of entries.values())
954
- if (entry.parked?.refused === true && entry.pending !== undefined)
955
- refusal ??= entry.parked.message;
956
- const base = {
957
- value,
958
- loadState,
959
- readOnly: readOnly || stopped,
960
- pending,
961
- flights: flights.size,
962
- ...(refusal !== undefined ? { refusal } : {}),
963
- ...(notice !== undefined ? { notice } : {}),
964
- ...(lastSyncedAt !== undefined ? { lastSyncedAt } : {}),
965
- ...(lastError !== undefined ? { lastError } : {}),
966
- };
967
- if (loadState === 'loading')
968
- return { ...base, status: 'loading', message: loadSlow ? LIVE_MESSAGES.loadingSlow : LIVE_MESSAGES.loading, retryable: false };
969
- if (loadState === 'failed')
970
- return { ...base, status: 'error', message: LIVE_MESSAGES.loadFailed, retryable: lastError === undefined || !isAccessError(lastError.code) };
971
- if (loadState === 'cached') {
972
- const message = readOnly || stopped ? LIVE_MESSAGES.cachedReadOnly : pending > 0 ? LIVE_MESSAGES.cachedEdits : LIVE_MESSAGES.cached;
973
- return { ...base, status: readOnly || stopped ? 'readonly' : 'cached', message, retryable: true };
974
- }
975
- if (stopped)
976
- return { ...base, status: 'error', message: LIVE_MESSAGES.accessEnded, retryable: false };
977
- if (readOnly)
978
- return { ...base, status: 'readonly', message: LIVE_MESSAGES.readOnly, retryable: false };
979
- let parked;
980
- let failing = false;
981
- for (const entry of entries.values()) {
982
- if (entry.parked !== undefined && entry.pending !== undefined)
983
- parked ??= entry.parked;
984
- if (entry.failures > 0 && (entry.pending !== undefined || entry.inflight !== undefined))
985
- failing = true;
986
- }
987
- if (parked !== undefined) {
988
- return {
989
- ...base,
990
- status: 'error',
991
- message: `${parked.message} ${LIVE_MESSAGES.parkedSuffix}`,
992
- hint: parked.refused ? LIVE_MESSAGES.refusedHint : LIVE_MESSAGES.retryHint,
993
- retryable: !parked.refused,
994
- };
995
- }
996
- if (backpressureSince !== undefined && failing) {
997
- const long = timers.now() - backpressureSince >= BACKPRESSURE_ERROR_AFTER_MS;
998
- return { ...base, status: long ? 'error' : 'offline', message: LIVE_MESSAGES.busy, retryable: true };
999
- }
1000
- if (failing)
1001
- return { ...base, status: 'offline', message: base.flights > 0 ? LIVE_MESSAGES.unreachableNow : LIVE_MESSAGES.unreachableLater, retryable: true };
1002
- if (base.flights > 0)
1003
- return { ...base, status: 'saving', message: LIVE_MESSAGES.saving, retryable: false };
1004
- if (pending > 0)
1005
- return { ...base, status: 'dirty', message: LIVE_MESSAGES.waiting(pending), retryable: false };
1006
- return {
1007
- ...base,
1008
- status: 'ready',
1009
- message: lastSyncedAt !== undefined ? LIVE_MESSAGES.syncedAt(formatTime(lastSyncedAt)) : LIVE_MESSAGES.synced,
1010
- hint: LIVE_MESSAGES.syncedHint,
1011
- retryable: false,
1012
- };
1013
- }
1014
- function sameSnapshot(a, b) {
1015
- return (a.value === b.value &&
1016
- a.loadState === b.loadState &&
1017
- a.status === b.status &&
1018
- a.message === b.message &&
1019
- a.hint === b.hint &&
1020
- a.retryable === b.retryable &&
1021
- a.readOnly === b.readOnly &&
1022
- a.pending === b.pending &&
1023
- a.flights === b.flights &&
1024
- a.refusal === b.refusal &&
1025
- a.notice === b.notice &&
1026
- a.lastSyncedAt === b.lastSyncedAt &&
1027
- a.lastError === b.lastError);
1028
- }
1029
- function emit(change) {
1030
- saveCopySoon();
1031
- const next = buildSnapshot();
1032
- if (!change.valueChanged && sameSnapshot(snapshot, next))
1033
- return;
1034
- snapshot = next;
1035
- for (const listener of [...listeners])
1036
- listener(change);
1037
- }
1038
- function emitStatus() {
1039
- emit({ remote: false, valueChanged: false, previous: value });
1040
- }
1041
- /* ------------------------------------------------------------------ API */
1042
- load();
1043
- return {
1044
- getSnapshot: () => snapshot,
1045
- value: () => value,
1046
- subscribe(listener) {
1047
- listeners.add(listener);
1048
- return () => void listeners.delete(listener);
1049
- },
1050
- commit,
1051
- canEdit,
1052
- flush,
1053
- retry,
1054
- setReadOnly(next) {
1055
- if (readOnly === next)
1056
- return;
1057
- readOnly = next;
1058
- if (!readOnly)
1059
- schedule();
1060
- emitStatus();
1061
- },
1062
- documentBytes: () => {
1063
- materialize();
1064
- return totalBytes;
1065
- },
1066
- dispose() {
1067
- if (disposed)
1068
- return;
1069
- disposed = true;
1070
- unsubscribe();
1071
- clearLoadTimers();
1072
- if (cacheTimer !== undefined) {
1073
- timers.clearTimeout(cacheTimer);
1074
- cacheTimer = undefined;
1075
- saveCopyNow();
1076
- }
1077
- for (const handle of [syncTimer, materializeTimer])
1078
- if (handle !== undefined)
1079
- timers.clearTimeout(handle);
1080
- for (const flight of flights)
1081
- if (flight.timer !== undefined)
1082
- timers.clearTimeout(flight.timer);
1083
- flights = new Set();
1084
- listeners.clear();
1085
- const waiters = flushWaiters;
1086
- flushWaiters = [];
1087
- for (const resolve of waiters)
1088
- resolve();
1089
- },
1090
- };
1091
- }
1
+ export * from '@open-artifact/live-store/engine';