@agentshouse/mdruntime 0.1.0-c2c52158b7381 → 0.1.0-c2e8a6f99866d

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/AGENTS.md +14 -10
  2. package/dist/adapter.d.ts +14 -5
  3. package/dist/adapter.d.ts.map +1 -1
  4. package/dist/bootstrap.d.ts +13 -6
  5. package/dist/bootstrap.d.ts.map +1 -1
  6. package/dist/bootstrap.js +86 -88
  7. package/dist/chain.d.ts +11 -0
  8. package/dist/chain.d.ts.map +1 -0
  9. package/dist/chain.js +64 -0
  10. package/dist/change-set.d.ts +5 -9
  11. package/dist/change-set.d.ts.map +1 -1
  12. package/dist/change-set.js +10 -86
  13. package/dist/chat/list.d.ts.map +1 -1
  14. package/dist/chat/list.js +1 -1
  15. package/dist/chat/message.d.ts.map +1 -1
  16. package/dist/chat/message.js +8 -1
  17. package/dist/chat/messages.d.ts +1 -0
  18. package/dist/chat/messages.d.ts.map +1 -1
  19. package/dist/chat/messages.js +9 -1
  20. package/dist/chat/provider.d.ts +2 -0
  21. package/dist/chat/provider.d.ts.map +1 -1
  22. package/dist/chat/provider.js +17 -1
  23. package/dist/client.d.ts +1 -0
  24. package/dist/client.d.ts.map +1 -1
  25. package/dist/client.js +1 -0
  26. package/dist/collection.d.ts +5 -19
  27. package/dist/collection.d.ts.map +1 -1
  28. package/dist/collection.js +22 -360
  29. package/dist/compile.d.ts +11 -0
  30. package/dist/compile.d.ts.map +1 -0
  31. package/dist/compile.js +712 -0
  32. package/dist/connection.d.ts +2 -2
  33. package/dist/connection.d.ts.map +1 -1
  34. package/dist/connection.js +1396 -624
  35. package/dist/conversation.d.ts +1 -1
  36. package/dist/conversation.d.ts.map +1 -1
  37. package/dist/conversation.js +26 -17
  38. package/dist/entry.d.ts +2 -1
  39. package/dist/entry.d.ts.map +1 -1
  40. package/dist/entry.js +17 -23
  41. package/dist/index.d.ts +1 -1
  42. package/dist/index.d.ts.map +1 -1
  43. package/dist/live.d.ts +22 -0
  44. package/dist/live.d.ts.map +1 -0
  45. package/dist/live.js +123 -0
  46. package/dist/memory.d.ts +1 -0
  47. package/dist/memory.d.ts.map +1 -1
  48. package/dist/memory.js +12 -0
  49. package/dist/observation.d.ts +12 -0
  50. package/dist/observation.d.ts.map +1 -1
  51. package/dist/observation.js +84 -0
  52. package/dist/overlay.d.ts +0 -3
  53. package/dist/overlay.d.ts.map +1 -1
  54. package/dist/overlay.js +1 -151
  55. package/dist/reach.d.ts.map +1 -1
  56. package/dist/reach.js +15 -35
  57. package/dist/reads.js +1 -1
  58. package/dist/refresh.d.ts +10 -13
  59. package/dist/refresh.d.ts.map +1 -1
  60. package/dist/refresh.js +74 -70
  61. package/dist/replica.d.ts +55 -38
  62. package/dist/replica.d.ts.map +1 -1
  63. package/dist/replica.js +273 -214
  64. package/dist/rows.d.ts +29 -31
  65. package/dist/rows.d.ts.map +1 -1
  66. package/dist/rows.js +113 -69
  67. package/dist/save.d.ts +9 -10
  68. package/dist/save.d.ts.map +1 -1
  69. package/dist/save.js +59 -57
  70. package/dist/search.d.ts +2 -2
  71. package/dist/search.d.ts.map +1 -1
  72. package/dist/search.js +60 -28
  73. package/dist/shared.d.ts +0 -3
  74. package/dist/shared.d.ts.map +1 -1
  75. package/dist/shared.js +1 -31
  76. package/dist/store.d.ts +79 -62
  77. package/dist/store.d.ts.map +1 -1
  78. package/dist/store.js +164 -330
  79. package/dist/stored.d.ts +143 -0
  80. package/dist/stored.d.ts.map +1 -0
  81. package/dist/stored.js +561 -0
  82. package/dist/tables.d.ts +246 -168
  83. package/dist/tables.d.ts.map +1 -1
  84. package/dist/tables.js +156 -25
  85. package/dist/turn.d.ts +1 -1
  86. package/dist/turn.d.ts.map +1 -1
  87. package/dist/turn.js +28 -20
  88. package/dist/validation.d.ts +30 -0
  89. package/dist/validation.d.ts.map +1 -0
  90. package/dist/validation.js +162 -0
  91. package/dist/windows.d.ts +2 -1
  92. package/dist/windows.d.ts.map +1 -1
  93. package/dist/windows.js +30 -25
  94. package/package.json +2 -3
  95. package/dist/read-side.d.ts +0 -27
  96. package/dist/read-side.d.ts.map +0 -1
  97. package/dist/read-side.js +0 -1179
@@ -1,18 +1,18 @@
1
- import { roomTarget, target } from '@agentshouse/mdmodel';
1
+ import { escapePointer, parseFrontmatter, roomTarget, target } from '@agentshouse/mdmodel';
2
2
  import { attachedModel, NO_ROOM_TYPES, roomTypesOf } from './attached-model.js';
3
3
  import { cached, modelTypes, NO_MODEL, sameModel, } from './model.js';
4
4
  import { observing, whole } from './observation.js';
5
5
  import { filterOf, identityOf, planOf, plain, queryKey, } from './query.js';
6
- import { createReadSide } from './read-side.js';
6
+ import { createLiveQueries } from './live.js';
7
7
  import { seededResult } from './seed.js';
8
- import { documentKey, markdownPath, roomOfKey } from './rows.js';
8
+ import { documentKey, roomOfKey } from './rows.js';
9
9
  import { createSearchIndex, searchPlanOf, } from './search.js';
10
10
  import { capped, MAXIMUM_WINDOW } from './windows.js';
11
11
  import { bootstrapRoom } from './bootstrap.js';
12
- import { changeSet, changing, entering, mergeChangeSets } from './change-set.js';
12
+ import { changeSet, mergeChangeSets } from './change-set.js';
13
13
  import { createConversationStore, } from './conversation.js';
14
14
  import { leaving as leavingIn, partnerOf, queuedFile, } from './change.js';
15
- import { indexReferrers, referencesIndexed, validRoot, } from './collection.js';
15
+ import { available, collectionOf, unreadable as unreadableIn, validRoot, } from './collection.js';
16
16
  import { runtimeError } from './failure.js';
17
17
  import { createLog } from './log.js';
18
18
  import { binaryRecord } from './binary.js';
@@ -20,19 +20,21 @@ import { openedFiles } from './opened.js';
20
20
  import { ABSENT_TARGET, absentBinary, absentReference, binaryAnswer, binaryLinkAnswer, calledResult, failure, historyAnswer, LOCKED_UNREADABLE, lockedRoom, undeclared, UNCERTAIN, UNREADABLE, } from './outcome.js';
21
21
  import { createRecovery } from './recovery.js';
22
22
  import { doorReads } from './reads.js';
23
- import { refreshRoom } from './refresh.js';
23
+ import { advancement } from './refresh.js';
24
24
  import { buildBatch, buildMove, joined, limiting, queuedDocuments, requeuing, } from './save.js';
25
- import { applyReturned, compositionOf, confirmedOf, landed, readReclassified, replicating, sameMembership, standing, } from './replica.js';
25
+ import { compositionOf, entryAt, landed, movedBy, overlayOf, replaying, returning, returnedPaths, sameMembership, slicing, slicingWith, standing, storedRecord, touchedBy, } from './replica.js';
26
26
  import { binaryBase, changedParts, changesOf, held, heldFile, joinedDraft, moveTarget, readParts, returnedBy, returnedInto, staleWrite, underneath, withdrawnDrafts, } from './room-draft.js';
27
27
  import { same, saving, STARTING, transferring, } from './status.js';
28
28
  import { detectStorageLoss, nextGeneration } from './durability.js';
29
29
  import { readinessOf, SETTLED_ROOMS, } from './readiness.js';
30
30
  import { FencedMap } from './fence.js';
31
- import { keysIn, tallying } from './shared.js';
32
- import { finished, inChunks, ROW_WEIGHT, stepped, yielded } from './turn.js';
31
+ import { fingerprint } from './stored.js';
32
+ import { sameBinary } from './overlay.js';
33
+ import { paced, ROW_WEIGHT, stepped, yielded } from './turn.js';
33
34
  import { identityTime, roomDraftRevision } from './identity.js';
34
35
  import { beneath, normalize } from './paths.js';
35
36
  import { openRuntimeStore, } from './store.js';
37
+ import { changedCleanly, sameComposition, validatedCollection, } from './validation.js';
36
38
  import { unconfirmed } from './transfer.js';
37
39
  import { governs, holds, narrowed, planReach, } from './reach.js';
38
40
  function roomsAlone(plan) {
@@ -61,26 +63,19 @@ const OPENING_LANE = 'opening';
61
63
  const CONVERSATION_LANE = 'conversations';
62
64
  const ROOM_READS = 256;
63
65
  const ROOM_ATTEMPTS = 16;
66
+ const SEARCH_BATCH = 256;
67
+ const ERASE_BATCH = 512;
68
+ const REDERIVE_BATCH = 256;
69
+ const WALK_BYTES = 1 << 20;
70
+ const QUERY_BYTES = 1 << 20;
71
+ const STAGE_BYTES = 1 << 20;
72
+ const CLEAN_CHANGES = 2_000;
64
73
  // A Room that locks offline stays readable this long after the connection drops, so a laptop waking or changing network keeps working while a disconnected device locks soon after.
65
74
  const LOCK_GRACE_MS = 120_000;
66
75
  function operationOf(input) {
67
76
  const operation = input?.operation_id;
68
77
  return typeof operation === 'string' ? operation : null;
69
78
  }
70
- const outside = new Map();
71
- function outsideOf(roots) {
72
- const key = JSON.stringify(roots);
73
- const held = outside.get(key);
74
- if (held !== undefined)
75
- return held;
76
- const counted = (path) => (roots.some((root) => beneath(root, path)) ? 0 : 1);
77
- const made = {
78
- documents: (path) => (markdownPath(path) ? counted(path) : 0),
79
- binaries: counted,
80
- };
81
- outside.set(key, made);
82
- return made;
83
- }
84
79
  function abandonedOn(signal) {
85
80
  return new Promise((resolve) => {
86
81
  if (signal.aborted)
@@ -184,7 +179,7 @@ export function composeRuntime(options) {
184
179
  const queueSubscriptions = new Set();
185
180
  const roomDraftListeners = new Set();
186
181
  const presentationListeners = new Set();
187
- const readSide = createReadSide();
182
+ const liveQueries = createLiveQueries((plan, cover) => answeredPlan(plan, cover));
188
183
  let withdrawConversations;
189
184
  let currencyQueued = null;
190
185
  let socketOpen = false;
@@ -198,7 +193,6 @@ export function composeRuntime(options) {
198
193
  let member = null;
199
194
  const memberListeners = new Set();
200
195
  const prepared = new Set();
201
- const projecting = new Set();
202
196
  const unpreparedDemand = new Set();
203
197
  const unindexable = new Set();
204
198
  let preparing = false;
@@ -213,7 +207,19 @@ export function composeRuntime(options) {
213
207
  const transfers = new FencedMap(isFenced);
214
208
  const conversations = createConversationStore({
215
209
  agents: declaredAgents,
216
- documents: ({ locator }) => rowsQuery(planOf({ rooms: [locator], select: { fields: 'all' } }, roomFacts())),
210
+ documents: (declaration, after) => rowsQuery(planOf({
211
+ rooms: [declaration.locator],
212
+ types: [declaration.type],
213
+ select: {
214
+ fields: [
215
+ declaration.identity,
216
+ declaration.name,
217
+ ...(declaration.retired === undefined ? [] : [declaration.retired]),
218
+ ].map((field) => `/${escapePointer(field)}`),
219
+ },
220
+ limit: MAXIMUM_WINDOW,
221
+ ...(after === undefined ? {} : { after }),
222
+ }, roomFacts())),
217
223
  call: (name, input) => invokeAction(name, input),
218
224
  durably,
219
225
  stream: () => options.adapter.conversations,
@@ -224,8 +230,25 @@ export function composeRuntime(options) {
224
230
  expired: (operation) => pastHorizon(operation),
225
231
  });
226
232
  const searchIndex = createSearchIndex({
227
- room: (room) => readSide.room(room),
228
- document: (room, path) => readSide.document(room, path),
233
+ async *room(room) {
234
+ const local = await openLocal();
235
+ for (let after = null;;) {
236
+ const key = replicas.get(room)?.record.key;
237
+ if (key === undefined)
238
+ return;
239
+ const page = await local.searched(key, room, after, SEARCH_BATCH, null);
240
+ yield page.documents;
241
+ if (page.next === null)
242
+ return;
243
+ after = page.next;
244
+ }
245
+ },
246
+ async documents(room, paths) {
247
+ const key = replicas.get(room)?.record.key;
248
+ if (key === undefined || paths.length === 0)
249
+ return [];
250
+ return (await (await openLocal()).searched(key, room, null, paths.length, paths)).documents;
251
+ },
229
252
  });
230
253
  const roomsToIndex = new Set();
231
254
  const conversationsToIndex = new Set();
@@ -249,16 +272,36 @@ export function composeRuntime(options) {
249
272
  if (room.size === 0)
250
273
  roomDrafts.delete(locator);
251
274
  }
252
- function roomDraftOf(record) {
275
+ async function visibleOf(locator, paths) {
276
+ const held = replicas.get(locator);
277
+ if (held === undefined)
278
+ return undefined;
279
+ const local = await openLocal();
280
+ const slice = await stepped(slicing(await local.entries(held.record.key, 'visible', paths), paths));
281
+ return { ...slice, collection: collectionOf(slice.documents, compositionOf(held.record)) };
282
+ }
283
+ function roomDraftOf(record, visible) {
253
284
  return structuredClone({
254
285
  room: record.locator,
255
286
  path: record.path,
256
287
  revision: record.revision,
257
288
  parts: record.parts,
258
- changedUnderneath: underneath(record.parts, record.path, replicas.get(record.locator)?.visible),
289
+ changedUnderneath: underneath(record.parts, record.path, visible),
259
290
  returned: record.returned,
260
291
  });
261
292
  }
293
+ async function roomDraftsOf(records) {
294
+ const rooms = new Map();
295
+ for (const record of records) {
296
+ const paths = rooms.get(record.locator) ?? [];
297
+ paths.push(record.path);
298
+ rooms.set(record.locator, paths);
299
+ }
300
+ const layers = new Map();
301
+ for (const [locator, paths] of rooms)
302
+ layers.set(locator, await visibleOf(locator, paths));
303
+ return records.map((record) => roomDraftOf(record, layers.get(record.locator)));
304
+ }
262
305
  function snapshotRoomDrafts() {
263
306
  const records = [...roomDrafts.values()].flatMap((room) => [...room.values()]);
264
307
  records.sort((left, right) => {
@@ -268,18 +311,24 @@ export function composeRuntime(options) {
268
311
  return left.path < right.path ? -1 : 1;
269
312
  return 0;
270
313
  });
271
- return records.map(roomDraftOf);
314
+ return roomDraftsOf(records);
272
315
  }
316
+ let draftsShown = Promise.resolve();
273
317
  function publishRoomDrafts(waitingOnly) {
274
318
  if (closed)
275
319
  return;
276
- const value = snapshotRoomDrafts();
277
- for (const listener of roomDraftListeners) {
278
- if (waitingOnly && listener.ready)
279
- continue;
280
- listener.ready = true;
281
- listener.notify(value);
282
- }
320
+ const snapshot = snapshotRoomDrafts();
321
+ draftsShown = draftsShown.then(async () => {
322
+ const value = await snapshot.catch(() => null);
323
+ if (value === null || closed)
324
+ return;
325
+ for (const listener of roomDraftListeners) {
326
+ if (waitingOnly && listener.ready)
327
+ continue;
328
+ listener.ready = true;
329
+ listener.notify(value);
330
+ }
331
+ });
283
332
  }
284
333
  function presentationOf(app) {
285
334
  const value = presentations.get(app);
@@ -291,7 +340,7 @@ export function composeRuntime(options) {
291
340
  const stored = next(presentations.get(app));
292
341
  await local.present({ app, value: stored });
293
342
  presentations.set(app, stored);
294
- publishChangeSet(changeSet('', null, null, { presentation: app }));
343
+ publishChangeSet(changeSet('', { presentation: app }));
295
344
  });
296
345
  }
297
346
  function emitPresentation(app) {
@@ -342,13 +391,7 @@ export function composeRuntime(options) {
342
391
  return { attentionRequired, reset };
343
392
  }
344
393
  function documentsIn(locator) {
345
- const replica = replicas.get(locator);
346
- return replica === undefined ? 0 : finished(counting(replica));
347
- }
348
- function* counting(replica) {
349
- const weighed = outsideOf(replica.confirmed.record.typedSelection?.roots ?? []);
350
- return ((yield* tallying(replica.visible.documents, weighed.documents)) +
351
- (yield* tallying(replica.visible.binaries, weighed.binaries)));
394
+ return replicas.get(locator)?.count ?? 0;
352
395
  }
353
396
  function listed() {
354
397
  return structuredClone([...rooms.values()].map((room) => {
@@ -429,7 +472,7 @@ export function composeRuntime(options) {
429
472
  return { listed: roomsListed, rooms: held };
430
473
  }
431
474
  function emitRooms() {
432
- readSide.recover();
475
+ liveQueries.recover();
433
476
  queueReadiness();
434
477
  const value = listed();
435
478
  for (const listener of roomListeners)
@@ -488,56 +531,29 @@ export function composeRuntime(options) {
488
531
  }
489
532
  }
490
533
  function applyChangeSet(set) {
491
- const held = {
492
- ...set,
493
- documents: set.documents.filter(({ locator }) => prepared.has(locator) || projecting.has(locator)),
494
- };
495
- const applied = readSide.apply(held, () => searchIndex.changed(held));
496
- if (applied instanceof Promise)
497
- void applied.catch(unpublished);
534
+ liveQueries.touched(set);
535
+ searchIndex.changed(set);
498
536
  }
499
537
  function unpublished(error) {
500
538
  journal.record(`publication failed ${error instanceof Error ? error.name : 'unknown'}`);
501
539
  }
502
- async function referenced(locator) {
503
- for (;;) {
504
- const record = replicas.get(locator)?.confirmed.record;
505
- if (record === undefined || closed)
506
- return;
507
- const held = { text: record.documents, binaries: record.binaries };
508
- const composition = compositionOf(record);
509
- if (referencesIndexed(held, composition))
510
- return;
511
- await indexReferrers(held, composition);
512
- }
513
- }
514
540
  async function prepareReplica(locator) {
515
- await referenced(locator);
516
- const replica = replicas.get(locator);
517
- if (replica === undefined || closed || prepared.has(locator))
518
- return;
519
- const visible = replica.visible;
520
- const entered = [];
521
- projecting.add(locator);
522
- try {
523
- await inChunks(await stepped(keysIn(visible.documents, visible.binaries)), (path) => ROW_WEIGHT + (visible.documents[path]?.length ?? 0), (chunk) => {
524
- const current = replicas.get(locator);
525
- if (current === undefined || closed)
526
- return;
527
- const set = entering(locator, current, chunk);
528
- entered.push(set);
529
- return readSide.apply(set);
530
- });
531
- if (!replicas.has(locator) || closed)
541
+ for (;;) {
542
+ const held = replicas.get(locator);
543
+ if (held === undefined || closed || prepared.has(locator))
532
544
  return;
545
+ const key = held.record.key;
546
+ const done = await exclusive(async () => (await openLocal()).rederive(key, REDERIVE_BATCH));
547
+ if (replicas.get(locator)?.record.key !== key)
548
+ continue;
549
+ if (!done)
550
+ continue;
533
551
  prepared.add(locator);
552
+ publishChangeSet(changeSet(locator, { documents: null }));
553
+ liveQueries.recover();
554
+ queueReadiness();
555
+ return;
534
556
  }
535
- finally {
536
- projecting.delete(locator);
537
- }
538
- conversations.changed(mergeChangeSets(entered));
539
- readSide.recover();
540
- queueReadiness();
541
557
  }
542
558
  function unprepared() {
543
559
  return [...unpreparedDemand, ...replicas.keys()].find((locator) => !prepared.has(locator) && !unindexable.has(locator));
@@ -593,6 +609,7 @@ export function composeRuntime(options) {
593
609
  const pending = [];
594
610
  const coverage = [];
595
611
  const overlay = [];
612
+ const read = [];
596
613
  for (const locator of locators) {
597
614
  const replica = replicas.get(locator);
598
615
  if (replica === undefined) {
@@ -606,15 +623,16 @@ export function composeRuntime(options) {
606
623
  }
607
624
  coverage.push({
608
625
  room: locator,
609
- generation: replica.confirmed.record.transfer,
626
+ generation: replica.record.transfer,
610
627
  current: !(catching.has(locator) || owed.has(locator) || transfers.has(locator)),
611
628
  });
629
+ read.push({ locator, key: replica.record.key });
612
630
  for (const batch of replica.queue)
613
631
  overlay.push(batch.id);
614
632
  }
615
633
  if (pending.length > 0)
616
634
  return { pending };
617
- return { coverage: { rooms: coverage, indexes: [], overlay } };
635
+ return { coverage: { rooms: coverage, indexes: [], overlay }, rooms: read };
618
636
  }
619
637
  function awaitPreparation(cover) {
620
638
  if (!('pending' in cover))
@@ -726,7 +744,7 @@ export function composeRuntime(options) {
726
744
  function establish(name) {
727
745
  targets = { ...targets, [name]: { established: true } };
728
746
  queueReadiness();
729
- void durably((local) => local.keepTargets(targets)).catch(() => {
747
+ void durably((local) => local.fact('targets', targets)).catch(() => {
730
748
  journal.record(`target record failed ${name}`);
731
749
  });
732
750
  }
@@ -763,18 +781,12 @@ export function composeRuntime(options) {
763
781
  readinessWatches.delete(watch);
764
782
  };
765
783
  }
766
- function layerBytes(layer) {
784
+ function pageBytes(page) {
767
785
  let bytes = 0;
768
- for (const [path, text] of Object.entries(layer.documents))
769
- bytes += (path.length + text.length) * 2;
786
+ for (const { path, content } of page.entries)
787
+ bytes += (path.length + (content?.length ?? 0)) * 2;
770
788
  return bytes;
771
789
  }
772
- function grownBy(next, previous) {
773
- if (previous === undefined)
774
- return layerBytes(next);
775
- const documents = Object.fromEntries(Object.entries(next.documents).filter(([path, text]) => previous.documents[path] !== text));
776
- return layerBytes({ ...next, documents });
777
- }
778
790
  function exceeded(error) {
779
791
  return (typeof error === 'object' &&
780
792
  error !== null &&
@@ -883,6 +895,75 @@ export function composeRuntime(options) {
883
895
  shown();
884
896
  }
885
897
  }
898
+ function sameCover(left, right) {
899
+ return JSON.stringify(left) === JSON.stringify(right);
900
+ }
901
+ async function coveredRows(plan, cover) {
902
+ for (;;) {
903
+ const before = cover();
904
+ if ('pending' in before)
905
+ return { pending: before };
906
+ const local = await openLocal();
907
+ const keys = before.rooms.map(({ key }) => key);
908
+ const changes = local.changes(keys);
909
+ const current = () => local.changes(keys) === changes;
910
+ const rows = await windowRows(local, plan, before.rooms, current);
911
+ const after = cover();
912
+ if (rows !== null && sameCover(before, after))
913
+ return { rows, cover: before, current };
914
+ }
915
+ }
916
+ async function windowRows(local, plan, rooms, unchanged) {
917
+ const first = await local.query(plan, rooms, QUERY_BYTES);
918
+ if (first.more === undefined || !('documents' in first)) {
919
+ const { more: _more, ...rows } = first;
920
+ return rows;
921
+ }
922
+ const documents = [...first.documents];
923
+ for (let more = first.more; more !== undefined;) {
924
+ if (!unchanged())
925
+ return null;
926
+ const page = await local.query({
927
+ ...plan,
928
+ after: { text: more, values: JSON.parse(more) },
929
+ limit: plan.limit === null ? null : plan.limit - documents.length,
930
+ total: false,
931
+ }, rooms, QUERY_BYTES);
932
+ if (!('documents' in page))
933
+ return null;
934
+ documents.push(...page.documents);
935
+ more = page.more;
936
+ if (more === undefined) {
937
+ return {
938
+ documents,
939
+ ...(page.next === undefined ? {} : { next: page.next }),
940
+ ...(first.total === undefined ? {} : { total: first.total }),
941
+ };
942
+ }
943
+ }
944
+ return null;
945
+ }
946
+ function keyedOf(rows, coverage) {
947
+ if ('groups' in rows) {
948
+ return {
949
+ result: { ...rows, coverage },
950
+ keys: rows.groups.map((group) => JSON.stringify(group.key)),
951
+ };
952
+ }
953
+ return {
954
+ result: { ...rows, coverage },
955
+ keys: rows.documents.map((document) => documentKey(document.room, document.path)),
956
+ };
957
+ }
958
+ async function answeredPlan(plan, cover) {
959
+ const answered = await coveredRows(plan, cover);
960
+ if ('pending' in answered) {
961
+ const pending = answered.pending;
962
+ return { result: pending, keys: [], current: () => true };
963
+ }
964
+ const held = answered.cover;
965
+ return { ...keyedOf(answered.rows, held.coverage), current: answered.current };
966
+ }
886
967
  async function rowsQuery(plan) {
887
968
  const before = covered(plan);
888
969
  awaitPreparation(before);
@@ -891,11 +972,42 @@ export function composeRuntime(options) {
891
972
  return seededResult(seeded).result;
892
973
  if ('pending' in before)
893
974
  return before;
894
- const { rows, at: cover } = await readSide.query(plan, () => covered(plan));
895
- awaitPreparation(cover);
896
- if ('pending' in cover)
975
+ const answered = await coveredRows(plan, () => {
976
+ const cover = covered(plan);
977
+ awaitPreparation(cover);
897
978
  return cover;
898
- return { ...rows, coverage: cover.coverage };
979
+ });
980
+ if ('pending' in answered)
981
+ return answered.pending;
982
+ const held = answered.cover;
983
+ return { ...answered.rows, coverage: held.coverage };
984
+ }
985
+ async function everyRow(plan) {
986
+ const window = { ...plan, limit: MAXIMUM_WINDOW, after: null, total: false };
987
+ const first = await rowsQuery(window);
988
+ if ('pending' in first || first.next === undefined)
989
+ return first;
990
+ const documents = 'documents' in first ? [...first.documents] : [];
991
+ const groups = 'groups' in first ? [...first.groups] : [];
992
+ for (let next = first.next; next !== undefined;) {
993
+ const values = JSON.parse(next);
994
+ const page = await rowsQuery({
995
+ ...window,
996
+ after: { text: next, values: values.length === 0 ? null : values },
997
+ });
998
+ if ('pending' in page)
999
+ return page;
1000
+ if ('documents' in page)
1001
+ documents.push(...page.documents);
1002
+ else
1003
+ groups.push(...page.groups);
1004
+ if (JSON.stringify(page.coverage) !== JSON.stringify(first.coverage))
1005
+ return everyRow(plan);
1006
+ next = page.next;
1007
+ }
1008
+ return 'documents' in first
1009
+ ? { documents, coverage: first.coverage }
1010
+ : { groups, coverage: first.coverage };
899
1011
  }
900
1012
  async function indexConversation(conversation) {
901
1013
  searchIndex.indexMessages(conversation, [], false);
@@ -953,7 +1065,10 @@ export function composeRuntime(options) {
953
1065
  for (const locator of set.queues)
954
1066
  emitQueue(locator);
955
1067
  if (set.drafts.length > 0 ||
956
- set.documents.some((change) => roomDrafts.get(change.locator)?.has(change.path) === true)) {
1068
+ set.documents.some((change) => {
1069
+ const held = roomDrafts.get(change.locator);
1070
+ return held !== undefined && (change.path === null || held.has(change.path));
1071
+ })) {
957
1072
  publishRoomDrafts(false);
958
1073
  }
959
1074
  for (const app of set.presentations)
@@ -966,9 +1081,21 @@ export function composeRuntime(options) {
966
1081
  if (reason === 'revoked-authorization') {
967
1082
  await exclusive(async () => {
968
1083
  const revoked = { status: 'unavailable', reason: 'revoked' };
1084
+ const records = [...replicas].map(([locator, held]) => ({
1085
+ locator,
1086
+ held,
1087
+ record: { ...held.record, model: revoked, selectionModel: revoked },
1088
+ }));
969
1089
  try {
970
1090
  const local = await openLocal();
971
- await local.revokeModels([...replicas.values()].map((replica) => replica.confirmed.record));
1091
+ await local.commit({
1092
+ rooms: records.map(({ locator, record }) => ({
1093
+ locator,
1094
+ key: record.key,
1095
+ record: storedRecord(record),
1096
+ composition: compositionOf(record),
1097
+ })),
1098
+ });
972
1099
  }
973
1100
  catch {
974
1101
  journal.record('model revocation persistence failed');
@@ -983,22 +1110,17 @@ export function composeRuntime(options) {
983
1110
  models.set(key, revoked);
984
1111
  liveModels.delete(key);
985
1112
  }
986
- const revocations = [];
987
- for (const locator of [...rooms.keys()]) {
988
- const before = replicas.get(locator) ?? null;
989
- const after = before === null
990
- ? null
991
- : await stepped(replicating(await confirmedOf({ ...before.confirmed.record, model: revoked, selectionModel: revoked }, before.confirmed), before.queue, before));
992
- const set = await stepped(changing(locator, before, after, { room: true }));
993
- revocations.push([locator, before, after, set]);
994
- }
995
1113
  const changed = [];
996
- for (const [locator, before, after, set] of revocations) {
1114
+ for (const { locator, held, record } of records) {
1115
+ if (replicas.get(locator) !== held)
1116
+ continue;
1117
+ recomposed(locator, held.record, record);
1118
+ replicas.set(locator, { ...held, record });
1119
+ }
1120
+ for (const locator of [...rooms.keys()]) {
997
1121
  const room = rooms.get(locator);
998
- if (room === undefined || (replicas.get(locator) ?? null) !== before)
1122
+ if (room === undefined)
999
1123
  continue;
1000
- if (after !== null)
1001
- replicas.set(locator, after);
1002
1124
  rooms.set(locator, {
1003
1125
  ...room,
1004
1126
  model: modelTypes(revoked),
@@ -1006,9 +1128,10 @@ export function composeRuntime(options) {
1006
1128
  ? null
1007
1129
  : { ...room.typedSelection, model: modelTypes(revoked) },
1008
1130
  });
1009
- changed.push(set);
1131
+ changed.push(changeSet(locator, { documents: null, room: true }));
1010
1132
  }
1011
1133
  publishChangeSet(mergeChangeSets(changed));
1134
+ prepare();
1012
1135
  });
1013
1136
  }
1014
1137
  }
@@ -1039,7 +1162,7 @@ export function composeRuntime(options) {
1039
1162
  model: modelTypes(modelOf(room.locator, 'typed-selection')),
1040
1163
  },
1041
1164
  providerOwnedPaths: room.providerOwnedPaths,
1042
- roomVersion: replicas.get(room.locator)?.confirmed.record.roomVersion ?? null,
1165
+ roomVersion: replicas.get(room.locator)?.record.roomVersion ?? null,
1043
1166
  synchronizationAvailable,
1044
1167
  refusal: scopedRefusals.get(room.locator) ?? sourceRefusals.get(room.locator) ?? null,
1045
1168
  });
@@ -1071,7 +1194,7 @@ export function composeRuntime(options) {
1071
1194
  else
1072
1195
  unlistedKept(unlisted);
1073
1196
  for (const room of listed) {
1074
- const held = replicas.get(room.locator)?.confirmed.record;
1197
+ const held = replicas.get(room.locator)?.record;
1075
1198
  if (held !== undefined && held.replica.selection !== room.replica.selection) {
1076
1199
  list(held, false);
1077
1200
  continue;
@@ -1102,7 +1225,7 @@ export function composeRuntime(options) {
1102
1225
  }
1103
1226
  function unlistedKept(unlisted) {
1104
1227
  for (const locator of unlisted) {
1105
- const held = replicas.get(locator)?.confirmed.record;
1228
+ const held = replicas.get(locator)?.record;
1106
1229
  if (held !== undefined)
1107
1230
  list(held, false);
1108
1231
  else
@@ -1113,7 +1236,6 @@ export function composeRuntime(options) {
1113
1236
  const fresh = [...new Set(locators)].filter((locator) => !fences.has(locator));
1114
1237
  const removed = fresh.map((locator) => ({
1115
1238
  locator,
1116
- shown: prepared.has(locator) || projecting.has(locator) ? (replicas.get(locator) ?? null) : null,
1117
1239
  drafts: [...(roomDrafts.get(locator)?.keys() ?? [])],
1118
1240
  }));
1119
1241
  for (const locator of fresh) {
@@ -1150,36 +1272,48 @@ export function composeRuntime(options) {
1150
1272
  if (fresh.length === 0)
1151
1273
  return fresh;
1152
1274
  searchIndex.forget(fresh);
1153
- const withdrawn = readSide.apply(stepped(withdrawing(removed)), (set) => conversations.changed(set));
1154
- if (withdrawn instanceof Promise)
1155
- void withdrawn.catch(unpublished);
1156
- publishChangeSet(mergeChangeSets(removed.map(({ locator, drafts }) => changeSet(locator, null, null, { room: true, queue: true, drafts }))));
1157
- readSide.recover();
1275
+ publishChangeSet(mergeChangeSets(removed.map(({ locator, drafts }) => changeSet(locator, { documents: null, room: true, queue: true, drafts }))));
1276
+ liveQueries.recover();
1158
1277
  moveTo({ ...status, ...attention() });
1159
1278
  return fresh;
1160
1279
  }
1161
- function* withdrawing(removed) {
1162
- const sets = [];
1163
- for (const { locator, shown } of removed)
1164
- sets.push(yield* changing(locator, shown, null));
1165
- return mergeChangeSets(sets);
1166
- }
1167
- async function erased(local, locators) {
1168
- await local.erase(locators);
1280
+ async function eraseFenced(locators) {
1169
1281
  for (const locator of locators) {
1282
+ while (await durably((local) => local.erase(locator, ERASE_BATCH)))
1283
+ await yielded();
1170
1284
  const held = fences.get(locator);
1171
1285
  if (held !== undefined)
1172
1286
  held.erased = true;
1173
1287
  }
1174
1288
  }
1289
+ let sweeping = false;
1290
+ function sweep() {
1291
+ if (sweeping || closed)
1292
+ return;
1293
+ sweeping = true;
1294
+ void (async () => {
1295
+ try {
1296
+ while (!closed && (await durably((local) => local.erase(null, ERASE_BATCH)))) {
1297
+ await yielded();
1298
+ }
1299
+ }
1300
+ catch {
1301
+ journal.record('erasure failed');
1302
+ }
1303
+ finally {
1304
+ sweeping = false;
1305
+ }
1306
+ })();
1307
+ }
1175
1308
  async function loseRooms(locators) {
1176
1309
  const fresh = fence(locators);
1177
1310
  if (fresh.length === 0)
1178
1311
  return;
1179
1312
  await durably(async (local) => {
1180
- await erased(local, fresh);
1313
+ await local.fence(fresh);
1181
1314
  await unpair(local);
1182
1315
  });
1316
+ await eraseFenced(fresh);
1183
1317
  }
1184
1318
  function restored(key, persisted) {
1185
1319
  const prior = models.get(key);
@@ -1200,50 +1334,30 @@ export function composeRuntime(options) {
1200
1334
  return model;
1201
1335
  }
1202
1336
  async function hydrate(opened) {
1203
- const [records, queues, kept, presented, held, chatDrafts, lastCompleted, declared, named, known, staged, heldTargets,] = await Promise.all([
1204
- opened.all(),
1337
+ const [held, queues, kept, heldConversations, chatDrafts] = await Promise.all([
1338
+ opened.opened(),
1205
1339
  opened.queues(),
1206
1340
  opened.roomDrafts(),
1207
- opened.presentation(),
1208
1341
  opened.conversations(),
1209
1342
  opened.chatDrafts(),
1210
- opened.lastCompleted(),
1211
- opened.providerLimits(),
1212
- opened.capabilities(),
1213
- opened.member(),
1214
- opened.transfers(),
1215
- opened.targets(),
1216
1343
  ]);
1344
+ const records = [...held.rooms.values()];
1345
+ const counts = await opened.counts(records.map(({ key, typedSelection }) => ({ key, roots: typedSelection?.roots ?? [] })));
1217
1346
  const queuedBatches = new Map(queues.map((queue) => [queue.locator, queue.pending.map(({ batch }) => batch)]));
1218
- const previousReplicas = new Map(replicas);
1219
- const read = new Map();
1220
- for (const record of records) {
1221
- const model = restored(modelKey(record.locator, 'collection'), record.model);
1222
- const selectionModel = restored(modelKey(record.locator, 'typed-selection'), record.selectionModel);
1223
- const prior = previousReplicas.get(record.locator);
1224
- const confirmed = await confirmedOf({ ...record, model, selectionModel }, prior?.confirmed);
1225
- const replica = await stepped(replicating(confirmed, queuedBatches.get(record.locator) ?? [], prior));
1226
- await stepped(counting(replica));
1227
- read.set(record.locator, replica);
1228
- }
1229
- const sets = new Map();
1230
- for (const locator of new Set([...previousReplicas.keys(), ...read.keys()])) {
1231
- sets.set(locator, await stepped(changing(locator, previousReplicas.get(locator) ?? null, read.get(locator) ?? null)));
1232
- }
1233
- targets = heldTargets;
1347
+ targets = held.targets;
1234
1348
  transfers.clear();
1235
- for (const record of staged)
1236
- transfers.set(record.locator, record);
1237
- limits = declared;
1238
- capabilities = named;
1239
- member = known;
1349
+ for (const [locator, record] of held.transfers)
1350
+ transfers.set(locator, record);
1351
+ limits = held.providerLimits;
1352
+ capabilities = held.capabilities;
1353
+ member = held.member;
1240
1354
  roomDrafts.clear();
1241
1355
  for (const record of kept)
1242
1356
  rememberRoomDraft(record);
1243
1357
  presentations.clear();
1244
- for (const record of presented)
1358
+ for (const record of held.presentation)
1245
1359
  presentations.set(record.app, record.value);
1246
- conversations.hydrate(held, chatDrafts);
1360
+ conversations.hydrate(heldConversations, chatDrafts);
1247
1361
  pendings.clear();
1248
1362
  for (const queue of queues) {
1249
1363
  for (const { batch, state } of queue.pending) {
@@ -1251,17 +1365,27 @@ export function composeRuntime(options) {
1251
1365
  }
1252
1366
  }
1253
1367
  replicas.clear();
1368
+ prepared.clear();
1254
1369
  queuesWithoutReplica.clear();
1255
1370
  for (const [locator, queue] of queuedBatches) {
1256
- if (!read.has(locator))
1371
+ if (!held.rooms.has(locator))
1257
1372
  queuesWithoutReplica.set(locator, queue);
1258
1373
  }
1259
- for (const record of records) {
1260
- if (fences.has(record.locator))
1261
- continue;
1262
- const replica = read.get(record.locator);
1263
- const { model, selectionModel } = replica.confirmed.record;
1264
- replicas.set(record.locator, replica);
1374
+ records.forEach((stored, index) => {
1375
+ if (fences.has(stored.locator))
1376
+ return;
1377
+ const model = restored(modelKey(stored.locator, 'collection'), stored.model);
1378
+ const selectionModel = restored(modelKey(stored.locator, 'typed-selection'), stored.selectionModel);
1379
+ const record = { ...stored, model, selectionModel };
1380
+ replicas.set(record.locator, {
1381
+ record,
1382
+ queue: queuedBatches.get(record.locator) ?? [],
1383
+ overlaid: held.overlaid.get(record.key) ?? new Set(),
1384
+ count: counts[index] ?? 0,
1385
+ confirmed: 0,
1386
+ });
1387
+ if (!held.unprepared.has(record.key))
1388
+ prepared.add(record.locator);
1265
1389
  const existing = rooms.get(record.locator);
1266
1390
  if (existing === undefined)
1267
1391
  list(record, false);
@@ -1280,11 +1404,15 @@ export function composeRuntime(options) {
1280
1404
  },
1281
1405
  });
1282
1406
  }
1283
- }
1407
+ });
1408
+ for (const key of held.orphans)
1409
+ await opened.release(key);
1410
+ if (held.erasing || held.orphans.length > 0)
1411
+ queueMicrotask(sweep);
1284
1412
  return {
1285
- lastCompleted,
1413
+ lastCompleted: held.lastCompleted,
1286
1414
  locators: new Set([...queuedBatches.keys(), ...replicas.keys()]),
1287
- read: mergeChangeSets([...sets].flatMap(([locator, set]) => (fences.has(locator) ? [] : [set]))),
1415
+ read: mergeChangeSets([...replicas.keys()].map((locator) => changeSet(locator, { documents: null }))),
1288
1416
  };
1289
1417
  }
1290
1418
  async function readState(opened) {
@@ -1323,8 +1451,13 @@ export function composeRuntime(options) {
1323
1451
  emitStorageLoss();
1324
1452
  }
1325
1453
  const lastCompleted = await readState(opened);
1326
- if (interrupted.length > 0)
1327
- await erased(opened, interrupted);
1454
+ if (interrupted.length > 0) {
1455
+ queueMicrotask(() => {
1456
+ void eraseFenced(interrupted).catch(() => {
1457
+ journal.record('erasure failed');
1458
+ });
1459
+ });
1460
+ }
1328
1461
  moveTo({ ...status, lastCompleted, ...attention() });
1329
1462
  void exclusive(() => unpair(opened)).catch(() => {
1330
1463
  journal.record('unpair failed');
@@ -1355,15 +1488,31 @@ export function composeRuntime(options) {
1355
1488
  return state !== undefined && state.submission.state !== 'submitted' && !state.lost;
1356
1489
  };
1357
1490
  }
1358
- function landedWhole(locator, batch, confirmed, current) {
1359
- if (!landed(batch, confirmed, current))
1491
+ const REWRITE_LINKS = ['field', 'body'];
1492
+ const CHECKED_LINKS = ['field', 'section'];
1493
+ async function sliceAt(record, view, queue, extra = [], moved = []) {
1494
+ const local = await openLocal();
1495
+ const needed = new Set([...touchedBy(queue), ...extra]);
1496
+ const movedPaths = [...new Set([...movedBy(queue, record.locator), ...moved])];
1497
+ if (movedPaths.length > 0) {
1498
+ for (const { path } of await local.referrers(record.key, view, movedPaths, REWRITE_LINKS)) {
1499
+ needed.add(path);
1500
+ }
1501
+ }
1502
+ const paths = [...needed];
1503
+ return stepped(slicing(await local.entries(record.key, view, paths), paths));
1504
+ }
1505
+ async function landedWhole(locator, batch, confirmedLayer, current) {
1506
+ if (!landed(batch, confirmedLayer, current))
1360
1507
  return false;
1361
1508
  const partner = partnerOf(batch, locator);
1362
1509
  if (partner === null)
1363
1510
  return true;
1364
1511
  const other = replicas.get(partner);
1365
1512
  const half = other?.queue.find((one) => one.id === batch.id);
1366
- return other !== undefined && half !== undefined && landed(half, other.confirmed);
1513
+ if (other === undefined || half === undefined)
1514
+ return false;
1515
+ return landed(half, await sliceAt(other.record, 'confirmed', [half]));
1367
1516
  }
1368
1517
  function carriedBack(locator, unlanded, written) {
1369
1518
  const back = new Map();
@@ -1401,7 +1550,7 @@ export function composeRuntime(options) {
1401
1550
  roomDrafts: records.filter((record) => record.locator === room),
1402
1551
  consumedRoomDrafts: elsewhere.filter((one) => one.locator === room).map(({ path }) => path),
1403
1552
  })),
1404
- changes: [...rooms].map((room) => changeSet(room, null, null, {
1553
+ changes: [...rooms].map((room) => changeSet(room, {
1405
1554
  drafts: [...records, ...elsewhere]
1406
1555
  .filter((one) => one.locator === room)
1407
1556
  .map(({ path }) => path),
@@ -1414,25 +1563,106 @@ export function composeRuntime(options) {
1414
1563
  },
1415
1564
  };
1416
1565
  }
1417
- async function settle(local, locator, before, confirmed, queue, settling = {}) {
1566
+ function recomposed(locator, before, after) {
1567
+ if (before !== undefined &&
1568
+ before.key === after.key &&
1569
+ fingerprint(compositionOf(before)) === fingerprint(compositionOf(after))) {
1570
+ return;
1571
+ }
1572
+ if (before !== undefined && before.key === after.key)
1573
+ prepared.delete(locator);
1574
+ }
1575
+ function committedRoom(locator, record, answer) {
1576
+ if (answer.prepared[locator] === false)
1577
+ prepared.delete(locator);
1578
+ else if (answer.prepared[locator] === true)
1579
+ prepared.add(locator);
1580
+ if (!prepared.has(locator))
1581
+ prepare([locator]);
1582
+ return { count: answer.counts[locator] ?? replicas.get(locator)?.count ?? 0 };
1583
+ }
1584
+ function replayed(record, confirmedLayer, queue) {
1585
+ return stepped(replaying(confirmedLayer, queue, record.locator, compositionOf(record)));
1586
+ }
1587
+ async function overlayDiff(record, before, visible, touched) {
1588
+ const kept = before !== undefined && before.record.key === record.key
1589
+ ? before.overlaid
1590
+ : new Set();
1591
+ return {
1592
+ overlay: {
1593
+ set: await stepped(overlayOf(visible, touched)),
1594
+ cleared: [...kept].filter((path) => !touched.has(path)),
1595
+ },
1596
+ touched,
1597
+ };
1598
+ }
1599
+ async function settle(local, locator, before, record, queue, settling = {}) {
1600
+ return together(local, [await settlement(local, locator, before, record, queue, settling)]);
1601
+ }
1602
+ function unpairedIn(locator, batch) {
1603
+ if (!queueIn(locator).some((held) => held.id === batch.id))
1604
+ return false;
1605
+ if (pendings.get(pendingKey(locator, batch.id))?.accepted !== undefined)
1606
+ return false;
1607
+ return !leavingIn(batch, locator) || withdrawable(locator)(batch);
1608
+ }
1609
+ async function together(local, settlements) {
1610
+ const settled = new Set(settlements.map(({ locator }) => locator));
1611
+ const all = [...settlements];
1612
+ for (let wave = settlements; wave.length > 0;) {
1613
+ const partnered = new Map();
1614
+ for (const { locator, left } of wave) {
1615
+ for (const batch of left) {
1616
+ const partner = partnerOf(batch, locator);
1617
+ if (partner === null || settled.has(partner) || !unpairedIn(partner, batch))
1618
+ continue;
1619
+ partnered.set(partner, [...(partnered.get(partner) ?? []), batch]);
1620
+ }
1621
+ }
1622
+ const partners = [];
1623
+ for (const [partner, batches] of partnered) {
1624
+ settled.add(partner);
1625
+ partners.push(await unpairing(local, partner, batches));
1626
+ }
1627
+ all.push(...partners);
1628
+ wave = partners;
1629
+ }
1630
+ const answer = await local.commit({
1631
+ rooms: all.flatMap(({ commit }) => commit.rooms),
1632
+ writes: all.flatMap(({ commit }) => commit.writes),
1633
+ });
1634
+ return mergeChangeSets(all.map((one) => one.apply(answer)));
1635
+ }
1636
+ async function settlement(local, locator, before, record, queue, settling = {}) {
1637
+ const view = settling.view ?? 'confirmed';
1418
1638
  const previousQueue = before?.queue ?? queuesWithoutReplica.get(locator) ?? [];
1419
- const { withdrawn, kept } = await stepped(standing(confirmed, queue, withdrawable(locator), localSaves));
1639
+ const overrides = settling.changes;
1640
+ const read = await sliceAt(record, view, queue, [
1641
+ ...(before !== undefined && before.record.key === record.key ? before.overlaid : []),
1642
+ ...(overrides?.set.map(({ path }) => path) ?? []),
1643
+ ...(overrides?.removed ?? []),
1644
+ ]);
1645
+ const confirmedLayer = overrides === undefined
1646
+ ? read
1647
+ : await stepped(slicingWith(read, overrides.set, overrides.removed));
1648
+ const composition = compositionOf(record);
1649
+ const { withdrawn, kept } = await stepped(standing(confirmedLayer, queue, withdrawable(locator), localSaves, locator, composition));
1420
1650
  const founded = kept.filter((batch) => !queue.includes(batch));
1421
1651
  const queueChanged = kept.length !== previousQueue.length || founded.length > 0;
1422
- const unqueued = withdrawn.map((batch) => ({
1423
- batch,
1424
- landed: landedWhole(locator, batch, confirmed),
1425
- }));
1652
+ const unqueued = [];
1653
+ for (const batch of withdrawn) {
1654
+ unqueued.push({ batch, landed: await landedWhole(locator, batch, confirmedLayer) });
1655
+ }
1426
1656
  const returning = [
1427
- ...[...unqueued].reverse().map(({ batch, landed }) => ({
1657
+ ...[...unqueued].reverse().map(({ batch, landed: held }) => ({
1428
1658
  batch,
1429
- returned: localSaves || landed ? null : returnedBy('withdrawn'),
1659
+ returned: localSaves || held ? null : returnedBy('withdrawn'),
1430
1660
  })),
1431
- ...(settling.leaving === undefined ? [] : [settling.leaving]),
1661
+ ...(settling.leaving ?? []),
1432
1662
  ];
1433
1663
  const written = new Map();
1434
- for (const { batch, returned } of returning) {
1435
- if (returned === null || batch.between?.destination.room === locator)
1664
+ for (const { batch, returned: reason } of returning) {
1665
+ if (reason === null || batch.between?.destination.room === locator)
1436
1666
  continue;
1437
1667
  for (const saved of pendings.get(pendingKey(locator, batch.id))?.drafts ?? []) {
1438
1668
  const existing = written.get(saved.path) ?? roomDrafts.get(locator)?.get(saved.path);
@@ -1441,73 +1671,85 @@ export function composeRuntime(options) {
1441
1671
  path: saved.path,
1442
1672
  revision: roomDraftRevision(),
1443
1673
  parts: returnedInto(existing?.parts, saved.parts),
1444
- returned,
1674
+ returned: reason,
1445
1675
  });
1446
1676
  }
1447
1677
  }
1448
1678
  const carried = carriedBack(locator, [
1449
- ...[...unqueued].reverse().flatMap(({ batch, landed }) => (landed ? [] : [batch])),
1450
- ...(settling.leaving === undefined || settling.leaving.returned === null
1451
- ? []
1452
- : [settling.leaving.batch]),
1679
+ ...[...unqueued].reverse().flatMap(({ batch, landed: held }) => (held ? [] : [batch])),
1680
+ ...(settling.leaving ?? []).flatMap(({ batch, returned }) => returned === null ? [] : [batch]),
1453
1681
  ], written);
1454
- await local.keepAuthored({
1455
- locator,
1456
- queue: kept,
1457
- founded,
1458
- roomDrafts: [...written.values()],
1459
- consumedRoomDrafts: carried.taken,
1460
- }, ...carried.writes);
1682
+ const replayedLayer = await replayed(record, confirmedLayer, kept);
1683
+ const { overlay, touched } = await overlayDiff(record, before, replayedLayer.visible, replayedLayer.touched);
1684
+ const commit = {
1685
+ rooms: [
1686
+ {
1687
+ locator,
1688
+ key: record.key,
1689
+ record: storedRecord(record),
1690
+ roots: record.typedSelection?.roots ?? [],
1691
+ ...settling.commit,
1692
+ ...(overrides === undefined
1693
+ ? {}
1694
+ : { confirmed: { set: overrides.set, removed: overrides.removed } }),
1695
+ overlay,
1696
+ },
1697
+ ],
1698
+ writes: [
1699
+ {
1700
+ locator,
1701
+ queue: kept,
1702
+ founded,
1703
+ roomDrafts: [...written.values()],
1704
+ consumedRoomDrafts: carried.taken,
1705
+ },
1706
+ ...carried.writes,
1707
+ ],
1708
+ };
1461
1709
  const staying = new Set(kept.map((batch) => batch.id));
1462
- for (const batch of previousQueue) {
1463
- if (!staying.has(batch.id))
1464
- pendings.delete(pendingKey(locator, batch.id));
1465
- }
1466
- for (const record of written.values())
1467
- rememberRoomDraft(record);
1468
- carried.remember();
1469
- const drafts = [...written.keys(), ...carried.taken];
1470
- try {
1471
- await local.write({
1472
- next: confirmed.record,
1473
- previous: before?.confirmed.record ?? null,
1474
- ...(settling.stored === undefined ? {} : { stored: settling.stored }),
1475
- });
1476
- }
1477
- catch (error) {
1478
- const held = before === undefined ? null : await stepped(replicating(before.confirmed, kept, before));
1479
- const set = await stepped(changing(locator, before ?? null, held, { queue: queueChanged, drafts }));
1480
- if (held !== null)
1481
- replicas.set(locator, held);
1482
- else if (kept.length > 0)
1483
- queuesWithoutReplica.set(locator, kept);
1484
- else
1710
+ return {
1711
+ locator,
1712
+ left: previousQueue.filter((batch) => !staying.has(batch.id)),
1713
+ commit,
1714
+ apply: (answer) => {
1715
+ for (const batch of previousQueue) {
1716
+ if (!staying.has(batch.id))
1717
+ pendings.delete(pendingKey(locator, batch.id));
1718
+ }
1719
+ for (const draft of written.values())
1720
+ rememberRoomDraft(draft);
1721
+ carried.remember();
1722
+ const drafts = [...written.keys(), ...carried.taken];
1485
1723
  queuesWithoutReplica.delete(locator);
1486
- publishChangeSet(mergeChangeSets([set, ...carried.changes]));
1487
- moveTo({ ...status, ...attention() });
1488
- throw error;
1489
- }
1490
- const changed = { room: settling.room, queue: queueChanged, drafts };
1491
- const next = await stepped(replicating(confirmed, kept, before));
1492
- await stepped(counting(next));
1493
- const set = await stepped(changing(locator, before ?? null, next, changed));
1494
- return () => {
1495
- queuesWithoutReplica.delete(locator);
1496
- for (const { batch, landed } of unqueued) {
1497
- const outcome = localSaves ? 'dropped' : landed ? 'landed' : 'withdrawn';
1498
- journal.record(`outcome ${locator} ${batch.id} ${outcome}`);
1499
- }
1500
- replicas.set(locator, next);
1501
- return mergeChangeSets([
1502
- heldChange(locator, before ?? null, next, set, changed),
1503
- ...carried.changes,
1504
- ]);
1724
+ for (const { batch, landed: held } of unqueued) {
1725
+ const outcome = localSaves ? 'dropped' : held ? 'landed' : 'withdrawn';
1726
+ journal.record(`outcome ${locator} ${batch.id} ${outcome}`);
1727
+ }
1728
+ const changed = answer.changed[locator];
1729
+ const confirmedChanged = settling.commit?.published !== undefined ||
1730
+ settling.commit?.refreshed === true ||
1731
+ overrides !== undefined;
1732
+ recomposed(locator, before?.record, record);
1733
+ const { count } = committedRoom(locator, record, answer);
1734
+ replicas.set(locator, {
1735
+ record,
1736
+ queue: kept,
1737
+ overlaid: touched,
1738
+ count,
1739
+ confirmed: (before?.confirmed ?? 0) + (confirmedChanged ? 1 : 0),
1740
+ });
1741
+ return mergeChangeSets([
1742
+ changeSet(locator, {
1743
+ documents: changed === undefined ? [] : changed,
1744
+ room: settling.room,
1745
+ queue: queueChanged,
1746
+ drafts,
1747
+ }),
1748
+ ...carried.changes,
1749
+ ]);
1750
+ },
1505
1751
  };
1506
1752
  }
1507
- function heldChange(locator, before, next, set, changed) {
1508
- const held = replicas.get(locator) ?? null;
1509
- return held === next ? set : changeSet(locator, before, held, changed);
1510
- }
1511
1753
  function adoptModels(locator, record) {
1512
1754
  for (const [attached, model] of [
1513
1755
  ['collection', record.model],
@@ -1539,44 +1781,54 @@ export function composeRuntime(options) {
1539
1781
  selection: taken(answers?.selection, held.selection, 'typed-selection'),
1540
1782
  };
1541
1783
  }
1542
- function publish(local, room, confirmed, landing, live) {
1543
- const { base, stored } = landing;
1784
+ function publish(local, room, record, landing, live) {
1544
1785
  return exclusive(async () => {
1545
- if (stored !== undefined) {
1546
- await indexReferrers({ text: confirmed.record.documents, binaries: confirmed.record.binaries }, compositionOf(confirmed.record));
1547
- }
1548
1786
  if (!live()) {
1549
1787
  caughtUpTo(room.locator);
1550
1788
  return true;
1551
1789
  }
1552
1790
  const prior = replicas.get(room.locator);
1553
- if (base !== undefined &&
1554
- (prior?.confirmed.record.roomVersion !== base.record.roomVersion ||
1555
- prior.confirmed.record.revisions !== base.record.revisions)) {
1791
+ if (landing.kind === 'refreshed' &&
1792
+ (prior === undefined || prior.record.key !== record.key || prior.confirmed !== landing.base)) {
1793
+ await local.unrefresh(record.key);
1556
1794
  return false;
1557
1795
  }
1558
- adoptModels(room.locator, confirmed.record);
1796
+ adoptModels(room.locator, record);
1559
1797
  const queue = prior?.queue ?? queuesWithoutReplica.get(room.locator) ?? [];
1560
- const roomVersion = confirmed.record.roomVersion;
1798
+ const roomVersion = record.roomVersion;
1561
1799
  const accepted = queue.filter((batch) => {
1562
- const accepted = pendings.get(pendingKey(room.locator, batch.id))?.accepted;
1563
- return (accepted !== undefined && options.adapter.reached(room.locator, roomVersion, accepted));
1800
+ const at = pendings.get(pendingKey(room.locator, batch.id))?.accepted;
1801
+ return at !== undefined && options.adapter.reached(room.locator, roomVersion, at);
1564
1802
  });
1565
- let settled;
1803
+ let set;
1566
1804
  try {
1567
- settled = await settle(local, room.locator, prior, confirmed, queue.filter((batch) => !accepted.includes(batch)), { room: true, ...(stored === undefined ? {} : { stored }) });
1805
+ set = await settle(local, room.locator, prior, record, queue.filter((batch) => !accepted.includes(batch)), landing.kind === 'refreshed'
1806
+ ? {
1807
+ room: true,
1808
+ view: 'refreshed',
1809
+ commit: { refreshed: true, composition: compositionOf(record) },
1810
+ }
1811
+ : {
1812
+ room: true,
1813
+ view: landing.refreshed ? 'refreshed' : 'confirmed',
1814
+ commit: {
1815
+ published: { from: prior?.record.key ?? null },
1816
+ composition: compositionOf(record),
1817
+ ...(landing.refreshed ? { refreshed: true } : {}),
1818
+ },
1819
+ });
1568
1820
  }
1569
1821
  catch (error) {
1570
1822
  if (!exceeded(error))
1571
1823
  throw error;
1572
- await exceededOn(room.locator, grownBy(confirmed.record, prior?.confirmed.record));
1824
+ await exceededOn(room.locator, null);
1573
1825
  caughtUpTo(room.locator);
1574
1826
  return true;
1575
1827
  }
1576
- const set = settled();
1577
- if (stored !== undefined) {
1828
+ if (landing.kind === 'bootstrapped') {
1578
1829
  transfers.delete(room.locator);
1579
1830
  catching.delete(room.locator);
1831
+ sweep();
1580
1832
  }
1581
1833
  for (const batch of accepted)
1582
1834
  pendings.delete(pendingKey(room.locator, batch.id));
@@ -1587,8 +1839,6 @@ export function composeRuntime(options) {
1587
1839
  sourceRefusals.delete(room.locator);
1588
1840
  declarationRefusals.delete(room.locator);
1589
1841
  list(room, true);
1590
- if (prior === undefined)
1591
- prepared.add(room.locator);
1592
1842
  publishChangeSet(set);
1593
1843
  await unpair(local);
1594
1844
  moveTo({ ...status, ...attention() });
@@ -1601,7 +1851,6 @@ export function composeRuntime(options) {
1601
1851
  }
1602
1852
  async function savedRoom(locator) {
1603
1853
  const local = await openLocal();
1604
- await referenced(locator);
1605
1854
  const replica = replicas.get(locator);
1606
1855
  const room = rooms.get(locator);
1607
1856
  if (replica === undefined || room === undefined) {
@@ -1632,10 +1881,12 @@ export function composeRuntime(options) {
1632
1881
  const target = await savedRoom(locator);
1633
1882
  const saved = [];
1634
1883
  const files = [];
1884
+ const filed = read.filter(({ parts }) => parts.file !== undefined).map(({ path }) => path);
1885
+ const visible = filed.length === 0 ? undefined : await visibleOf(locator, filed);
1635
1886
  for (const { path, parts } of read) {
1636
1887
  const file = parts.file === undefined
1637
1888
  ? undefined
1638
- : await heldFile(parts.file.blob, binaryBase(target.replica.visible, path));
1889
+ : await heldFile(parts.file.blob, binaryBase(visible, path));
1639
1890
  if (parts.file !== undefined && file !== undefined) {
1640
1891
  files.push({ key: file.localId, blob: parts.file.blob });
1641
1892
  }
@@ -1655,8 +1906,173 @@ export function composeRuntime(options) {
1655
1906
  return [[path, { ...record, path: to, revision: roomDraftRevision() }]];
1656
1907
  }));
1657
1908
  }
1658
- function sideOf(locator, replica, room) {
1659
- const record = replica.confirmed.record;
1909
+ async function savingIn(replica, changes, extra, leaving = []) {
1910
+ const record = replica.record;
1911
+ const local = await openLocal();
1912
+ const paths = [
1913
+ ...extra,
1914
+ ...changes.flatMap((change) => change.kind === 'move' ? [change.path, change.to] : [change.path]),
1915
+ ];
1916
+ const moved = [
1917
+ ...leaving,
1918
+ ...changes.flatMap((change) => (change.kind === 'move' ? [change.path] : [])),
1919
+ ];
1920
+ const removed = changes.flatMap((change) => (change.kind === 'remove' ? [change.path] : []));
1921
+ const referring = removed.length === 0
1922
+ ? []
1923
+ : (await local.referrers(record.key, 'confirmed', removed, CHECKED_LINKS)).map(({ path }) => path);
1924
+ const confirmedLayer = await sliceAt(record, 'confirmed', replica.queue, [...paths, ...referring], moved);
1925
+ const { visible } = await replayed(record, confirmedLayer, replica.queue);
1926
+ const composition = compositionOf(record);
1927
+ return {
1928
+ record,
1929
+ confirmed: confirmedLayer,
1930
+ visible,
1931
+ *visibleOver(queue) {
1932
+ return (yield* replaying(confirmedLayer, queue, record.locator, composition)).visible;
1933
+ },
1934
+ };
1935
+ }
1936
+ async function unmodeledIn(record) {
1937
+ const composition = compositionFor(record);
1938
+ if (available(composition))
1939
+ return false;
1940
+ const local = await openLocal();
1941
+ const roots = record.typedSelection?.roots ?? [];
1942
+ if (composition.model.status === 'unavailable' &&
1943
+ (await local.typedWithin(record.key, 'visible', roots, false))) {
1944
+ return true;
1945
+ }
1946
+ return (composition.selection !== null &&
1947
+ composition.selection.model.status === 'unavailable' &&
1948
+ (await local.typedWithin(record.key, 'visible', roots, true)));
1949
+ }
1950
+ function textDiffers(left, right) {
1951
+ return ((left?.content ?? undefined) !== (right?.content ?? undefined) ||
1952
+ !sameBinary(left?.binary ?? undefined, right?.binary ?? undefined));
1953
+ }
1954
+ function textOf(entry) {
1955
+ if (entry === null)
1956
+ return undefined;
1957
+ return entry.binary !== null ? null : entry.content;
1958
+ }
1959
+ function walkOf(local, key, view) {
1960
+ return async (after) => {
1961
+ const page = await local.walk(key, view, after, WALK_BYTES);
1962
+ return {
1963
+ entries: page.entries.map(({ path, content, binary }) => ({
1964
+ path,
1965
+ content: binary !== null ? null : content,
1966
+ })),
1967
+ next: page.next,
1968
+ };
1969
+ };
1970
+ }
1971
+ function lookupOf(local, key, view) {
1972
+ return async (paths) => {
1973
+ if (paths.length === 0)
1974
+ return new Map();
1975
+ return new Map((await local.typed(key, view, paths)).map(({ path, type, binary }) => [
1976
+ path,
1977
+ { type: type ?? undefined, binary },
1978
+ ]));
1979
+ };
1980
+ }
1981
+ function declaredOf(entry) {
1982
+ if (entry.binary !== null)
1983
+ return { type: undefined, binary: true };
1984
+ const read = parseFrontmatter(entry.content ?? '');
1985
+ return { type: read.ok ? (read.frontmatter.type ?? 'any') : undefined, binary: false };
1986
+ }
1987
+ function overriddenWalk(walk, overrides) {
1988
+ const sorted = [...overrides.keys()].sort();
1989
+ const beyond = (path) => {
1990
+ if (path === null)
1991
+ return 0;
1992
+ let low = 0;
1993
+ let high = sorted.length;
1994
+ while (low < high) {
1995
+ const middle = (low + high) >>> 1;
1996
+ if (sorted[middle] <= path)
1997
+ low = middle + 1;
1998
+ else
1999
+ high = middle;
2000
+ }
2001
+ return low;
2002
+ };
2003
+ return async (after) => {
2004
+ const page = await walk(after);
2005
+ const end = page.next;
2006
+ const added = sorted
2007
+ .slice(beyond(after), end === null ? sorted.length : beyond(end))
2008
+ .flatMap((path) => {
2009
+ const entry = overrides.get(path);
2010
+ return entry === null || entry === undefined
2011
+ ? []
2012
+ : [{ path, content: entry.binary !== null ? null : entry.content }];
2013
+ });
2014
+ const entries = [...page.entries.filter(({ path }) => !overrides.has(path)), ...added].sort((left, right) => (left.path < right.path ? -1 : left.path > right.path ? 1 : 0));
2015
+ return { entries, next: end };
2016
+ };
2017
+ }
2018
+ function overriddenLookup(lookup, overrides) {
2019
+ return async (paths) => {
2020
+ const known = new Map(await lookup(paths.filter((path) => !overrides.has(path))));
2021
+ for (const path of paths) {
2022
+ const entry = overrides.get(path);
2023
+ if (entry === undefined)
2024
+ continue;
2025
+ if (entry === null)
2026
+ known.delete(path);
2027
+ else
2028
+ known.set(path, declaredOf(entry));
2029
+ }
2030
+ return known;
2031
+ };
2032
+ }
2033
+ function validatorOf(record, confirmedLayer) {
2034
+ return async (resulting, touched) => {
2035
+ const composition = compositionFor(record);
2036
+ const local = await openLocal();
2037
+ const changed = [];
2038
+ const overrides = new Map();
2039
+ const pace = paced();
2040
+ for (const path of new Set([...confirmedLayer.held, ...touched])) {
2041
+ await pace(ROW_WEIGHT);
2042
+ const was = entryAt(confirmedLayer, path);
2043
+ const now = entryAt(resulting, path);
2044
+ if (!textDiffers(was, now))
2045
+ continue;
2046
+ overrides.set(path, now);
2047
+ if (textOf(was) !== textOf(now))
2048
+ changed.push({ path, was: textOf(was), now: textOf(now) });
2049
+ }
2050
+ const lookup = overriddenLookup(lookupOf(local, record.key, 'confirmed'), overrides);
2051
+ const clean = available(composition) &&
2052
+ changed.length <= CLEAN_CHANGES &&
2053
+ (await changedCleanly(changed, compositionOf(record), composition, {
2054
+ lookup,
2055
+ texts: textsOf(local, record.key, 'confirmed'),
2056
+ referrers: referrersOf(local, record.key, 'confirmed'),
2057
+ }));
2058
+ if (clean)
2059
+ return { verdict: 'accept' };
2060
+ return validatedCollection(overriddenWalk(walkOf(local, record.key, 'confirmed'), overrides), lookup, composition);
2061
+ };
2062
+ }
2063
+ function textsOf(local, key, view) {
2064
+ return async (paths) => new Map((await local.entries(key, view, paths)).map(({ path, content, binary }) => [
2065
+ path,
2066
+ binary !== null ? null : content,
2067
+ ]));
2068
+ }
2069
+ function referrersOf(local, key, view) {
2070
+ return async (targets) => [
2071
+ ...new Set((await local.referrers(key, view, targets, CHECKED_LINKS)).map(({ path }) => path)),
2072
+ ];
2073
+ }
2074
+ async function sideOf(locator, replica, room, saving) {
2075
+ const record = replica.record;
1660
2076
  return {
1661
2077
  room: locator,
1662
2078
  handle: room.handle,
@@ -1664,37 +2080,71 @@ export function composeRuntime(options) {
1664
2080
  composition: compositionFor(record),
1665
2081
  operations: room.operations,
1666
2082
  providerOwnedPaths: room.providerOwnedPaths,
1667
- visible: replica.visible,
2083
+ visible: saving.visible,
1668
2084
  queued: queuedDocuments(replica.queue),
1669
- clean: {
1670
- text: record.documents,
1671
- binaries: record.binaries,
1672
- composition: compositionOf(record),
1673
- },
2085
+ unmodeled: await unmodeledIn(record),
2086
+ validate: validatorOf(record, saving.confirmed),
1674
2087
  };
1675
2088
  }
1676
2089
  function movedInto(locator, saved) {
1677
2090
  const into = saved.find(({ parts }) => parts.move?.room !== undefined && parts.move.room !== locator);
1678
2091
  return into?.parts.move?.room ?? null;
1679
2092
  }
2093
+ async function saveCommit(local, files, rooms) {
2094
+ const touched = new Map();
2095
+ const commits = [];
2096
+ for (const { locator, saving, before, queue } of rooms) {
2097
+ const replayedLayer = await replayed(saving.record, saving.confirmed, queue);
2098
+ const diff = await overlayDiff(saving.record, before, replayedLayer.visible, replayedLayer.touched);
2099
+ touched.set(locator, diff.touched);
2100
+ commits.push({
2101
+ locator,
2102
+ key: saving.record.key,
2103
+ roots: saving.record.typedSelection?.roots ?? [],
2104
+ overlay: diff.overlay,
2105
+ });
2106
+ }
2107
+ try {
2108
+ const answer = await local.commit({
2109
+ rooms: commits,
2110
+ writes: rooms.map(({ write }) => write),
2111
+ files,
2112
+ whole: true,
2113
+ });
2114
+ return { answer, touched };
2115
+ }
2116
+ catch {
2117
+ return null;
2118
+ }
2119
+ }
2120
+ function savedReplica(locator, before, queue, touched, answer) {
2121
+ replicas.set(locator, {
2122
+ ...before,
2123
+ queue,
2124
+ overlaid: touched,
2125
+ count: answer.counts[locator] ?? before.count,
2126
+ });
2127
+ }
1680
2128
  async function committedMove(locator, destination, { local, replica, room }, saved, consumed, writtenBy, files) {
1681
2129
  const mixed = saved.find(({ parts }) => Object.keys(parts).length !== 1 || parts.move?.room !== destination);
1682
2130
  if (mixed !== undefined) {
1683
2131
  return { outcome: 'refused', code: 'mixed-room-move', retryable: false, file: mixed.path };
1684
2132
  }
1685
- const changed = saved.flatMap(({ path, parts }) => changedParts(path, underneath(parts, path, replica.visible)));
2133
+ const moves = saved.map(({ path, parts }) => ({ path, to: moveTarget(parts.move) }));
2134
+ const sourceSaving = await savingIn(replica, moves.map(({ path }) => ({ kind: 'remove', path })), saved.map(({ path }) => path), moves.map(({ path }) => path));
2135
+ const changed = saved.flatMap(({ path, parts }) => changedParts(path, underneath(parts, path, sourceSaving.visible)));
1686
2136
  if (changed.length > 0)
1687
2137
  return { outcome: 'refused', code: 'changed-underneath', retryable: false, parts: changed };
1688
- await referenced(destination);
1689
2138
  const target = replicas.get(destination);
1690
2139
  const listedRoom = rooms.get(destination);
1691
2140
  if (target === undefined || listedRoom === undefined) {
1692
2141
  throw runtimeError('room-not-held', `No Local Replica for ${destination}`);
1693
2142
  }
2143
+ const arriving = await savingIn(target, [], moves.map(({ to }) => to));
1694
2144
  const built = await buildMove({
1695
- source: sideOf(locator, replica, room),
1696
- destination: sideOf(destination, target, listedRoom),
1697
- moves: saved.map(({ path, parts }) => ({ path, to: moveTarget(parts.move) })),
2145
+ source: await sideOf(locator, replica, room, sourceSaving),
2146
+ destination: await sideOf(destination, target, listedRoom, arriving),
2147
+ moves,
1698
2148
  providerLimits: limits,
1699
2149
  local: localSaves,
1700
2150
  ...(writtenBy === undefined ? {} : { writtenBy }),
@@ -1721,35 +2171,38 @@ export function composeRuntime(options) {
1721
2171
  const left = [...consumed, ...carrying.map(({ from }) => from)];
1722
2172
  const leavingQueue = [...replica.queue, built.leaving];
1723
2173
  const arrivingQueue = [...target.queue, built.arriving];
1724
- let fencedRoom;
1725
- try {
1726
- fencedRoom = await local.saveAuthored(files, {
2174
+ const committedBoth = await saveCommit(local, files, [
2175
+ {
1727
2176
  locator,
2177
+ saving: sourceSaving,
2178
+ before: replica,
1728
2179
  queue: leavingQueue,
1729
- saved: new Map([[id, saved]]),
1730
- consumedRoomDrafts: left,
1731
- }, {
2180
+ write: {
2181
+ locator,
2182
+ queue: leavingQueue,
2183
+ saved: new Map([[id, saved]]),
2184
+ consumedRoomDrafts: left,
2185
+ },
2186
+ },
2187
+ {
1732
2188
  locator: destination,
2189
+ saving: arriving,
2190
+ before: target,
1733
2191
  queue: arrivingQueue,
1734
- saved: new Map([[id, marked]]),
1735
- roomDrafts: carried,
1736
- });
1737
- }
1738
- catch {
2192
+ write: {
2193
+ locator: destination,
2194
+ queue: arrivingQueue,
2195
+ saved: new Map([[id, marked]]),
2196
+ roomDrafts: carried,
2197
+ },
2198
+ },
2199
+ ]);
2200
+ if (committedBoth === null)
1739
2201
  return { outcome: 'refused', code: 'durable-write', retryable: true };
2202
+ const { answer, touched } = committedBoth;
2203
+ if (answer.fenced !== null) {
2204
+ throw runtimeError('room-not-held', `No Local Replica for ${answer.fenced}`);
1740
2205
  }
1741
- if (fencedRoom !== null) {
1742
- throw runtimeError('room-not-held', `No Local Replica for ${fencedRoom}`);
1743
- }
1744
- const nextSource = await stepped(replicating(replica.confirmed, leavingQueue, replica));
1745
- const nextDestination = await stepped(replicating(target.confirmed, arrivingQueue, target));
1746
- const sets = [
1747
- await stepped(changing(locator, replica, nextSource, { queue: true, drafts: left })),
1748
- await stepped(changing(destination, target, nextDestination, {
1749
- queue: true,
1750
- drafts: carried.map(({ path }) => path),
1751
- })),
1752
- ];
1753
2206
  if (fences.has(locator) || fences.has(destination)) {
1754
2207
  throw runtimeError('room-not-held', `No Local Replica for ${locator}`);
1755
2208
  }
@@ -1767,9 +2220,20 @@ export function composeRuntime(options) {
1767
2220
  forgetRoomDraft(locator, path);
1768
2221
  for (const record of carried)
1769
2222
  rememberRoomDraft(record);
1770
- replicas.set(locator, nextSource);
1771
- replicas.set(destination, nextDestination);
1772
- publishChangeSet(mergeChangeSets(sets));
2223
+ savedReplica(locator, replica, leavingQueue, touched.get(locator), answer);
2224
+ savedReplica(destination, target, arrivingQueue, touched.get(destination), answer);
2225
+ publishChangeSet(mergeChangeSets([
2226
+ changeSet(locator, {
2227
+ documents: answer.changed[locator] ?? [],
2228
+ queue: true,
2229
+ drafts: left,
2230
+ }),
2231
+ changeSet(destination, {
2232
+ documents: answer.changed[destination] ?? [],
2233
+ queue: true,
2234
+ drafts: carried.map(({ path }) => path),
2235
+ }),
2236
+ ]));
1773
2237
  queuedSave(locator);
1774
2238
  return { outcome: 'saved', identifier: id };
1775
2239
  }
@@ -1785,15 +2249,16 @@ export function composeRuntime(options) {
1785
2249
  return committedMove(locator, into, target, saved, consumed, writtenBy, files);
1786
2250
  }
1787
2251
  const { local, replica, room } = target;
1788
- const changed = saved.flatMap(({ path, parts }) => changedParts(path, underneath(parts, path, replica.visible)));
2252
+ const parts = saved.map(({ path, parts: held }) => changesOf(path, held));
2253
+ const changes = parts.flatMap(({ changes: made }) => made);
2254
+ const saving = await savingIn(replica, changes, [
2255
+ ...saved.map(({ path }) => path),
2256
+ ...parts.flatMap(({ bases }) => bases.map(({ path }) => path)),
2257
+ ]);
2258
+ const changed = saved.flatMap(({ path, parts: held }) => changedParts(path, underneath(held, path, saving.visible)));
1789
2259
  if (changed.length > 0)
1790
2260
  return { outcome: 'refused', code: 'changed-underneath', retryable: false, parts: changed };
1791
- const parts = saved.map(({ path, parts }) => changesOf(path, parts));
1792
- const changes = parts.flatMap(({ changes }) => changes);
1793
- const visibleOver = function* (queue) {
1794
- return (yield* replicating(replica.confirmed, queue, replica)).visible;
1795
- };
1796
- const requeue = await stepped(requeuing(replica.queue, changes, replica.visible, withdrawable(locator), visibleOver));
2261
+ const requeue = await stepped(requeuing(replica.queue, changes, saving.visible, withdrawable(locator), saving.visibleOver));
1797
2262
  if (requeue.kind === 'refused')
1798
2263
  return { outcome: 'refused', ...requeue.refusal };
1799
2264
  const foreign = [...requeue.withdrawn.keys()].some((origin) => {
@@ -1811,22 +2276,19 @@ export function composeRuntime(options) {
1811
2276
  const withdrawing = requeue.withdrawn.size > 0;
1812
2277
  const input = {
1813
2278
  local: localSaves,
1814
- baseRoomVersion: replica.confirmed.record.roomVersion,
1815
- composition: compositionFor(replica.confirmed.record),
2279
+ baseRoomVersion: replica.record.roomVersion,
2280
+ composition: compositionFor(replica.record),
1816
2281
  operations: room.operations,
1817
2282
  providerOwnedPaths: room.providerOwnedPaths,
1818
2283
  providerLimits: limits,
1819
2284
  handle: room.handle,
1820
- visible: replica.visible,
2285
+ visible: saving.visible,
1821
2286
  changes,
1822
2287
  bases: parts.flatMap(({ bases }) => bases),
1823
2288
  queued: queuedDocuments(replica.queue),
1824
2289
  writtenBy,
1825
- clean: {
1826
- text: replica.confirmed.record.documents,
1827
- binaries: replica.confirmed.record.binaries,
1828
- composition: compositionOf(replica.confirmed.record),
1829
- },
2290
+ unmodeled: await unmodeledIn(replica.record),
2291
+ validate: validatorOf(replica.record, saving.confirmed),
1830
2292
  };
1831
2293
  const checked = withdrawing
1832
2294
  ? await buildBatch({ ...input, queued: new Set([...input.queued, ...requeue.queued]) })
@@ -1851,8 +2313,7 @@ export function composeRuntime(options) {
1851
2313
  ? null
1852
2314
  : await buildBatch({
1853
2315
  ...input,
1854
- visible: (await stepped(replicating(replica.confirmed, requeue.queue, replica)))
1855
- .visible,
2316
+ visible: await stepped(saving.visibleOver(requeue.queue)),
1856
2317
  changes: requeue.changes,
1857
2318
  queued: queuedDocuments(requeue.queue),
1858
2319
  });
@@ -1885,34 +2346,34 @@ export function composeRuntime(options) {
1885
2346
  for (const replaced of withdrawn.values()) {
1886
2347
  if (replaced === null || !batches.includes(replaced))
1887
2348
  continue;
1888
- const limited = await stepped(limiting(batches, replaced, limits, visibleOver));
2349
+ const limited = await stepped(limiting(batches, replaced, limits, saving.visibleOver));
1889
2350
  if (limited !== null)
1890
2351
  return { outcome: 'refused', ...limited };
1891
2352
  }
1892
2353
  const carried = carriedDrafts(locator, saved, consumed);
1893
2354
  const left = [...consumed, ...carried.keys()];
1894
2355
  const arrived = [...carried.values()];
1895
- let fencedRoom;
1896
- try {
1897
- fencedRoom = await local.saveAuthored(files, {
2356
+ const done = await saveCommit(local, files, [
2357
+ {
1898
2358
  locator,
2359
+ saving,
2360
+ before: replica,
1899
2361
  queue: batches,
1900
- saved: drafted,
1901
- consumedRoomDrafts: left,
1902
- roomDrafts: arrived,
1903
- });
1904
- }
1905
- catch {
2362
+ write: {
2363
+ locator,
2364
+ queue: batches,
2365
+ saved: drafted,
2366
+ consumedRoomDrafts: left,
2367
+ roomDrafts: arrived,
2368
+ },
2369
+ },
2370
+ ]);
2371
+ if (done === null)
1906
2372
  return { outcome: 'refused', code: 'durable-write', retryable: true };
2373
+ const { answer, touched } = done;
2374
+ if (answer.fenced !== null) {
2375
+ throw runtimeError('room-not-held', `No Local Replica for ${answer.fenced}`);
1907
2376
  }
1908
- if (fencedRoom !== null) {
1909
- throw runtimeError('room-not-held', `No Local Replica for ${fencedRoom}`);
1910
- }
1911
- const next = await stepped(replicating(replica.confirmed, batches, replica));
1912
- const set = await stepped(changing(locator, replica, next, {
1913
- queue: true,
1914
- drafts: [...left, ...arrived.map(({ path }) => path)],
1915
- }));
1916
2377
  if (fences.has(locator))
1917
2378
  throw runtimeError('room-not-held', `No Local Replica for ${locator}`);
1918
2379
  for (const origin of withdrawn.keys())
@@ -1928,8 +2389,12 @@ export function composeRuntime(options) {
1928
2389
  forgetRoomDraft(locator, path);
1929
2390
  for (const record of arrived)
1930
2391
  rememberRoomDraft(record);
1931
- replicas.set(locator, next);
1932
- publishChangeSet(set);
2392
+ savedReplica(locator, replica, batches, touched.get(locator), answer);
2393
+ publishChangeSet(changeSet(locator, {
2394
+ documents: answer.changed[locator] ?? [],
2395
+ queue: true,
2396
+ drafts: [...left, ...arrived.map(({ path }) => path)],
2397
+ }));
1933
2398
  queuedSave(locator);
1934
2399
  const identifier = join?.batch.id ??
1935
2400
  built?.batch.id ??
@@ -1949,9 +2414,8 @@ export function composeRuntime(options) {
1949
2414
  if (fences.has(held))
1950
2415
  continue;
1951
2416
  pendings.set(pendingKey(held, batch.id), { ...state, submission, lost });
1952
- const replica = replicas.get(held);
1953
- if (replica !== undefined)
1954
- publishChangeSet(changeSet(held, replica, replica, { queue: true }));
2417
+ if (replicas.has(held))
2418
+ publishChangeSet(changeSet(held, { queue: true }));
1955
2419
  }
1956
2420
  moveTo({ ...status, ...attention() });
1957
2421
  }
@@ -1995,7 +2459,7 @@ export function composeRuntime(options) {
1995
2459
  if (fencedSince(locator, since))
1996
2460
  return failure({ kind: 'room-not-readable', locator });
1997
2461
  if (result.kind === 'scoped-refusal') {
1998
- const room = replicas.get(result.locator)?.confirmed.record;
2462
+ const room = replicas.get(result.locator)?.record;
1999
2463
  if (room !== undefined)
2000
2464
  await scoped(result, room, await openLocal());
2001
2465
  }
@@ -2026,8 +2490,9 @@ export function composeRuntime(options) {
2026
2490
  return UNREADABLE;
2027
2491
  if (locked(at.room))
2028
2492
  return LOCKED_UNREADABLE;
2029
- const visible = replicas.get(at.room)?.visible;
2030
- if (visible?.documents[at.path] !== undefined || visible?.binaries[at.path] !== undefined) {
2493
+ const held = replicas.get(at.room);
2494
+ if (held !== undefined &&
2495
+ (await (await openLocal()).entries(held.record.key, 'visible', [at.path])).length > 0) {
2031
2496
  return { kind: 'document', ...at };
2032
2497
  }
2033
2498
  const history = await readHistory(at.room, at.path);
@@ -2085,7 +2550,7 @@ export function composeRuntime(options) {
2085
2550
  if (locator !== null && fencedSince(locator, since))
2086
2551
  return failure({ kind: 'room-not-readable', locator });
2087
2552
  if (settled.kind === 'scoped-refusal') {
2088
- const room = replicas.get(settled.locator)?.confirmed.record;
2553
+ const room = replicas.get(settled.locator)?.record;
2089
2554
  if (room !== undefined)
2090
2555
  await scoped(settled, room, await openLocal());
2091
2556
  }
@@ -2106,36 +2571,43 @@ export function composeRuntime(options) {
2106
2571
  return held;
2107
2572
  return transferBinary(request, (authorization) => keptFiles.fetch(name, () => options.adapter.binary(authorization, request)), binaryAnswer);
2108
2573
  }
2109
- function binaryPath(locator, path, revision) {
2574
+ async function binaryPath(locator, path, revision) {
2110
2575
  if (revision !== undefined)
2111
2576
  return { kind: 'path', locator, path, revision };
2112
- const visible = replicas.get(locator)?.visible;
2113
- const held = visible?.binaries[path];
2114
- if (visible === undefined || held === undefined)
2577
+ const replica = replicas.get(locator);
2578
+ if (replica === undefined)
2115
2579
  return null;
2116
- if (held.state === 'pending')
2117
- return { kind: 'pending', localId: held.localId };
2118
- const record = binaryRecord(path, held, visible.revisions[path]);
2580
+ const local = await openLocal();
2581
+ const [held] = await local.entries(replica.record.key, 'visible', [path]);
2582
+ if (held?.binary === undefined || held.binary === null)
2583
+ return null;
2584
+ if (held.binary.state === 'pending')
2585
+ return { kind: 'pending', localId: held.binary.localId };
2586
+ const record = binaryRecord(path, held.binary, held.revision ?? undefined);
2119
2587
  if (record.state !== 'accepted')
2120
2588
  return null;
2121
- const replica = replicas.get(locator);
2122
- const queued = (replica?.queue ?? []).flatMap((batch) => batch.changes);
2589
+ const queued = replica.queue.flatMap((batch) => batch.changes);
2590
+ const sources = queued.flatMap((change) => (change.kind === 'move' ? [change.path] : []));
2591
+ const confirmedLayer = await stepped(slicing(await local.entries(replica.record.key, 'confirmed', [path, ...sources]), [
2592
+ path,
2593
+ ...sources,
2594
+ ]));
2123
2595
  const holds = (at) => {
2124
- const kept = replica?.confirmed.record.binaries[at];
2596
+ const kept = confirmedLayer.binaries[at];
2125
2597
  return (kept?.state === 'accepted' &&
2126
2598
  kept.fileRef === record.fileRef &&
2127
- replica?.confirmed.record.revisions[at] === record.revision);
2599
+ confirmedLayer.revisions[at] === record.revision);
2128
2600
  };
2129
- let confirmed = path;
2130
- for (let index = queued.length - 1; index >= 0 && !holds(confirmed); index -= 1) {
2601
+ let confirmedAt = path;
2602
+ for (let index = queued.length - 1; index >= 0 && !holds(confirmedAt); index -= 1) {
2131
2603
  const change = queued[index];
2132
- if (change.kind === 'move' && change.to === confirmed)
2133
- confirmed = change.path;
2604
+ if (change.kind === 'move' && change.to === confirmedAt)
2605
+ confirmedAt = change.path;
2134
2606
  }
2135
2607
  return {
2136
2608
  kind: 'path',
2137
2609
  locator,
2138
- path: confirmed,
2610
+ path: confirmedAt,
2139
2611
  revision: record.revision,
2140
2612
  fileRef: record.fileRef,
2141
2613
  };
@@ -2145,7 +2617,7 @@ export function composeRuntime(options) {
2145
2617
  const drafted = roomDrafts.get(locator)?.get(path)?.parts.file;
2146
2618
  const asked = drafted !== undefined && drafted.localId === revision
2147
2619
  ? { kind: 'pending', localId: revision }
2148
- : binaryPath(locator, path, revision);
2620
+ : await binaryPath(locator, path, revision);
2149
2621
  if (asked === null)
2150
2622
  return absentBinary(path);
2151
2623
  if (asked.kind === 'pending') {
@@ -2163,7 +2635,7 @@ export function composeRuntime(options) {
2163
2635
  }
2164
2636
  async function linkBinaryPath(locator, path, revision) {
2165
2637
  await ready;
2166
- const asked = binaryPath(locator, path, revision);
2638
+ const asked = await binaryPath(locator, path, revision);
2167
2639
  if (asked === null || asked.kind === 'pending')
2168
2640
  return absentBinary(path);
2169
2641
  return transferBinary(asked, (held) => options.adapter.link(held, asked), binaryLinkAnswer);
@@ -2224,7 +2696,7 @@ export function composeRuntime(options) {
2224
2696
  try {
2225
2697
  const result = await options.adapter.action(authorization, { name, input });
2226
2698
  if (result.kind === 'scoped-refusal') {
2227
- const room = replicas.get(result.locator)?.confirmed.record;
2699
+ const room = replicas.get(result.locator)?.record;
2228
2700
  if (room !== undefined)
2229
2701
  await scoped(result, room, await openLocal());
2230
2702
  }
@@ -2299,85 +2771,277 @@ export function composeRuntime(options) {
2299
2771
  }
2300
2772
  function reachedBy(locator, roomVersion) {
2301
2773
  const held = replicas.get(locator);
2302
- return (held !== undefined &&
2303
- options.adapter.reached(locator, held.confirmed.record.roomVersion, roomVersion));
2774
+ return (held !== undefined && options.adapter.reached(locator, held.record.roomVersion, roomVersion));
2304
2775
  }
2305
- function writtenAfter(locator, roomVersion, path) {
2306
- const written = replicas.get(locator)?.confirmed.record.written[path];
2776
+ async function writtenAfter(locator, roomVersion, path) {
2777
+ const held = replicas.get(locator);
2778
+ if (held === undefined)
2779
+ return false;
2780
+ const [entry] = await (await openLocal()).entries(held.record.key, 'confirmed', [path]);
2781
+ const written = entry?.written ?? undefined;
2307
2782
  return (written !== undefined &&
2308
2783
  options.adapter.reached(locator, written, roomVersion) &&
2309
2784
  !options.adapter.reached(locator, roomVersion, written));
2310
2785
  }
2311
- async function pullRefresh(authorized, confirmed, live) {
2312
- if (!live())
2786
+ function overriddenTexts(texts, overrides) {
2787
+ return async (paths) => {
2788
+ const known = new Map(await texts(paths.filter((path) => !overrides.has(path))));
2789
+ for (const path of paths) {
2790
+ const entry = overrides.get(path);
2791
+ if (entry === undefined)
2792
+ continue;
2793
+ if (entry === null)
2794
+ known.delete(path);
2795
+ else
2796
+ known.set(path, entry.binary !== null ? null : entry.content);
2797
+ }
2798
+ return known;
2799
+ };
2800
+ }
2801
+ async function unreadableRoom(texts, walk, paths) {
2802
+ if (paths !== null) {
2803
+ const text = {};
2804
+ for (const [path, content] of await texts(paths))
2805
+ if (content !== null)
2806
+ text[path] = content;
2807
+ return stepped(unreadableIn(text, Object.keys(text)));
2808
+ }
2809
+ for (let after = null;;) {
2810
+ const page = await walk(after);
2811
+ const text = {};
2812
+ for (const { path, content } of page.entries)
2813
+ if (content !== null)
2814
+ text[path] = content;
2815
+ const refused = await stepped(unreadableIn(text, Object.keys(text)));
2816
+ if (refused !== null)
2817
+ return refused;
2818
+ if (page.next === null)
2819
+ return null;
2820
+ after = page.next;
2821
+ }
2822
+ }
2823
+ async function validatedRoom(local, record, view, previous, overrides = null) {
2824
+ const composition = compositionOf(record);
2825
+ const modeled = record.model.status !== 'unavailable' || record.selectionModel.status !== 'unavailable';
2826
+ const lookup = overrides === null
2827
+ ? lookupOf(local, record.key, view)
2828
+ : overriddenLookup(lookupOf(local, record.key, view), overrides);
2829
+ const texts = overrides === null
2830
+ ? textsOf(local, record.key, view)
2831
+ : overriddenTexts(textsOf(local, record.key, view), overrides);
2832
+ const walk = overrides === null
2833
+ ? walkOf(local, record.key, view)
2834
+ : overriddenWalk(walkOf(local, record.key, view), overrides);
2835
+ let changes = null;
2836
+ let reshaped = false;
2837
+ if (previous !== null && previous.changed.length <= CLEAN_CHANGES) {
2838
+ const [was, now] = await Promise.all([
2839
+ local.entries(record.key, 'confirmed', previous.changed),
2840
+ overrides === null
2841
+ ? local.entries(record.key, view, previous.changed)
2842
+ : Promise.resolve([]),
2843
+ ]);
2844
+ const before = new Map(was.map((entry) => [entry.path, entry]));
2845
+ const after = overrides === null
2846
+ ? new Map(now.map((entry) => [entry.path, entry]))
2847
+ : overrides;
2848
+ changes = previous.changed.flatMap((path) => {
2849
+ const one = before.get(path) ?? null;
2850
+ const other = after.get(path) ?? null;
2851
+ reshaped ||= textDiffers(one, other);
2852
+ const held = textOf(one);
2853
+ const next = textOf(other);
2854
+ return held === next ? [] : [{ path, was: held, now: next }];
2855
+ });
2856
+ }
2857
+ const named = changes === null ? null : changes.map(({ path }) => path);
2858
+ const textChanged = changes === null || changes.length > 0 || reshaped;
2859
+ const modelsChanged = previous === null || !sameComposition(compositionOf(previous.record), composition);
2860
+ if (modeled && (textChanged || modelsChanged)) {
2861
+ const clean = previous !== null &&
2862
+ changes !== null &&
2863
+ (await changedCleanly(changes, compositionOf(previous.record), composition, {
2864
+ lookup,
2865
+ texts,
2866
+ referrers: referrersOf(local, record.key, 'confirmed'),
2867
+ }));
2868
+ if (clean)
2869
+ return null;
2870
+ const verdict = await validatedCollection(walk, lookup, composition);
2871
+ if (verdict.verdict === 'accept')
2872
+ return null;
2873
+ if (verdict.rule !== 'model-unavailable')
2874
+ return { kind: 'refused', refusal: verdict };
2875
+ return unreadableRoom(texts, walk, named);
2876
+ }
2877
+ if (textChanged)
2878
+ return unreadableRoom(texts, walk, named);
2879
+ return null;
2880
+ }
2881
+ function refreshingOf(local, key, liveness) {
2882
+ return {
2883
+ prior: async (paths) => new Map((await local.entries(key, 'refreshed', paths)).map((entry) => [entry.path, entry])),
2884
+ async stage(page) {
2885
+ const batches = batched(page.entries);
2886
+ for (const [index, entries] of batches.entries()) {
2887
+ const last = index === batches.length - 1;
2888
+ const staged = await exclusive(async () => {
2889
+ if (!liveness())
2890
+ return false;
2891
+ await local.refresh(key, last
2892
+ ? { ...page, entries }
2893
+ : { entries, removals: [], folders: [], folderRemovals: [] });
2894
+ return true;
2895
+ });
2896
+ if (!staged)
2897
+ return false;
2898
+ }
2899
+ return true;
2900
+ },
2901
+ };
2902
+ }
2903
+ function batched(entries) {
2904
+ const batches = [[]];
2905
+ let spent = 0;
2906
+ for (const entry of entries) {
2907
+ const size = (entry.content?.length ?? 0) + ROW_WEIGHT;
2908
+ if (spent + size > STAGE_BYTES && batches.at(-1).length > 0) {
2909
+ batches.push([]);
2910
+ spent = 0;
2911
+ }
2912
+ batches.at(-1).push(entry);
2913
+ spent += size;
2914
+ }
2915
+ return batches;
2916
+ }
2917
+ async function pullRefresh(authorized, held, liveness) {
2918
+ if (!liveness())
2313
2919
  return unconfirmed;
2314
2920
  const turn = generation;
2921
+ const { record } = held;
2922
+ const base = held.confirmed;
2923
+ const local = await openLocal();
2315
2924
  try {
2316
- const result = await refreshRoom(reads, authorized, confirmed, (models, held) => (live() ? remodel(confirmed.record.locator, models, held) : held), live);
2317
- const detail = result.kind === 'advanced' ? result.confirmed.record.roomVersion : result.kind;
2318
- journal.record(`transfer refresh ${confirmed.record.locator} ${detail}`);
2319
- return result;
2925
+ await exclusive(() => local.unrefresh(record.key));
2926
+ const result = await advancement(reads, authorized, record, (models, kept) => (liveness() ? remodel(record.locator, models, kept) : kept), liveness, refreshingOf(local, record.key, liveness));
2927
+ if (result.kind !== 'advanced') {
2928
+ journal.record(`transfer refresh ${record.locator} ${result.kind}`);
2929
+ return result;
2930
+ }
2931
+ const refused = await validatedRoom(local, result.record, 'refreshed', {
2932
+ record,
2933
+ changed: [...result.changed],
2934
+ });
2935
+ if (refused !== null) {
2936
+ journal.record(`transfer refresh ${record.locator} refused`);
2937
+ return refused;
2938
+ }
2939
+ journal.record(`transfer refresh ${record.locator} ${result.record.roomVersion}`);
2940
+ return { kind: 'advanced', record: result.record, base };
2320
2941
  }
2321
2942
  catch (error) {
2322
- journal.record(`transfer refresh ${confirmed.record.locator} transport-uncertainty`);
2943
+ journal.record(`transfer refresh ${record.locator} transport-uncertainty`);
2323
2944
  throw error;
2324
2945
  }
2325
2946
  finally {
2326
- reads.finished(confirmed.record.locator, turn);
2327
- }
2947
+ reads.finished(record.locator, turn);
2948
+ }
2949
+ }
2950
+ const heldContents = new Map();
2951
+ function heldContentOf(local, key) {
2952
+ const known = heldContents.get(key);
2953
+ if (known !== undefined)
2954
+ return known;
2955
+ if (heldContents.size > ROOM_READS)
2956
+ heldContents.clear();
2957
+ const made = {
2958
+ files: (after, limit) => local.files(key, 'confirmed', after, limit),
2959
+ async contents(paths) {
2960
+ return new Map((await local.entries(key, 'confirmed', paths)).flatMap(({ path, content, revision }) => content === null || revision === null ? [] : [[path, { content, revision }]]));
2961
+ },
2962
+ };
2963
+ heldContents.set(key, made);
2964
+ return made;
2328
2965
  }
2329
- function staging(local, live) {
2966
+ function staging(local, liveness) {
2330
2967
  return {
2331
2968
  held: (locator) => transfers.get(locator) ?? null,
2332
- staged: (record) => local.staged(record),
2333
- stage: (record, page) => exclusive(async () => {
2334
- if (!live())
2335
- return false;
2336
- const bytes = layerBytes(page);
2337
- if (!(await admitted(local, record.locator, bytes)))
2338
- return false;
2339
- try {
2340
- await local.stage(record, page);
2341
- }
2342
- catch (error) {
2343
- if (!exceeded(error))
2344
- throw error;
2345
- await exceededOn(record.locator, bytes);
2346
- return false;
2969
+ async stage(record, page, composition) {
2970
+ const bytes = pageBytes(page);
2971
+ const batches = batched(page.entries);
2972
+ let key = record.key;
2973
+ for (const [index, entries] of batches.entries()) {
2974
+ const last = index === batches.length - 1;
2975
+ const staged = await exclusive(async () => {
2976
+ if (!liveness())
2977
+ return null;
2978
+ if (index === 0 && !(await admitted(local, record.locator, bytes)))
2979
+ return null;
2980
+ try {
2981
+ return await local.stage(record, composition, { entries, folders: last ? page.folders : [] }, key, last);
2982
+ }
2983
+ catch (error) {
2984
+ if (!exceeded(error))
2985
+ throw error;
2986
+ await exceededOn(record.locator, bytes);
2987
+ return null;
2988
+ }
2989
+ });
2990
+ if (staged === null)
2991
+ return null;
2992
+ key = staged;
2347
2993
  }
2348
- transfers.set(record.locator, record);
2349
- readSide.recover();
2350
- return true;
2351
- }),
2994
+ transfers.set(record.locator, { ...record, key });
2995
+ liveQueries.recover();
2996
+ return key;
2997
+ },
2998
+ async refreshing(key) {
2999
+ await exclusive(() => local.unrefresh(key));
3000
+ return refreshingOf(local, key, liveness);
3001
+ },
2352
3002
  abandon: (locator) => exclusive(async () => {
2353
- if (!live())
3003
+ if (!liveness())
2354
3004
  return;
2355
3005
  await local.abandon(locator);
2356
3006
  transfers.delete(locator);
2357
- readSide.recover();
3007
+ liveQueries.recover();
3008
+ sweep();
2358
3009
  }),
3010
+ setAside: (locator) => exclusive(async () => {
3011
+ if (!liveness())
3012
+ return;
3013
+ await local.abandon(locator, true);
3014
+ transfers.delete(locator);
3015
+ liveQueries.recover();
3016
+ }),
3017
+ release: (key) => exclusive(async () => {
3018
+ await local.release(key);
3019
+ sweep();
3020
+ }),
3021
+ content: (key) => heldContentOf(local, key),
3022
+ validate: (record, view) => validatedRoom(local, record, view, null),
2359
3023
  };
2360
3024
  }
2361
- function reusable(room) {
2362
- const record = replicas.get(room.locator)?.confirmed.record;
3025
+ function reusable(room, local) {
3026
+ const record = replicas.get(room.locator)?.record;
2363
3027
  if (record === undefined || record.replica.selection !== room.replica.selection)
2364
3028
  return null;
2365
- return { documents: record.documents, revisions: record.revisions };
3029
+ return heldContentOf(local, record.key);
2366
3030
  }
2367
- async function pullBootstrap(authorized, room, local, live) {
2368
- if (!live())
3031
+ async function pullBootstrap(authorized, room, local, liveness) {
3032
+ if (!liveness())
2369
3033
  return unconfirmed;
2370
3034
  const turn = generation;
2371
3035
  if (replicas.has(room.locator)) {
2372
3036
  catching.add(room.locator);
2373
- readSide.recover();
3037
+ liveQueries.recover();
2374
3038
  }
2375
3039
  try {
2376
3040
  const result = await bootstrapRoom(reads, authorized, room, {
2377
3041
  collection: modelOf(room.locator),
2378
3042
  selection: modelOf(room.locator, 'typed-selection'),
2379
- }, (models, held) => (live() ? remodel(room.locator, models, held) : held), live, staging(local, live), reusable(room));
2380
- const detail = result.kind === 'staged' ? result.confirmed.record.roomVersion : result.kind;
3043
+ }, (models, held) => (liveness() ? remodel(room.locator, models, held) : held), liveness, staging(local, liveness), reusable(room, local));
3044
+ const detail = result.kind === 'staged' ? result.record.roomVersion : result.kind;
2381
3045
  journal.record(`transfer bootstrap ${room.locator} ${detail}`);
2382
3046
  if (result.kind !== 'staged')
2383
3047
  caughtUpTo(room.locator);
@@ -2394,10 +3058,10 @@ export function composeRuntime(options) {
2394
3058
  }
2395
3059
  function caughtUpTo(locator) {
2396
3060
  if (catching.delete(locator))
2397
- readSide.recover();
3061
+ liveQueries.recover();
2398
3062
  }
2399
3063
  function behind(room) {
2400
- const held = replicas.get(room.locator)?.confirmed.record.roomVersion;
3064
+ const held = replicas.get(room.locator)?.record.roomVersion;
2401
3065
  return (held !== undefined &&
2402
3066
  [announced.get(room.locator), room.roomVersion].some((later) => later !== undefined && !options.adapter.reached(room.locator, held, later)));
2403
3067
  }
@@ -2410,12 +3074,11 @@ export function composeRuntime(options) {
2410
3074
  return advanceOnce(room, local, authorized, live);
2411
3075
  }
2412
3076
  async function advanceOnce(room, local, authorized, live) {
2413
- await referenced(room.locator);
2414
3077
  if (!live() || paused.has(room.locator))
2415
3078
  return unconfirmed;
2416
3079
  const held = replicas.get(room.locator);
2417
3080
  if (held !== undefined) {
2418
- const refreshed = await pullRefresh(authorized, held.confirmed, live);
3081
+ const refreshed = await pullRefresh(authorized, held, live);
2419
3082
  if (!live())
2420
3083
  return unconfirmed;
2421
3084
  if (refreshed.kind === 'still-current') {
@@ -2430,10 +3093,11 @@ export function composeRuntime(options) {
2430
3093
  return null;
2431
3094
  }
2432
3095
  if (refreshed.kind === 'advanced') {
2433
- if (!(await publish(local, room, refreshed.confirmed, { base: held.confirmed }, live))) {
2434
- const now = replicas.get(room.locator)?.confirmed.record.roomVersion;
3096
+ const landing = { kind: 'refreshed', base: refreshed.base };
3097
+ if (!(await publish(local, room, refreshed.record, landing, live))) {
3098
+ const now = replicas.get(room.locator)?.record.roomVersion;
2435
3099
  const reached = now !== undefined &&
2436
- options.adapter.reached(room.locator, now, refreshed.confirmed.record.roomVersion);
3100
+ options.adapter.reached(room.locator, now, refreshed.record.roomVersion);
2437
3101
  return reached ? null : advance(room, local, authorized, live);
2438
3102
  }
2439
3103
  await retainModel(local, room, 'collection', modelOf(room.locator));
@@ -2443,7 +3107,7 @@ export function composeRuntime(options) {
2443
3107
  return refreshed;
2444
3108
  if (opened !== null) {
2445
3109
  owed.add(room.locator);
2446
- readSide.recover();
3110
+ liveQueries.recover();
2447
3111
  return unconfirmed;
2448
3112
  }
2449
3113
  if (reloadsDue.get(room.locator) === generation)
@@ -2454,7 +3118,7 @@ export function composeRuntime(options) {
2454
3118
  return unconfirmed;
2455
3119
  if (bootstrapped.kind !== 'staged')
2456
3120
  return bootstrapped;
2457
- await publish(local, room, bootstrapped.confirmed, { stored: bootstrapped.stored }, live);
3121
+ await publish(local, room, bootstrapped.record, { kind: 'bootstrapped', refreshed: bootstrapped.refreshed }, live);
2458
3122
  return null;
2459
3123
  }
2460
3124
  async function withhold(withheld, room, local, progress) {
@@ -2527,46 +3191,90 @@ export function composeRuntime(options) {
2527
3191
  (accepted.author === undefined ? '' : ` author ${accepted.author}`) +
2528
3192
  (accepted.time === undefined ? '' : ` time ${accepted.time}`));
2529
3193
  }
2530
- async function adoptAccepted(room, local, submitted, accepted) {
2531
- await referenced(room.locator);
3194
+ function settling(acceptance) {
3195
+ return acceptance !== null && 'apply' in acceptance;
3196
+ }
3197
+ function refusalOf(acceptance) {
3198
+ return settling(acceptance) ? (acceptance.refused ?? null) : acceptance;
3199
+ }
3200
+ async function acceptance(room, local, submitted, accepted) {
2532
3201
  if (!continuing())
2533
3202
  return unconfirmed;
2534
3203
  const current = replicas.get(room.locator);
2535
3204
  if (current === undefined || !current.queue.some((batch) => batch.id === submitted.id)) {
2536
3205
  return null;
2537
3206
  }
2538
- const held = current.confirmed.record.roomVersion;
3207
+ const held = current.record.roomVersion;
2539
3208
  logAccepted(room.locator, submitted, accepted);
2540
- const applied = options.adapter.reached(room.locator, held, accepted.roomVersion)
2541
- ? { kind: 'staged', confirmed: current.confirmed }
2542
- : await applyReturned(current.confirmed, submitted, accepted, options.adapter.follows(room.locator, held, accepted.roomVersion)
2543
- ? accepted.roomVersion
2544
- : held, accepted.roomVersion);
2545
- if (applied.kind !== 'staged') {
2546
- const key = pendingKey(room.locator, submitted.id);
2547
- const state = pendings.get(key);
2548
- await local.accept(room.locator, submitted.id, accepted.roomVersion);
2549
- if (state !== undefined && !fences.has(room.locator))
2550
- pendings.set(key, { ...state, accepted: accepted.roomVersion });
2551
- emitQueue(room.locator);
2552
- return applied;
2553
- }
2554
- const settled = await settle(local, room.locator, current, applied.confirmed, current.queue.filter((batch) => batch.id !== submitted.id), { room: true });
2555
- pendings.delete(pendingKey(room.locator, submitted.id));
2556
- unreadable.delete(room.locator);
2557
- sourceRefusals.delete(room.locator);
2558
- declarationRefusals.delete(room.locator);
2559
- const set = settled();
2560
- list(room, true);
2561
- publishChangeSet(set);
2562
- moveTo({ ...status, ...attention() });
2563
- return null;
3209
+ let record = current.record;
3210
+ let changes;
3211
+ if (!options.adapter.reached(room.locator, held, accepted.roomVersion)) {
3212
+ const paths = [...returnedPaths(submitted, accepted)];
3213
+ const confirmedLayer = await stepped(slicing(await local.entries(record.key, 'confirmed', paths), paths));
3214
+ const back = await stepped(returning(confirmedLayer, submitted, accepted, accepted.roomVersion));
3215
+ record = {
3216
+ ...record,
3217
+ roomVersion: options.adapter.follows(room.locator, held, accepted.roomVersion)
3218
+ ? accepted.roomVersion
3219
+ : held,
3220
+ };
3221
+ const overrides = new Map([
3222
+ ...back.set.map((entry) => [entry.path, entry]),
3223
+ ...back.removed.map((path) => [path, null]),
3224
+ ]);
3225
+ const refused = await validatedRoom(local, record, 'confirmed', { record: current.record, changed: [...overrides.keys()] }, overrides);
3226
+ if (refused !== null) {
3227
+ const key = pendingKey(room.locator, submitted.id);
3228
+ return {
3229
+ locator: room.locator,
3230
+ left: [],
3231
+ commit: {
3232
+ rooms: [],
3233
+ writes: [
3234
+ {
3235
+ locator: room.locator,
3236
+ accepted: [{ id: submitted.id, roomVersion: accepted.roomVersion }],
3237
+ },
3238
+ ],
3239
+ },
3240
+ refused,
3241
+ apply() {
3242
+ const state = pendings.get(key);
3243
+ if (state !== undefined && !fences.has(room.locator))
3244
+ pendings.set(key, { ...state, accepted: accepted.roomVersion });
3245
+ return changeSet(room.locator, { queue: true });
3246
+ },
3247
+ };
3248
+ }
3249
+ changes = back;
3250
+ }
3251
+ const settled = await settlement(local, room.locator, current, record, current.queue.filter((batch) => batch.id !== submitted.id), { room: true, ...(changes === undefined ? {} : { changes }) });
3252
+ return {
3253
+ ...settled,
3254
+ apply(answer) {
3255
+ const set = settled.apply(answer);
3256
+ pendings.delete(pendingKey(room.locator, submitted.id));
3257
+ unreadable.delete(room.locator);
3258
+ sourceRefusals.delete(room.locator);
3259
+ declarationRefusals.delete(room.locator);
3260
+ list(room, true);
3261
+ return set;
3262
+ },
3263
+ };
3264
+ }
3265
+ async function adoptedTogether(local, acceptances) {
3266
+ const settlements = acceptances.filter(settling);
3267
+ if (settlements.length > 0) {
3268
+ publishChangeSet(await together(local, settlements));
3269
+ moveTo({ ...status, ...attention() });
3270
+ }
3271
+ await unpair(local);
2564
3272
  }
2565
3273
  async function adopted(room, local, submitted, accepted) {
2566
3274
  const failed = await exclusive(async () => {
2567
- const adopting = await adoptAccepted(room, local, submitted, accepted);
2568
- await unpair(local);
2569
- return adopting;
3275
+ const adopting = await acceptance(room, local, submitted, accepted);
3276
+ await adoptedTogether(local, [adopting]);
3277
+ return refusalOf(adopting);
2570
3278
  });
2571
3279
  if (failed === null)
2572
3280
  return true;
@@ -2577,19 +3285,20 @@ export function composeRuntime(options) {
2577
3285
  async function adoptedMove(room, partner, local, submitted, accepted) {
2578
3286
  const other = discovered.get(partner);
2579
3287
  const failed = await exclusive(async () => {
2580
- const left = await adoptAccepted(room, local, submitted, accepted);
3288
+ const left = await acceptance(room, local, submitted, accepted);
2581
3289
  const arriving = replicas.get(partner)?.queue.find((batch) => batch.id === submitted.id);
2582
3290
  const arrived = other === undefined || arriving === undefined || accepted.destination === undefined
2583
3291
  ? null
2584
- : await adoptAccepted(other, local, arriving, {
3292
+ : await acceptance(other, local, arriving, {
2585
3293
  kind: 'accepted',
2586
3294
  ...accepted.destination,
2587
3295
  });
2588
- if (other !== undefined && arrived?.kind === 'refused' && continuing()) {
2589
- refusedSource(other, arrived);
3296
+ await adoptedTogether(local, [left, arrived]);
3297
+ const refusedArrival = refusalOf(arrived);
3298
+ if (other !== undefined && refusedArrival?.kind === 'refused' && continuing()) {
3299
+ refusedSource(other, refusedArrival);
2590
3300
  }
2591
- await unpair(local);
2592
- return left;
3301
+ return refusalOf(left);
2593
3302
  });
2594
3303
  submitLater(partner);
2595
3304
  if (failed === null)
@@ -2615,79 +3324,102 @@ export function composeRuntime(options) {
2615
3324
  continue;
2616
3325
  if (queueIn(partner).some((held) => held.id === batch.id))
2617
3326
  continue;
2618
- if (pendings.get(pendingKey(locator, batch.id))?.accepted !== undefined)
2619
- continue;
2620
- if (leavingIn(batch, locator) && !withdrawable(locator)(batch))
2621
- continue;
2622
- return { locator, batch };
3327
+ if (unpairedIn(locator, batch))
3328
+ return { locator, batch };
2623
3329
  }
2624
3330
  }
2625
3331
  return null;
2626
3332
  }
2627
- async function unpair(local) {
2628
- const handled = new Set();
2629
- for (let held = unpaired(handled); held !== null && continuing(); held = unpaired(handled)) {
2630
- const { locator, batch } = held;
2631
- handled.add(pendingKey(locator, batch.id));
2632
- journal.record(`outcome ${locator} ${batch.id} ${localSaves ? 'dropped' : 'unpaired'}`);
2633
- const leaving = {
2634
- batch,
2635
- returned: !localSaves && leavingIn(batch, locator) ? returnedBy('withdrawn') : null,
3333
+ async function unpairing(local, locator, batches) {
3334
+ const ids = new Set(batches.map(({ id }) => id));
3335
+ const leaving = batches.map((batch) => ({
3336
+ batch,
3337
+ returned: !localSaves && leavingIn(batch, locator) ? returnedBy('withdrawn') : null,
3338
+ }));
3339
+ const outcome = localSaves ? 'dropped' : 'unpaired';
3340
+ const current = replicas.get(locator);
3341
+ if (current !== undefined) {
3342
+ const settled = await settlement(local, locator, current, current.record, current.queue.filter((one) => !ids.has(one.id)), { leaving });
3343
+ return {
3344
+ ...settled,
3345
+ apply(answer) {
3346
+ for (const batch of batches)
3347
+ journal.record(`outcome ${locator} ${batch.id} ${outcome}`);
3348
+ const set = settled.apply(answer);
3349
+ submitLater(locator);
3350
+ return set;
3351
+ },
2636
3352
  };
2637
- const current = replicas.get(locator);
2638
- if (current !== undefined) {
2639
- const settled = await settle(local, locator, current, current.confirmed, current.queue.filter((one) => one.id !== batch.id), { leaving });
2640
- publishChangeSet(settled());
3353
+ }
3354
+ const queue = queueIn(locator).filter((one) => !ids.has(one.id));
3355
+ const written = new Map();
3356
+ for (const { batch, returned } of leaving) {
3357
+ if (returned === null)
2641
3358
  continue;
2642
- }
2643
- const queue = queueIn(locator).filter((one) => one.id !== batch.id);
2644
- const written = new Map((leaving.returned === null
2645
- ? []
2646
- : (pendings.get(pendingKey(locator, batch.id))?.drafts ?? [])).map((saved) => [
2647
- saved.path,
2648
- {
3359
+ for (const saved of pendings.get(pendingKey(locator, batch.id))?.drafts ?? []) {
3360
+ written.set(saved.path, {
2649
3361
  locator,
2650
3362
  path: saved.path,
2651
3363
  revision: roomDraftRevision(),
2652
- parts: returnedInto(roomDrafts.get(locator)?.get(saved.path)?.parts, saved.parts),
2653
- returned: leaving.returned,
2654
- },
2655
- ]));
2656
- const carried = carriedBack(locator, leaving.returned === null ? [] : [batch], written);
2657
- await local.keepAuthored({
2658
- locator,
2659
- queue,
2660
- roomDrafts: [...written.values()],
2661
- consumedRoomDrafts: carried.taken,
2662
- }, ...carried.writes);
2663
- pendings.delete(pendingKey(locator, batch.id));
2664
- if (queue.length > 0)
2665
- queuesWithoutReplica.set(locator, queue);
2666
- else
2667
- queuesWithoutReplica.delete(locator);
2668
- for (const record of written.values())
2669
- rememberRoomDraft(record);
2670
- carried.remember();
2671
- publishChangeSet(mergeChangeSets([
2672
- changeSet(locator, null, null, {
2673
- queue: true,
2674
- drafts: [...written.keys(), ...carried.taken],
2675
- }),
2676
- ...carried.changes,
2677
- ]));
3364
+ parts: returnedInto((written.get(saved.path) ?? roomDrafts.get(locator)?.get(saved.path))?.parts, saved.parts),
3365
+ returned,
3366
+ });
3367
+ }
3368
+ }
3369
+ const carried = carriedBack(locator, leaving.flatMap(({ batch, returned }) => (returned === null ? [] : [batch])), written);
3370
+ return {
3371
+ locator,
3372
+ left: batches,
3373
+ commit: {
3374
+ rooms: [],
3375
+ writes: [
3376
+ {
3377
+ locator,
3378
+ queue,
3379
+ roomDrafts: [...written.values()],
3380
+ consumedRoomDrafts: carried.taken,
3381
+ },
3382
+ ...carried.writes,
3383
+ ],
3384
+ },
3385
+ apply() {
3386
+ for (const batch of batches) {
3387
+ journal.record(`outcome ${locator} ${batch.id} ${outcome}`);
3388
+ pendings.delete(pendingKey(locator, batch.id));
3389
+ }
3390
+ if (queue.length > 0)
3391
+ queuesWithoutReplica.set(locator, queue);
3392
+ else
3393
+ queuesWithoutReplica.delete(locator);
3394
+ for (const record of written.values())
3395
+ rememberRoomDraft(record);
3396
+ carried.remember();
3397
+ submitLater(locator);
3398
+ return mergeChangeSets([
3399
+ changeSet(locator, {
3400
+ queue: true,
3401
+ drafts: [...written.keys(), ...carried.taken],
3402
+ }),
3403
+ ...carried.changes,
3404
+ ]);
3405
+ },
3406
+ };
3407
+ }
3408
+ async function unpair(local) {
3409
+ const handled = new Set();
3410
+ for (let held = unpaired(handled); held !== null && continuing(); held = unpaired(handled)) {
3411
+ const { locator, batch } = held;
3412
+ handled.add(pendingKey(locator, batch.id));
3413
+ publishChangeSet(await together(local, [await unpairing(local, locator, [batch])]));
2678
3414
  }
2679
3415
  if (handled.size === 0)
2680
3416
  return;
2681
3417
  moveTo({ ...status, ...attention() });
2682
- for (const key of handled)
2683
- submitLater(key.slice(0, key.indexOf('\u0000')));
2684
3418
  }
2685
3419
  async function discardHeld(head, room, local, signal) {
2686
3420
  const partner = partnerOf(head, room.locator);
2687
3421
  const other = partner === null ? undefined : discovered.get(partner);
2688
- await discardPending(room, local, signal);
2689
- if (other !== undefined && !signal.aborted)
2690
- await discardPending(other, local, signal);
3422
+ await discardPending(other === undefined ? [room] : [room, other], local, signal);
2691
3423
  }
2692
3424
  async function submitRoom(room, local, authorized, progress, signal) {
2693
3425
  while (continuing() && authorization !== undefined && !signal.aborted) {
@@ -2858,14 +3590,12 @@ export function composeRuntime(options) {
2858
3590
  if (current === undefined || !current.queue.some((batch) => batch.id === head.id)) {
2859
3591
  return;
2860
3592
  }
2861
- const settled = await settle(local, room.locator, current, current.confirmed, current.queue.filter((batch) => batch.id !== head.id), {
2862
- leaving: (result.kind === 'expired' && landedWhole(room.locator, head, current.confirmed)) ||
2863
- (result.kind === 'conflicted' &&
2864
- landedWhole(room.locator, head, current.confirmed, result.affected))
2865
- ? { ...left, returned: null }
2866
- : left,
2867
- });
2868
- publishChangeSet(settled());
3593
+ const confirmedLayer = await sliceAt(current.record, 'confirmed', [head]);
3594
+ const held = (result.kind === 'expired' &&
3595
+ (await landedWhole(room.locator, head, confirmedLayer))) ||
3596
+ (result.kind === 'conflicted' &&
3597
+ (await landedWhole(room.locator, head, confirmedLayer, result.affected)));
3598
+ publishChangeSet(await settle(local, room.locator, current, current.record, current.queue.filter((batch) => batch.id !== head.id), { leaving: [held ? { ...left, returned: null } : left] }));
2869
3599
  await unpair(local);
2870
3600
  moveTo({ ...status, ...attention() });
2871
3601
  });
@@ -2893,44 +3623,67 @@ export function composeRuntime(options) {
2893
3623
  function pastHorizon(identity) {
2894
3624
  return (outcomeRetention !== null && identityTime(identity) + outcomeRetention * 1000 <= Date.now());
2895
3625
  }
2896
- async function discardPending(room, local, signal) {
2897
- const { locator } = room;
3626
+ async function discardPending(rooms, local, signal) {
2898
3627
  const discarded = await exclusive(async () => {
2899
- if (fences.has(locator))
2900
- return false;
2901
- const before = replicas.get(locator);
2902
- const queue = before?.queue ?? queuesWithoutReplica.get(locator) ?? [];
2903
- fences.set(locator, { at: discoveries, erased: false });
2904
- lastFenced.set(locator, ++fencings);
2905
- try {
2906
- await local.discard(locator);
2907
- }
2908
- finally {
2909
- fences.delete(locator);
3628
+ const held = rooms
3629
+ .filter(({ locator }) => !fences.has(locator))
3630
+ .map((room) => ({
3631
+ room,
3632
+ before: replicas.get(room.locator),
3633
+ queue: queueIn(room.locator),
3634
+ }));
3635
+ if (held.length === 0)
3636
+ return [];
3637
+ const answer = await local.commit({
3638
+ rooms: held.flatMap(({ room, before }) => before === undefined
3639
+ ? []
3640
+ : [
3641
+ {
3642
+ locator: room.locator,
3643
+ key: before.record.key,
3644
+ roots: before.record.typedSelection?.roots ?? [],
3645
+ overlay: { set: [], cleared: [...before.overlaid] },
3646
+ },
3647
+ ]),
3648
+ discards: held.map(({ room }) => room.locator),
3649
+ });
3650
+ for (const { room, before, queue } of held) {
3651
+ const { locator } = room;
3652
+ lastFenced.set(locator, ++fencings);
3653
+ for (const batch of queue)
3654
+ pendings.delete(pendingKey(locator, batch.id));
3655
+ queuesWithoutReplica.delete(locator);
3656
+ if (before !== undefined) {
3657
+ replicas.set(locator, {
3658
+ ...before,
3659
+ queue: [],
3660
+ overlaid: new Set(),
3661
+ count: answer.counts[locator] ?? before.count,
3662
+ });
3663
+ }
3664
+ journal.record(`room discarded ${locator}`);
2910
3665
  }
2911
- for (const batch of queue)
2912
- pendings.delete(pendingKey(locator, batch.id));
2913
- queuesWithoutReplica.delete(locator);
2914
- const after = before === undefined ? null : await stepped(replicating(before.confirmed, [], before));
2915
- if (after !== null)
2916
- replicas.set(locator, after);
2917
- journal.record(`room discarded ${locator}`);
2918
- publishChangeSet(changeSet(locator, before ?? null, after, { queue: true }));
3666
+ publishChangeSet(mergeChangeSets(held.map(({ room }) => changeSet(room.locator, {
3667
+ documents: answer.changed[room.locator] ?? [],
3668
+ queue: true,
3669
+ }))));
2919
3670
  moveTo({ ...status, ...attention() });
2920
- return true;
3671
+ return held.map(({ room }) => room);
2921
3672
  });
2922
- if (!discarded)
2923
- return;
2924
- const progress = { complete: true, reachable: true };
2925
- await roomTask(locator, async (live) => {
2926
- await Promise.race([
2927
- reload(room, local, progress, () => live() && !signal.aborted),
2928
- abandonedOn(signal),
2929
- ]);
2930
- });
2931
- if (signal.aborted)
2932
- return;
2933
- await rest(progress);
3673
+ for (const room of discarded) {
3674
+ if (signal.aborted)
3675
+ return;
3676
+ const progress = { complete: true, reachable: true };
3677
+ await roomTask(room.locator, async (live) => {
3678
+ await Promise.race([
3679
+ reload(room, local, progress, () => live() && !signal.aborted),
3680
+ abandonedOn(signal),
3681
+ ]);
3682
+ });
3683
+ if (signal.aborted)
3684
+ return;
3685
+ await rest(progress);
3686
+ }
2934
3687
  }
2935
3688
  function awaitingAnswer(locator) {
2936
3689
  let release = () => { };
@@ -2947,7 +3700,7 @@ export function composeRuntime(options) {
2947
3700
  async function retainModel(local, room, attached, next) {
2948
3701
  const key = modelKey(room.locator, attached);
2949
3702
  const previous = modelOf(room.locator, attached);
2950
- const heldBefore = replicas.get(room.locator)?.confirmed.record;
3703
+ const heldBefore = replicas.get(room.locator)?.record;
2951
3704
  if (JSON.stringify(previous) === JSON.stringify(next) &&
2952
3705
  (heldBefore === undefined ||
2953
3706
  (JSON.stringify(heldBefore.operations) === JSON.stringify(room.operations) &&
@@ -2966,33 +3719,48 @@ export function composeRuntime(options) {
2966
3719
  let changed = null;
2967
3720
  if (held !== undefined) {
2968
3721
  const declared = {
2969
- ...held.confirmed.record,
3722
+ ...held.record,
2970
3723
  operations: room.operations,
2971
3724
  replica: room.replica,
2972
3725
  typedSelection: room.typedSelection ?? null,
2973
3726
  ...(attached === 'collection' ? { model: next } : { selectionModel: next }),
2974
3727
  };
2975
- const reclassified = sameMembership(declared, held.confirmed.record)
2976
- ? { kind: 'staged', confirmed: await confirmedOf(declared, held.confirmed) }
2977
- : await readReclassified(declared, held.confirmed);
2978
- const record = reclassified.kind === 'staged'
3728
+ let read = 'staged';
3729
+ if (!sameMembership(declared, held.record)) {
3730
+ const verdict = await validatedCollection(walkOf(local, held.record.key, 'confirmed'), lookupOf(local, held.record.key, 'confirmed'), compositionOf(declared));
3731
+ read =
3732
+ verdict.verdict !== 'refuse'
3733
+ ? 'staged'
3734
+ : verdict.rule === 'model-unavailable'
3735
+ ? 'deferred'
3736
+ : { kind: 'refused', refusal: verdict };
3737
+ }
3738
+ const record = read === 'staged'
2979
3739
  ? declared
2980
- : {
2981
- ...declared,
2982
- typedSelection: held.confirmed.record.typedSelection,
2983
- };
2984
- const read = reclassified.kind === 'deferred'
2985
- ? { kind: 'staged', confirmed: await confirmedOf(record, held.confirmed) }
2986
- : reclassified;
2987
- await local.write({ next: record, previous: held.confirmed.record });
2988
- const replica = read.kind === 'staged'
2989
- ? await stepped(replicating(read.confirmed, held.queue, held))
2990
- : { ...held, confirmed: { ...held.confirmed, record } };
2991
- await stepped(counting(replica));
2992
- const set = await stepped(changing(room.locator, held, replica, { room: true }));
2993
- replicas.set(room.locator, replica);
2994
- changed = heldChange(room.locator, held, replica, set, { room: true });
2995
- if (read.kind === 'staged') {
3740
+ : { ...declared, typedSelection: held.record.typedSelection };
3741
+ const confirmedLayer = await sliceAt(record, 'confirmed', held.queue, held.overlaid);
3742
+ const replayedLayer = await replayed(record, confirmedLayer, held.queue);
3743
+ const diff = await overlayDiff(record, held, replayedLayer.visible, replayedLayer.touched);
3744
+ const answer = await local.commit({
3745
+ rooms: [
3746
+ {
3747
+ locator: room.locator,
3748
+ key: record.key,
3749
+ record: storedRecord(record),
3750
+ roots: record.typedSelection?.roots ?? [],
3751
+ composition: compositionOf(record),
3752
+ overlay: diff.overlay,
3753
+ },
3754
+ ],
3755
+ });
3756
+ recomposed(room.locator, held.record, record);
3757
+ const { count } = committedRoom(room.locator, record, answer);
3758
+ replicas.set(room.locator, { ...held, record, overlaid: diff.touched, count });
3759
+ changed = changeSet(room.locator, {
3760
+ documents: answer.changed[room.locator] ?? [],
3761
+ room: true,
3762
+ });
3763
+ if (read === 'staged' || read === 'deferred') {
2996
3764
  if (declarationRefusals.delete(room.locator)) {
2997
3765
  sourceRefusals.delete(room.locator);
2998
3766
  unreadable.delete(room.locator);
@@ -3045,17 +3813,10 @@ export function composeRuntime(options) {
3045
3813
  }
3046
3814
  function heldRooms(listed) {
3047
3815
  return listed.flatMap((room) => {
3048
- const record = replicas.get(room.locator)?.confirmed.record;
3816
+ const record = replicas.get(room.locator)?.record;
3049
3817
  if (record === undefined || record.replica.selection !== room.replica.selection)
3050
3818
  return [];
3051
- return [
3052
- {
3053
- locator: room.locator,
3054
- roomVersion: record.roomVersion,
3055
- documents: record.documents,
3056
- revisions: record.revisions,
3057
- },
3058
- ];
3819
+ return [{ locator: room.locator, roomVersion: record.roomVersion }];
3059
3820
  });
3060
3821
  }
3061
3822
  async function synchronize(listed, local, progress, attempt, live) {
@@ -3083,7 +3844,7 @@ export function composeRuntime(options) {
3083
3844
  progress.complete = false;
3084
3845
  return false;
3085
3846
  }
3086
- const prior = replicas.get(room.locator)?.confirmed.record;
3847
+ const prior = replicas.get(room.locator)?.record;
3087
3848
  if (prior !== undefined && prior.replica.selection !== room.replica.selection) {
3088
3849
  list(prior, false);
3089
3850
  emitRooms();
@@ -3137,7 +3898,7 @@ export function composeRuntime(options) {
3137
3898
  await withhold(bootstrapped, room, local, progress);
3138
3899
  return;
3139
3900
  }
3140
- await publish(local, room, bootstrapped.confirmed, { stored: bootstrapped.stored }, live);
3901
+ await publish(local, room, bootstrapped.record, { kind: 'bootstrapped', refreshed: bootstrapped.refreshed }, live);
3141
3902
  await caughtUp(room, local, progress, live);
3142
3903
  }
3143
3904
  finally {
@@ -3224,11 +3985,12 @@ export function composeRuntime(options) {
3224
3985
  if (JSON.stringify(result.providerLimits) !== JSON.stringify(limits) ||
3225
3986
  JSON.stringify(result.capabilities) !== JSON.stringify(previousCapabilities)) {
3226
3987
  limits = result.providerLimits;
3227
- await local.declare(limits, capabilities);
3988
+ await local.fact('providerLimits', limits);
3989
+ await local.fact('capabilities', capabilities);
3228
3990
  }
3229
3991
  const signedIn = result.member ?? null;
3230
3992
  if (JSON.stringify(signedIn) !== JSON.stringify(member)) {
3231
- await local.keepMember(signedIn);
3993
+ await local.fact('member', signedIn);
3232
3994
  member = signedIn;
3233
3995
  emitMember();
3234
3996
  }
@@ -3432,7 +4194,7 @@ export function composeRuntime(options) {
3432
4194
  owed.clear();
3433
4195
  for (const locator of reloading)
3434
4196
  catching.add(locator);
3435
- readSide.recover();
4197
+ liveQueries.recover();
3436
4198
  return reloading;
3437
4199
  }
3438
4200
  async function reloadNow(replaced, progress) {
@@ -3599,7 +4361,7 @@ export function composeRuntime(options) {
3599
4361
  if (synced && !recorded) {
3600
4362
  const completedAt = new Date().toISOString();
3601
4363
  try {
3602
- await (await openLocal()).complete(completedAt);
4364
+ await (await openLocal()).fact('lastCompleted', completedAt);
3603
4365
  if (!live())
3604
4366
  return;
3605
4367
  recorded = true;
@@ -3639,8 +4401,8 @@ export function composeRuntime(options) {
3639
4401
  if (room === undefined || room.replica.admission !== 'admitted')
3640
4402
  return;
3641
4403
  if (held !== undefined &&
3642
- (held.confirmed.record.replica.selection !== room.replica.selection ||
3643
- options.adapter.reached(locator, held.confirmed.record.roomVersion, roomVersion))) {
4404
+ (held.record.replica.selection !== room.replica.selection ||
4405
+ options.adapter.reached(locator, held.record.roomVersion, roomVersion))) {
3644
4406
  return;
3645
4407
  }
3646
4408
  const during = underway?.generation === generation ? underway.progress : null;
@@ -3702,7 +4464,14 @@ export function composeRuntime(options) {
3702
4464
  void rediscover(event.scope);
3703
4465
  else
3704
4466
  void applyRoom(event.locator, event.roomVersion);
3705
- }, receipts);
4467
+ }, receipts, heldRevisions);
4468
+ }
4469
+ async function heldRevisions(locator, paths) {
4470
+ const held = replicas.get(locator);
4471
+ if (held === undefined || paths.length === 0)
4472
+ return new Map();
4473
+ const entries = await (await openLocal()).entries(held.record.key, 'confirmed', paths);
4474
+ return new Map(entries.flatMap(({ path, revision }) => revision === null ? [] : [[path, revision]]));
3706
4475
  }
3707
4476
  function writesHeld() {
3708
4477
  return serviceMode === 'write-halted' || haltedRefusal;
@@ -3794,7 +4563,7 @@ export function composeRuntime(options) {
3794
4563
  statusListeners.clear();
3795
4564
  serviceModeListeners.clear();
3796
4565
  memberListeners.clear();
3797
- readSide.close();
4566
+ liveQueries.close();
3798
4567
  queueSubscriptions.clear();
3799
4568
  roomDraftListeners.clear();
3800
4569
  presentationListeners.clear();
@@ -3840,7 +4609,7 @@ export function composeRuntime(options) {
3840
4609
  const watchers = seedWatchers.get(plan.key) ?? new Set();
3841
4610
  seedWatchers.set(plan.key, watchers);
3842
4611
  watchers.add(shown);
3843
- const stop = readSide.watch(plan, identityOf(plan), () => {
4612
+ const stop = liveQueries.watch(plan, identityOf(plan), () => {
3844
4613
  const cover = covered(plan);
3845
4614
  awaitPreparation(cover);
3846
4615
  return seedBehind(plan, cover);
@@ -3911,10 +4680,9 @@ export function composeRuntime(options) {
3911
4680
  if ('pending' in answered)
3912
4681
  continue;
3913
4682
  const searchedConversations = named.filter((one) => searchIndex.conversations().has(one));
3914
- await readSide.settled();
3915
4683
  const seen = searchIndex.changes();
3916
4684
  await searchIndex.caughtUp();
3917
- const selected = roomsAlone(answering) ? null : await rowsQuery(answering);
4685
+ const selected = roomsAlone(answering) ? null : await everyRow(answering);
3918
4686
  if (selected !== null && 'pending' in selected)
3919
4687
  continue;
3920
4688
  if (searchIndex.changes() !== seen ||
@@ -3927,7 +4695,7 @@ export function composeRuntime(options) {
3927
4695
  : new Set(reachedRooms(answering)), plan.collection
3928
4696
  ? new Map(answered.coverage.rooms.map(({ room }) => [
3929
4697
  room,
3930
- replicas.get(room)?.confirmed.record.typedSelection?.roots ?? [],
4698
+ replicas.get(room)?.record.typedSelection?.roots ?? [],
3931
4699
  ]))
3932
4700
  : null);
3933
4701
  const coverage = {
@@ -3947,13 +4715,17 @@ export function composeRuntime(options) {
3947
4715
  roomDraftListeners.add(listener);
3948
4716
  void exclusive(async () => {
3949
4717
  await openLocal();
3950
- if (closed || !roomDraftListeners.has(listener))
3951
- return;
3952
- listener.ready = true;
3953
4718
  }).then(() => {
3954
- if (closed || !listener.ready || !roomDraftListeners.has(listener))
4719
+ if (closed || !roomDraftListeners.has(listener))
3955
4720
  return;
3956
- listener.notify(snapshotRoomDrafts());
4721
+ const snapshot = snapshotRoomDrafts();
4722
+ draftsShown = draftsShown.then(async () => {
4723
+ const value = await snapshot.catch(() => null);
4724
+ if (value === null || closed || !roomDraftListeners.has(listener))
4725
+ return;
4726
+ listener.ready = true;
4727
+ listener.notify(value);
4728
+ });
3957
4729
  }, () => { });
3958
4730
  return () => {
3959
4731
  roomDraftListeners.delete(listener);
@@ -4068,9 +4840,9 @@ export function composeRuntime(options) {
4068
4840
  return listed;
4069
4841
  });
4070
4842
  },
4071
- read(room, path) {
4843
+ async read(room, path) {
4072
4844
  const record = roomDrafts.get(room)?.get(path);
4073
- return record === undefined ? null : roomDraftOf(record);
4845
+ return record === undefined ? null : ((await roomDraftsOf([record]))[0] ?? null);
4074
4846
  },
4075
4847
  write(room, path, revision, input) {
4076
4848
  return exclusive(async () => {
@@ -4080,11 +4852,11 @@ export function composeRuntime(options) {
4080
4852
  const local = await openLocal();
4081
4853
  const current = roomDrafts.get(room)?.get(path);
4082
4854
  if ((current?.revision ?? null) !== revision) {
4083
- return staleWrite(current === undefined ? null : roomDraftOf(current));
4855
+ return staleWrite(current === undefined ? null : ((await roomDraftsOf([current]))[0] ?? null));
4084
4856
  }
4085
4857
  const file = parts.file === undefined
4086
4858
  ? undefined
4087
- : await heldFile(parts.file.blob, binaryBase(replicas.get(room)?.visible, path));
4859
+ : await heldFile(parts.file.blob, binaryBase(await visibleOf(room, [path]), path));
4088
4860
  const record = {
4089
4861
  locator: room,
4090
4862
  path,
@@ -4096,8 +4868,8 @@ export function composeRuntime(options) {
4096
4868
  ? []
4097
4869
  : [{ key: file.localId, blob: parts.file.blob }]);
4098
4870
  rememberRoomDraft(record);
4099
- publishChangeSet(changeSet(room, null, null, { drafts: [path] }));
4100
- return { outcome: 'written', roomDraft: roomDraftOf(record) };
4871
+ publishChangeSet(changeSet(room, { drafts: [path] }));
4872
+ return { outcome: 'written', roomDraft: (await roomDraftsOf([record]))[0] ?? null };
4101
4873
  });
4102
4874
  },
4103
4875
  discard(room, path, revision) {
@@ -4105,11 +4877,11 @@ export function composeRuntime(options) {
4105
4877
  const local = await openLocal();
4106
4878
  const current = roomDrafts.get(room)?.get(path);
4107
4879
  if (current === undefined || current.revision !== revision) {
4108
- return staleWrite(current === undefined ? null : roomDraftOf(current));
4880
+ return staleWrite(current === undefined ? null : ((await roomDraftsOf([current]))[0] ?? null));
4109
4881
  }
4110
4882
  await local.discardRoomDraft(room, path);
4111
4883
  forgetRoomDraft(room, path);
4112
- publishChangeSet(changeSet(room, null, null, { drafts: [path] }));
4884
+ publishChangeSet(changeSet(room, { drafts: [path] }));
4113
4885
  moveTo({ ...status, ...attention() });
4114
4886
  return { outcome: 'written', roomDraft: null };
4115
4887
  });
@@ -4157,7 +4929,7 @@ export function composeRuntime(options) {
4157
4929
  const room = calledRooms(name, input).find((one) => locked(one));
4158
4930
  return room === undefined ? recoveredAction(name, input) : lockedRoom(room);
4159
4931
  },
4160
- reached(locator, roomVersion, path) {
4932
+ async reached(locator, roomVersion, path) {
4161
4933
  return path === undefined
4162
4934
  ? reachedBy(locator, roomVersion)
4163
4935
  : writtenAfter(locator, roomVersion, path);