@agentshouse/mdruntime 0.1.0-c5f333dfa0583 → 0.1.0-c63ad08fad0c3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/AGENTS.md +45 -38
  2. package/dist/adapter.d.ts +8 -0
  3. package/dist/adapter.d.ts.map +1 -1
  4. package/dist/bootstrap.d.ts +13 -2
  5. package/dist/bootstrap.d.ts.map +1 -1
  6. package/dist/bootstrap.js +87 -31
  7. package/dist/change-set.d.ts +1 -7
  8. package/dist/change-set.d.ts.map +1 -1
  9. package/dist/change-set.js +1 -4
  10. package/dist/compile.d.ts +6 -1
  11. package/dist/compile.d.ts.map +1 -1
  12. package/dist/compile.js +12 -3
  13. package/dist/connection.d.ts +1 -0
  14. package/dist/connection.d.ts.map +1 -1
  15. package/dist/connection.js +330 -282
  16. package/dist/conversation.d.ts.map +1 -1
  17. package/dist/conversation.js +5 -14
  18. package/dist/durability.d.ts +1 -0
  19. package/dist/durability.d.ts.map +1 -1
  20. package/dist/fulltext.d.ts +27 -0
  21. package/dist/fulltext.d.ts.map +1 -0
  22. package/dist/fulltext.js +368 -0
  23. package/dist/host.d.ts +1 -1
  24. package/dist/host.d.ts.map +1 -1
  25. package/dist/host.js +5 -3
  26. package/dist/identity.d.ts +0 -1
  27. package/dist/identity.d.ts.map +1 -1
  28. package/dist/identity.js +0 -3
  29. package/dist/index.d.ts +3 -3
  30. package/dist/index.d.ts.map +1 -1
  31. package/dist/messages.d.ts +28 -0
  32. package/dist/messages.d.ts.map +1 -0
  33. package/dist/messages.js +90 -0
  34. package/dist/query.d.ts +4 -1
  35. package/dist/query.d.ts.map +1 -1
  36. package/dist/query.js +12 -1
  37. package/dist/readiness.d.ts +17 -2
  38. package/dist/readiness.d.ts.map +1 -1
  39. package/dist/readiness.js +1 -1
  40. package/dist/reads.d.ts.map +1 -1
  41. package/dist/reads.js +3 -1
  42. package/dist/refresh.d.ts +1 -0
  43. package/dist/refresh.d.ts.map +1 -1
  44. package/dist/refresh.js +4 -0
  45. package/dist/replica.d.ts +3 -1
  46. package/dist/replica.d.ts.map +1 -1
  47. package/dist/search.d.ts +11 -43
  48. package/dist/search.d.ts.map +1 -1
  49. package/dist/search.js +0 -296
  50. package/dist/store.d.ts +17 -36
  51. package/dist/store.d.ts.map +1 -1
  52. package/dist/store.js +28 -227
  53. package/dist/stored.d.ts +9 -7
  54. package/dist/stored.d.ts.map +1 -1
  55. package/dist/stored.js +46 -40
  56. package/dist/tables.d.ts +34 -12
  57. package/dist/tables.d.ts.map +1 -1
  58. package/dist/tables.js +44 -5
  59. package/dist/words.d.ts +19 -0
  60. package/dist/words.d.ts.map +1 -0
  61. package/dist/words.js +150 -0
  62. package/package.json +2 -3
@@ -6,7 +6,7 @@ import { filterOf, identityOf, planOf, plain, queryKey, } from './query.js';
6
6
  import { createLiveQueries } from './live.js';
7
7
  import { seededResult } from './seed.js';
8
8
  import { documentKey, roomOfKey } from './rows.js';
9
- import { createSearchIndex, searchPlanOf, } from './search.js';
9
+ import { searchPlanOf } from './search.js';
10
10
  import { capped, MAXIMUM_WINDOW } from './windows.js';
11
11
  import { bootstrapRoom } from './bootstrap.js';
12
12
  import { changeSet, mergeChangeSets } from './change-set.js';
@@ -28,7 +28,6 @@ import { same, saving, STARTING, transferring, } from './status.js';
28
28
  import { detectStorageLoss, nextGeneration } from './durability.js';
29
29
  import { readinessOf, SETTLED_ROOMS, } from './readiness.js';
30
30
  import { FencedMap } from './fence.js';
31
- import { fingerprint } from './stored.js';
32
31
  import { sameBinary } from './overlay.js';
33
32
  import { paced, ROW_WEIGHT, stepped, yielded } from './turn.js';
34
33
  import { identityTime, roomDraftRevision } from './identity.js';
@@ -37,15 +36,6 @@ import { openRuntimeStore, } from './store.js';
37
36
  import { changedCleanly, sameComposition, validatedCollection, } from './validation.js';
38
37
  import { unconfirmed } from './transfer.js';
39
38
  import { governs, holds, narrowed, planReach, } from './reach.js';
40
- function roomsAlone(plan) {
41
- return (plan.reach === null &&
42
- plan.root.types === null &&
43
- plan.root.where === null &&
44
- plan.root.pathPrefix === null &&
45
- plan.paths === null &&
46
- plan.hop === null &&
47
- plan.backlinks === null);
48
- }
49
39
  function within(root, path) {
50
40
  return root === '' || beneath(root, path);
51
41
  }
@@ -63,12 +53,23 @@ const OPENING_LANE = 'opening';
63
53
  const CONVERSATION_LANE = 'conversations';
64
54
  const ROOM_READS = 256;
65
55
  const ROOM_ATTEMPTS = 16;
66
- const SEARCH_BATCH = 256;
67
56
  const ERASE_BATCH = 512;
68
57
  const REDERIVE_BATCH = 256;
69
58
  const WALK_BYTES = 1 << 20;
70
59
  const QUERY_BYTES = 1 << 20;
71
60
  const STAGE_BYTES = 1 << 20;
61
+ const STAGE_FLOOR = 1 << 16;
62
+ // A batch aims at this much database work, so a Save or a query waits behind at most one short write.
63
+ const STAGE_AIM_MS = 50;
64
+ const PARTS_HELD = 4;
65
+ // Bytes still arriving are reported at most this often, well inside the 2 s progress bound, so readiness is not rebuilt for every chunk the network hands over.
66
+ const ARRIVAL_REPORT_MS = 250;
67
+ const INSTALL_STEPS = [
68
+ 'writing',
69
+ 'downloading',
70
+ 'replaying',
71
+ 'listing',
72
+ ];
72
73
  const CLEAN_CHANGES = 2_000;
73
74
  // A Room that locks offline stays readable this long after the connection drops, so a laptop waking or changing network keeps working while a disconnected device locks soon after.
74
75
  const LOCK_GRACE_MS = 120_000;
@@ -95,7 +96,6 @@ export function composeRuntime(options) {
95
96
  const roomTypes = roomTypesOf(options.adapter.roomTypes);
96
97
  let authorization = options.authorization;
97
98
  let opening;
98
- let store;
99
99
  const fences = new Map();
100
100
  const isFenced = (locator) => fences.has(locator);
101
101
  let fencings = 0;
@@ -192,9 +192,8 @@ export function composeRuntime(options) {
192
192
  const serviceModeListeners = new Set();
193
193
  let member = null;
194
194
  const memberListeners = new Set();
195
- const prepared = new Set();
196
195
  const unpreparedDemand = new Set();
197
- const unindexable = new Set();
196
+ const unpreparable = new Set();
198
197
  let preparing = false;
199
198
  let targets = {};
200
199
  let persistence = options.shell.persistence;
@@ -205,6 +204,15 @@ export function composeRuntime(options) {
205
204
  let readinessQueued = false;
206
205
  const catching = new Set();
207
206
  const transfers = new FencedMap(isFenced);
207
+ const installs = new Map();
208
+ const arrivals = new Map();
209
+ let arrivalReported = 0;
210
+ let onScreen = options.shell.onScreen;
211
+ const shownWaiters = [];
212
+ let partsFree = PARTS_HELD;
213
+ const partWaiters = [];
214
+ let stageBytes = STAGE_FLOOR * 4;
215
+ let installWrites = Promise.resolve();
208
216
  const conversations = createConversationStore({
209
217
  agents: declaredAgents,
210
218
  documents: (declaration, after) => rowsQuery(planOf({
@@ -229,30 +237,6 @@ export function composeRuntime(options) {
229
237
  unanswered: resendLater,
230
238
  expired: (operation) => pastHorizon(operation),
231
239
  });
232
- const searchIndex = createSearchIndex({
233
- async *room(room) {
234
- const local = await openLocal();
235
- for (let after = null;;) {
236
- const key = replicas.get(room)?.record.key;
237
- if (key === undefined)
238
- return;
239
- const page = await local.searched(key, room, after, SEARCH_BATCH, null);
240
- yield page.documents;
241
- if (page.next === null)
242
- return;
243
- after = page.next;
244
- }
245
- },
246
- async documents(room, paths) {
247
- const key = replicas.get(room)?.record.key;
248
- if (key === undefined || paths.length === 0)
249
- return [];
250
- return (await (await openLocal()).searched(key, room, null, paths.length, paths)).documents;
251
- },
252
- });
253
- const roomsToIndex = new Set();
254
- const conversationsToIndex = new Set();
255
- let indexing = false;
256
240
  const journal = createLog();
257
241
  const keptFiles = openedFiles(options.database, isFenced);
258
242
  let notifyReady = () => { };
@@ -532,23 +516,26 @@ export function composeRuntime(options) {
532
516
  }
533
517
  function applyChangeSet(set) {
534
518
  liveQueries.touched(set);
535
- searchIndex.changed(set);
536
519
  }
537
520
  function unpublished(error) {
538
521
  journal.record(`publication failed ${error instanceof Error ? error.name : 'unknown'}`);
539
522
  }
523
+ function indexed(locator) {
524
+ return replicas.get(locator)?.indexed === true;
525
+ }
540
526
  async function prepareReplica(locator) {
541
527
  for (;;) {
542
528
  const held = replicas.get(locator);
543
- if (held === undefined || closed || prepared.has(locator))
529
+ if (held === undefined || closed || held.indexed)
544
530
  return;
545
531
  const key = held.record.key;
546
532
  const done = await exclusive(async () => (await openLocal()).rederive(key, REDERIVE_BATCH));
547
- if (replicas.get(locator)?.record.key !== key)
533
+ const now = replicas.get(locator);
534
+ if (now?.record.key !== key)
548
535
  continue;
549
536
  if (!done)
550
537
  continue;
551
- prepared.add(locator);
538
+ replicas.set(locator, { ...now, indexed: true });
552
539
  publishChangeSet(changeSet(locator, { documents: null }));
553
540
  liveQueries.recover();
554
541
  queueReadiness();
@@ -556,11 +543,11 @@ export function composeRuntime(options) {
556
543
  }
557
544
  }
558
545
  function unprepared() {
559
- return [...unpreparedDemand, ...replicas.keys()].find((locator) => !prepared.has(locator) && !unindexable.has(locator));
546
+ return [...unpreparedDemand, ...replicas.keys()].find((locator) => !indexed(locator) && !unpreparable.has(locator));
560
547
  }
561
- function unbuilt(locator, error) {
562
- unindexable.add(locator);
563
- journal.record(`index failed ${locator} ${error instanceof Error ? error.name : 'unknown'}`);
548
+ function unpreparedBy(locator, error) {
549
+ unpreparable.add(locator);
550
+ journal.record(`preparation failed ${locator} ${error instanceof Error ? error.name : 'unknown'}`);
564
551
  queueReadiness();
565
552
  }
566
553
  function prepare(locators = []) {
@@ -576,7 +563,7 @@ export function composeRuntime(options) {
576
563
  if (closed)
577
564
  return;
578
565
  unpreparedDemand.delete(waiting);
579
- await prepareReplica(waiting).catch((error) => unbuilt(waiting, error));
566
+ await prepareReplica(waiting).catch((error) => unpreparedBy(waiting, error));
580
567
  }
581
568
  }
582
569
  finally {
@@ -617,7 +604,7 @@ export function composeRuntime(options) {
617
604
  pending.push({ room: locator, awaits: 'acquisition' });
618
605
  continue;
619
606
  }
620
- if (!prepared.has(locator)) {
607
+ if (!replica.indexed) {
621
608
  pending.push({ room: locator, awaits: 'preparation' });
622
609
  continue;
623
610
  }
@@ -650,7 +637,7 @@ export function composeRuntime(options) {
650
637
  return { listed: true, rooms: named.filter((room) => rooms.has(room)) };
651
638
  return { listed: named.every((room) => replicas.has(room)), rooms: named };
652
639
  }
653
- function roomState(locator, search) {
640
+ function roomState(locator) {
654
641
  if (sourceRefusals.has(locator))
655
642
  return 'refused';
656
643
  const listedRoom = discovered.get(locator);
@@ -661,27 +648,17 @@ export function composeRuntime(options) {
661
648
  return 'unadmitted';
662
649
  if (!replicas.has(locator))
663
650
  return 'acquiring';
664
- if (!prepared.has(locator))
651
+ if (!indexed(locator))
665
652
  return 'preparing';
666
- if (search && !searchIndex.rooms().has(locator))
667
- return 'indexing';
668
653
  return 'complete';
669
654
  }
670
- function conversationCoverage(reach, search) {
655
+ function conversationCoverage(reach) {
671
656
  const declared = reach === null || reach.conversations;
672
657
  if (!declared || options.adapter.conversations === undefined || declaredAgents() === undefined)
673
658
  return 'undeclared';
674
659
  if (!conversations.known() || !conversations.current())
675
660
  return 'acquiring';
676
- if (!search)
677
- return 'complete';
678
- const unindexed = conversations
679
- .searchable()
680
- .filter((one) => conversations.acquired(one) && !searchIndex.conversations().has(one));
681
- if (unindexed.length === 0)
682
- return 'complete';
683
- index([], unindexed);
684
- return 'indexing';
661
+ return 'complete';
685
662
  }
686
663
  function connectionOf() {
687
664
  if (socketOpen)
@@ -708,25 +685,66 @@ export function composeRuntime(options) {
708
685
  ];
709
686
  });
710
687
  }
688
+ function installationOf(coverage) {
689
+ const rooms = { done: 0, total: 0 };
690
+ const parts = { done: 0, total: 0 };
691
+ const bytes = { done: 0, total: 0 };
692
+ const received = { done: 0, total: 0 };
693
+ const steps = new Set();
694
+ let waiting = false;
695
+ for (const { room, state } of coverage) {
696
+ if (state === 'unadmitted')
697
+ continue;
698
+ rooms.total += 1;
699
+ if (state === 'complete')
700
+ rooms.done += 1;
701
+ const transfer = transfers.get(room);
702
+ const held = installs.get(room);
703
+ const inventory = transfer?.inventory ?? replicas.get(room)?.record.inventory ?? null;
704
+ if (inventory !== null) {
705
+ const done = transfer?.done ?? inventory;
706
+ const share = held?.share ?? null;
707
+ parts.total += inventory.parts;
708
+ bytes.total += inventory.bytes;
709
+ parts.done += done.parts;
710
+ bytes.done += done.bytes;
711
+ if (share !== null && share.entries > 0) {
712
+ bytes.done += Math.floor((share.bytes * share.written) / share.entries);
713
+ }
714
+ received.total += inventory.bytes;
715
+ received.done += done.bytes + (share?.bytes ?? 0);
716
+ for (const arrived of arrivals.get(room)?.values() ?? [])
717
+ received.done += arrived;
718
+ }
719
+ const installing = held !== undefined && state !== 'complete';
720
+ if (installing)
721
+ steps.add(held.step);
722
+ waiting ||= installing || state === 'acquiring';
723
+ }
724
+ return {
725
+ step: INSTALL_STEPS.find((step) => steps.has(step)) ?? null,
726
+ paused: waiting && !onScreen ? 'hidden' : null,
727
+ rooms,
728
+ parts,
729
+ bytes,
730
+ received,
731
+ };
732
+ }
711
733
  function readinessFor(port, reach) {
712
734
  const required = requiredRooms(reach);
713
735
  const search = port.search === 'required';
714
- const coverage = required.rooms.map((room) => ({ room, state: roomState(room, search) }));
715
- const building = (state) => coverage
716
- .filter((room) => room.state === state && !unindexable.has(room.room))
736
+ const coverage = required.rooms.map((room) => ({ room, state: roomState(room) }));
737
+ const preparing = coverage
738
+ .filter((room) => room.state === 'preparing' && !unpreparable.has(room.room))
717
739
  .map(({ room }) => room);
718
- const preparing = building('preparing');
719
740
  if (preparing.length > 0)
720
741
  prepare(preparing);
721
- const indexing = building('indexing');
722
- if (indexing.length > 0)
723
- index(indexing, []);
724
742
  return readinessOf({
725
743
  code: port.code,
726
744
  coverage: {
727
745
  listed: required.listed,
728
746
  rooms: coverage,
729
- conversations: conversationCoverage(reach, search),
747
+ conversations: conversationCoverage(reach),
730
748
  search: search ? 'required' : 'unused',
731
749
  },
732
750
  persistence,
@@ -736,6 +754,7 @@ export function composeRuntime(options) {
736
754
  coverage.some(({ room, state }) => paused.has(room) || !SETTLED_ROOMS.has(state))
737
755
  ? capacity
738
756
  : port.capacity,
757
+ installation: installationOf(coverage),
739
758
  refused: refusedIn(required.rooms),
740
759
  lost: storageLoss !== null,
741
760
  established: targets[port.target]?.established === true,
@@ -861,7 +880,7 @@ export function composeRuntime(options) {
861
880
  pause(locator, { cause: 'storage-exceeded', needed: bytes, available });
862
881
  }
863
882
  function seedRoomHeld(room, version) {
864
- return prepared.has(room) && reachedBy(room, version);
883
+ return indexed(room) && reachedBy(room, version);
865
884
  }
866
885
  function seedHeld(at) {
867
886
  return Object.entries(at).every(([room, version]) => (roomsListed && !rooms.has(room)) || seedRoomHeld(room, version));
@@ -982,81 +1001,6 @@ export function composeRuntime(options) {
982
1001
  const held = answered.cover;
983
1002
  return { ...answered.rows, coverage: held.coverage };
984
1003
  }
985
- async function everyRow(plan) {
986
- const window = { ...plan, limit: MAXIMUM_WINDOW, after: null, total: false };
987
- const first = await rowsQuery(window);
988
- if ('pending' in first || first.next === undefined)
989
- return first;
990
- const documents = 'documents' in first ? [...first.documents] : [];
991
- const groups = 'groups' in first ? [...first.groups] : [];
992
- for (let next = first.next; next !== undefined;) {
993
- const values = JSON.parse(next);
994
- const page = await rowsQuery({
995
- ...window,
996
- after: { text: next, values: values.length === 0 ? null : values },
997
- });
998
- if ('pending' in page)
999
- return page;
1000
- if ('documents' in page)
1001
- documents.push(...page.documents);
1002
- else
1003
- groups.push(...page.groups);
1004
- if (JSON.stringify(page.coverage) !== JSON.stringify(first.coverage))
1005
- return everyRow(plan);
1006
- next = page.next;
1007
- }
1008
- return 'documents' in first
1009
- ? { documents, coverage: first.coverage }
1010
- : { groups, coverage: first.coverage };
1011
- }
1012
- async function indexConversation(conversation) {
1013
- searchIndex.indexMessages(conversation, [], false);
1014
- let after = null;
1015
- for (;;) {
1016
- const batch = await durably((local) => local.messagesAfter(conversation, after));
1017
- if (closed)
1018
- return;
1019
- searchIndex.indexMessages(conversation, batch.map(({ identity, text }) => ({ conversation, identity, text })), batch.length === 0);
1020
- if (batch.length === 0)
1021
- return;
1022
- after = batch[batch.length - 1].identity;
1023
- }
1024
- }
1025
- function indexNext() {
1026
- const room = [...roomsToIndex].find((locator) => !searchIndex.rooms().has(locator) && !unindexable.has(locator));
1027
- if (room !== undefined) {
1028
- roomsToIndex.delete(room);
1029
- return () => prepared.has(room)
1030
- ? searchIndex.indexRoom(room).catch((error) => unbuilt(room, error))
1031
- : undefined;
1032
- }
1033
- const conversation = [...conversationsToIndex].find((one) => !searchIndex.conversations().has(one));
1034
- if (conversation === undefined)
1035
- return null;
1036
- conversationsToIndex.delete(conversation);
1037
- return () => indexConversation(conversation);
1038
- }
1039
- function index(rooms, named) {
1040
- for (const room of rooms)
1041
- roomsToIndex.add(room);
1042
- for (const conversation of named)
1043
- conversationsToIndex.add(conversation);
1044
- if (indexing)
1045
- return;
1046
- indexing = true;
1047
- void (async () => {
1048
- try {
1049
- for (let unit = indexNext(); unit !== null && !closed; unit = indexNext()) {
1050
- await yielded();
1051
- await unit();
1052
- queueReadiness();
1053
- }
1054
- }
1055
- finally {
1056
- indexing = false;
1057
- }
1058
- })().catch(() => { });
1059
- }
1060
1004
  function publishChangeSet(set) {
1061
1005
  applyChangeSet(set);
1062
1006
  if (set.rooms.length > 0 ||
@@ -1086,9 +1030,10 @@ export function composeRuntime(options) {
1086
1030
  held,
1087
1031
  record: { ...held.record, model: revoked, selectionModel: revoked },
1088
1032
  }));
1033
+ let answer = null;
1089
1034
  try {
1090
1035
  const local = await openLocal();
1091
- await local.commit({
1036
+ answer = await local.commit({
1092
1037
  rooms: records.map(({ locator, record }) => ({
1093
1038
  locator,
1094
1039
  key: record.key,
@@ -1114,8 +1059,8 @@ export function composeRuntime(options) {
1114
1059
  for (const { locator, held, record } of records) {
1115
1060
  if (replicas.get(locator) !== held)
1116
1061
  continue;
1117
- recomposed(locator, held.record, record);
1118
- replicas.set(locator, { ...held, record });
1062
+ const indexed = answer === null ? held.indexed : committedRoom(locator, answer).indexed;
1063
+ replicas.set(locator, { ...held, record, indexed });
1119
1064
  }
1120
1065
  for (const locator of [...rooms.keys()]) {
1121
1066
  const room = rooms.get(locator);
@@ -1255,9 +1200,8 @@ export function composeRuntime(options) {
1255
1200
  unreadable.delete(locator);
1256
1201
  stale.delete(locator);
1257
1202
  repairable.delete(locator);
1258
- unindexable.delete(locator);
1203
+ unpreparable.delete(locator);
1259
1204
  paused.delete(locator);
1260
- prepared.delete(locator);
1261
1205
  scopedRefusals.delete(locator);
1262
1206
  sourceRefusals.delete(locator);
1263
1207
  declarationRefusals.delete(locator);
@@ -1271,7 +1215,6 @@ export function composeRuntime(options) {
1271
1215
  }
1272
1216
  if (fresh.length === 0)
1273
1217
  return fresh;
1274
- searchIndex.forget(fresh);
1275
1218
  publishChangeSet(mergeChangeSets(removed.map(({ locator, drafts }) => changeSet(locator, { documents: null, room: true, queue: true, drafts }))));
1276
1219
  liveQueries.recover();
1277
1220
  moveTo({ ...status, ...attention() });
@@ -1365,7 +1308,6 @@ export function composeRuntime(options) {
1365
1308
  }
1366
1309
  }
1367
1310
  replicas.clear();
1368
- prepared.clear();
1369
1311
  queuesWithoutReplica.clear();
1370
1312
  for (const [locator, queue] of queuedBatches) {
1371
1313
  if (!held.rooms.has(locator))
@@ -1383,9 +1325,8 @@ export function composeRuntime(options) {
1383
1325
  overlaid: held.overlaid.get(record.key) ?? new Set(),
1384
1326
  count: counts[index] ?? 0,
1385
1327
  confirmed: 0,
1328
+ indexed: !held.unindexed.has(record.key),
1386
1329
  });
1387
- if (!held.unprepared.has(record.key))
1388
- prepared.add(record.locator);
1389
1330
  const existing = rooms.get(record.locator);
1390
1331
  if (existing === undefined)
1391
1332
  list(record, false);
@@ -1431,8 +1372,7 @@ export function composeRuntime(options) {
1431
1372
  async function open() {
1432
1373
  if (closed)
1433
1374
  throw new Error('runtime closed');
1434
- const opened = await openRuntimeStore(isFenced, options.database);
1435
- store = opened;
1375
+ const opened = openRuntimeStore(isFenced, options.database);
1436
1376
  const interrupted = await opened.fences();
1437
1377
  for (const locator of interrupted) {
1438
1378
  fences.set(locator, { at: discoveries, erased: false });
@@ -1563,23 +1503,11 @@ export function composeRuntime(options) {
1563
1503
  },
1564
1504
  };
1565
1505
  }
1566
- function recomposed(locator, before, after) {
1567
- if (before !== undefined &&
1568
- before.key === after.key &&
1569
- fingerprint(compositionOf(before)) === fingerprint(compositionOf(after))) {
1570
- return;
1571
- }
1572
- if (before !== undefined && before.key === after.key)
1573
- prepared.delete(locator);
1574
- }
1575
- function committedRoom(locator, record, answer) {
1576
- if (answer.prepared[locator] === false)
1577
- prepared.delete(locator);
1578
- else if (answer.prepared[locator] === true)
1579
- prepared.add(locator);
1580
- if (!prepared.has(locator))
1506
+ function committedRoom(locator, answer) {
1507
+ const indexed = answer.indexed[locator] ?? replicas.get(locator)?.indexed ?? false;
1508
+ if (!indexed)
1581
1509
  prepare([locator]);
1582
- return { count: answer.counts[locator] ?? replicas.get(locator)?.count ?? 0 };
1510
+ return { count: answer.counts[locator] ?? replicas.get(locator)?.count ?? 0, indexed };
1583
1511
  }
1584
1512
  function replayed(record, confirmedLayer, queue) {
1585
1513
  return stepped(replaying(confirmedLayer, queue, record.locator, compositionOf(record)));
@@ -1729,14 +1657,14 @@ export function composeRuntime(options) {
1729
1657
  const confirmedChanged = settling.commit?.published !== undefined ||
1730
1658
  settling.commit?.refreshed === true ||
1731
1659
  overrides !== undefined;
1732
- recomposed(locator, before?.record, record);
1733
- const { count } = committedRoom(locator, record, answer);
1660
+ const { count, indexed } = committedRoom(locator, answer);
1734
1661
  replicas.set(locator, {
1735
1662
  record,
1736
1663
  queue: kept,
1737
1664
  overlaid: touched,
1738
1665
  count,
1739
1666
  confirmed: (before?.confirmed ?? 0) + (confirmedChanged ? 1 : 0),
1667
+ indexed,
1740
1668
  });
1741
1669
  return mergeChangeSets([
1742
1670
  changeSet(locator, {
@@ -1782,7 +1710,7 @@ export function composeRuntime(options) {
1782
1710
  };
1783
1711
  }
1784
1712
  function publish(local, room, record, landing, live) {
1785
- return exclusive(async () => {
1713
+ const landed = exclusive(async () => {
1786
1714
  if (!live()) {
1787
1715
  caughtUpTo(room.locator);
1788
1716
  return true;
@@ -1833,7 +1761,7 @@ export function composeRuntime(options) {
1833
1761
  for (const batch of accepted)
1834
1762
  pendings.delete(pendingKey(room.locator, batch.id));
1835
1763
  stale.delete(room.locator);
1836
- unindexable.delete(room.locator);
1764
+ unpreparable.delete(room.locator);
1837
1765
  if (unreadable.delete(room.locator))
1838
1766
  scopedRefusals.delete(room.locator);
1839
1767
  sourceRefusals.delete(room.locator);
@@ -1848,6 +1776,7 @@ export function composeRuntime(options) {
1848
1776
  }
1849
1777
  return true;
1850
1778
  });
1779
+ return landing.kind === 'refreshed' ? landed : landed.finally(() => installed(room.locator));
1851
1780
  }
1852
1781
  async function savedRoom(locator) {
1853
1782
  const local = await openLocal();
@@ -2878,41 +2807,108 @@ export function composeRuntime(options) {
2878
2807
  return unreadableRoom(texts, walk, named);
2879
2808
  return null;
2880
2809
  }
2881
- function refreshingOf(local, key, liveness) {
2810
+ function refreshingOf(local, key, liveness, replaying = null) {
2811
+ const write = replaying === null ? exclusive : installWrite;
2882
2812
  return {
2813
+ async turn() {
2814
+ if (replaying === null)
2815
+ return;
2816
+ installing(replaying, 'replaying');
2817
+ await screened();
2818
+ },
2883
2819
  prior: async (paths) => new Map((await local.entries(key, 'refreshed', paths)).map((entry) => [entry.path, entry])),
2884
2820
  async stage(page) {
2885
- const batches = batched(page.entries);
2886
- for (const [index, entries] of batches.entries()) {
2887
- const last = index === batches.length - 1;
2888
- const staged = await exclusive(async () => {
2821
+ for (let written = 0;;) {
2822
+ if (replaying !== null)
2823
+ await screened();
2824
+ const end = batchEnd(page.entries, written);
2825
+ const last = end === page.entries.length;
2826
+ const entries = page.entries.slice(written, end);
2827
+ const staged = await write(async () => {
2889
2828
  if (!liveness())
2890
2829
  return false;
2891
- await local.refresh(key, last
2830
+ await timed(() => local.refresh(key, last
2892
2831
  ? { ...page, entries }
2893
- : { entries, removals: [], folders: [], folderRemovals: [] });
2832
+ : { entries, removals: [], folders: [], folderRemovals: [] }));
2894
2833
  return true;
2895
2834
  });
2896
2835
  if (!staged)
2897
2836
  return false;
2837
+ if (last)
2838
+ return true;
2839
+ written = end;
2898
2840
  }
2899
- return true;
2900
2841
  },
2901
2842
  };
2902
2843
  }
2903
- function batched(entries) {
2904
- const batches = [[]];
2844
+ function batchEnd(entries, from) {
2845
+ let end = from;
2905
2846
  let spent = 0;
2906
- for (const entry of entries) {
2907
- const size = (entry.content?.length ?? 0) + ROW_WEIGHT;
2908
- if (spent + size > STAGE_BYTES && batches.at(-1).length > 0) {
2909
- batches.push([]);
2910
- spent = 0;
2911
- }
2912
- batches.at(-1).push(entry);
2847
+ while (end < entries.length) {
2848
+ const size = (entries[end].content?.length ?? 0) + ROW_WEIGHT;
2849
+ if (spent + size > stageBytes && end > from)
2850
+ break;
2913
2851
  spent += size;
2852
+ end += 1;
2853
+ }
2854
+ return end;
2855
+ }
2856
+ async function timed(work) {
2857
+ const started = performance.now();
2858
+ const done = await work();
2859
+ const spent = performance.now() - started;
2860
+ if (spent < STAGE_AIM_MS / 2)
2861
+ stageBytes = Math.min(STAGE_BYTES, Math.round(stageBytes * 1.5));
2862
+ else if (spent > STAGE_AIM_MS * 2) {
2863
+ stageBytes = Math.max(STAGE_FLOOR, Math.round(stageBytes / 1.5));
2914
2864
  }
2915
- return batches;
2865
+ return done;
2866
+ }
2867
+ function installWrite(work) {
2868
+ const next = installWrites.then(() => exclusive(work), () => exclusive(work));
2869
+ installWrites = next.then(() => undefined, () => undefined);
2870
+ return next;
2871
+ }
2872
+ function screened() {
2873
+ if (onScreen || closed)
2874
+ return Promise.resolve();
2875
+ queueReadiness();
2876
+ return new Promise((resolve) => shownWaiters.push(resolve));
2877
+ }
2878
+ async function heldPart() {
2879
+ while (partsFree === 0)
2880
+ await new Promise((admit) => partWaiters.push(admit));
2881
+ partsFree -= 1;
2882
+ return freed();
2883
+ }
2884
+ function freed() {
2885
+ let released = false;
2886
+ return () => {
2887
+ if (released)
2888
+ return;
2889
+ released = true;
2890
+ partsFree += 1;
2891
+ partWaiters.shift()?.();
2892
+ };
2893
+ }
2894
+ function installing(locator, step, share = null) {
2895
+ installs.set(locator, { step, share });
2896
+ queueReadiness();
2897
+ }
2898
+ function installed(locator) {
2899
+ if (installs.delete(locator))
2900
+ queueReadiness();
2901
+ }
2902
+ function screen(value) {
2903
+ if (value === onScreen)
2904
+ return;
2905
+ onScreen = value;
2906
+ journal.record(`installation ${value ? 'shown' : 'hidden'}`);
2907
+ queueReadiness();
2908
+ if (!value)
2909
+ return;
2910
+ for (const resolve of shownWaiters.splice(0))
2911
+ resolve();
2916
2912
  }
2917
2913
  async function pullRefresh(authorized, held, liveness) {
2918
2914
  if (!liveness())
@@ -2966,38 +2962,116 @@ export function composeRuntime(options) {
2966
2962
  function staging(local, liveness) {
2967
2963
  return {
2968
2964
  held: (locator) => transfers.get(locator) ?? null,
2969
- async stage(record, page, composition) {
2965
+ async turn(locator, step) {
2966
+ installing(locator, step);
2967
+ for (;;) {
2968
+ await screened();
2969
+ const release = step === 'downloading' ? await heldPart() : () => { };
2970
+ if (onScreen || closed)
2971
+ return release;
2972
+ release();
2973
+ }
2974
+ },
2975
+ spare() {
2976
+ if (!onScreen || closed || partsFree === 0)
2977
+ return null;
2978
+ partsFree -= 1;
2979
+ return freed();
2980
+ },
2981
+ arriving(locator) {
2982
+ let ended = false;
2983
+ return {
2984
+ at(token, bytes) {
2985
+ if (ended)
2986
+ return;
2987
+ const held = arrivals.get(locator) ?? new Map();
2988
+ arrivals.set(locator, held.set(token, bytes));
2989
+ const now = performance.now();
2990
+ if (now - arrivalReported < ARRIVAL_REPORT_MS)
2991
+ return;
2992
+ arrivalReported = now;
2993
+ queueReadiness();
2994
+ },
2995
+ end() {
2996
+ ended = true;
2997
+ if (arrivals.delete(locator))
2998
+ queueReadiness();
2999
+ },
3000
+ };
3001
+ },
3002
+ received: (locator, token) => local.transferPart(locator, token),
3003
+ keep: (locator, token, received) => installWrite(async () => {
3004
+ if (liveness())
3005
+ await local.keepPart(locator, token, received);
3006
+ }),
3007
+ announced(locator) {
3008
+ const later = announced.get(locator);
3009
+ return later === undefined ? [] : [later];
3010
+ },
3011
+ async stage(record, page, composition, received) {
3012
+ const { locator } = record;
2970
3013
  const bytes = pageBytes(page);
2971
- const batches = batched(page.entries);
2972
- let key = record.key;
2973
- for (const [index, entries] of batches.entries()) {
2974
- const last = index === batches.length - 1;
2975
- const staged = await exclusive(async () => {
3014
+ const whole = received?.byteLength ?? 0;
3015
+ const from = record.writing?.written ?? 0;
3016
+ const token = record.writing?.token ?? null;
3017
+ const entries = page.entries.length;
3018
+ let held = record;
3019
+ if (token !== null)
3020
+ arrivals.get(locator)?.delete(token);
3021
+ installing(locator, 'writing', { written: from, entries, bytes: whole });
3022
+ for (let written = from;;) {
3023
+ await screened();
3024
+ if (!liveness())
3025
+ return null;
3026
+ const end = batchEnd(page.entries, written);
3027
+ const last = end === entries;
3028
+ const next = last
3029
+ ? {
3030
+ ...held,
3031
+ writing: null,
3032
+ done: {
3033
+ parts: held.done.parts + (received === null ? 0 : 1),
3034
+ bytes: held.done.bytes + whole,
3035
+ },
3036
+ }
3037
+ : { ...held, writing: { token, written: end } };
3038
+ const key = await installWrite(async () => {
2976
3039
  if (!liveness())
2977
3040
  return null;
2978
- if (index === 0 && !(await admitted(local, record.locator, bytes)))
3041
+ if (written === from && !(await admitted(local, locator, bytes)))
2979
3042
  return null;
2980
3043
  try {
2981
- return await local.stage(record, composition, { entries, folders: last ? page.folders : [] }, key, last);
3044
+ return await timed(() => local.stage(next, composition, {
3045
+ entries: page.entries.slice(written, end),
3046
+ folders: last ? page.folders : [],
3047
+ }, last && received !== null && token !== null ? token : undefined));
2982
3048
  }
2983
3049
  catch (error) {
2984
3050
  if (!exceeded(error))
2985
3051
  throw error;
2986
- await exceededOn(record.locator, bytes);
3052
+ await exceededOn(locator, bytes);
2987
3053
  return null;
2988
3054
  }
2989
3055
  });
2990
- if (staged === null)
3056
+ if (key === null)
2991
3057
  return null;
2992
- key = staged;
3058
+ const arrived = !transfers.has(locator);
3059
+ held = { ...next, key };
3060
+ transfers.set(locator, held);
3061
+ if (arrived)
3062
+ liveQueries.recover();
3063
+ if (last) {
3064
+ installing(locator, held.pageToken === null ? 'replaying' : 'downloading');
3065
+ liveQueries.recover();
3066
+ return held;
3067
+ }
3068
+ installing(locator, 'writing', { written: end, entries, bytes: whole });
3069
+ written = end;
2993
3070
  }
2994
- transfers.set(record.locator, { ...record, key });
2995
- liveQueries.recover();
2996
- return key;
2997
3071
  },
2998
- async refreshing(key) {
3072
+ async refreshing(locator, key) {
2999
3073
  await exclusive(() => local.unrefresh(key));
3000
- return refreshingOf(local, key, liveness);
3074
+ return refreshingOf(local, key, liveness, locator);
3001
3075
  },
3002
3076
  abandon: (locator) => exclusive(async () => {
3003
3077
  if (!liveness())
@@ -3043,13 +3117,16 @@ export function composeRuntime(options) {
3043
3117
  }, (models, held) => (liveness() ? remodel(room.locator, models, held) : held), liveness, staging(local, liveness), reusable(room, local));
3044
3118
  const detail = result.kind === 'staged' ? result.record.roomVersion : result.kind;
3045
3119
  journal.record(`transfer bootstrap ${room.locator} ${detail}`);
3046
- if (result.kind !== 'staged')
3120
+ if (result.kind !== 'staged') {
3047
3121
  caughtUpTo(room.locator);
3122
+ installed(room.locator);
3123
+ }
3048
3124
  return result;
3049
3125
  }
3050
3126
  catch (error) {
3051
3127
  journal.record(`transfer bootstrap ${room.locator} transport-uncertainty`);
3052
3128
  caughtUpTo(room.locator);
3129
+ installed(room.locator);
3053
3130
  throw error;
3054
3131
  }
3055
3132
  finally {
@@ -3114,12 +3191,11 @@ export function composeRuntime(options) {
3114
3191
  return unconfirmed;
3115
3192
  }
3116
3193
  const bootstrapped = await pullBootstrap(authorized, room, local, live);
3117
- if (!live())
3118
- return unconfirmed;
3119
- if (bootstrapped.kind !== 'staged')
3120
- return bootstrapped;
3121
- await publish(local, room, bootstrapped.record, { kind: 'bootstrapped', refreshed: bootstrapped.refreshed }, live);
3122
- return null;
3194
+ if (bootstrapped.kind === 'staged') {
3195
+ await publish(local, room, bootstrapped.record, { kind: 'bootstrapped', refreshed: bootstrapped.refreshed }, live);
3196
+ return live() ? null : unconfirmed;
3197
+ }
3198
+ return live() ? bootstrapped : unconfirmed;
3123
3199
  }
3124
3200
  async function withhold(withheld, room, local, progress) {
3125
3201
  if (withheld.kind === 'unconfirmed')
@@ -3753,9 +3829,8 @@ export function composeRuntime(options) {
3753
3829
  },
3754
3830
  ],
3755
3831
  });
3756
- recomposed(room.locator, held.record, record);
3757
- const { count } = committedRoom(room.locator, record, answer);
3758
- replicas.set(room.locator, { ...held, record, overlaid: diff.touched, count });
3832
+ const { count, indexed } = committedRoom(room.locator, answer);
3833
+ replicas.set(room.locator, { ...held, record, overlaid: diff.touched, count, indexed });
3759
3834
  changed = changeSet(room.locator, {
3760
3835
  documents: answer.changed[room.locator] ?? [],
3761
3836
  room: true,
@@ -3892,10 +3967,9 @@ export function composeRuntime(options) {
3892
3967
  if (!live() || authorization === undefined)
3893
3968
  return;
3894
3969
  const bootstrapped = await pullBootstrap(authorization, room, local, live);
3895
- if (!live())
3896
- return;
3897
3970
  if (bootstrapped.kind !== 'staged') {
3898
- await withhold(bootstrapped, room, local, progress);
3971
+ if (live())
3972
+ await withhold(bootstrapped, room, local, progress);
3899
3973
  return;
3900
3974
  }
3901
3975
  await publish(local, room, bootstrapped.record, { kind: 'bootstrapped', refreshed: bootstrapped.refreshed }, live);
@@ -4543,6 +4617,8 @@ export function composeRuntime(options) {
4543
4617
  closed = true;
4544
4618
  if (rechecking !== null)
4545
4619
  clearInterval(rechecking);
4620
+ for (const resolve of shownWaiters.splice(0))
4621
+ resolve();
4546
4622
  connectionLive();
4547
4623
  lockListeners.clear();
4548
4624
  readinessWatches.clear();
@@ -4553,10 +4629,6 @@ export function composeRuntime(options) {
4553
4629
  withdrawServiceMode?.();
4554
4630
  conversations.close();
4555
4631
  void recovery.retire();
4556
- store?.close();
4557
- void opening?.then((opened) => {
4558
- opened.close();
4559
- }, () => { });
4560
4632
  roomListeners.clear();
4561
4633
  refusalListeners.clear();
4562
4634
  storageLossListeners.clear();
@@ -4632,9 +4704,8 @@ export function composeRuntime(options) {
4632
4704
  }
4633
4705
  async function searched(input, reach, locked) {
4634
4706
  const asked = searchPlanOf(input);
4635
- const plan = { ...asked, limit: capped(asked.limit) };
4636
4707
  const filter = {
4637
- ...filterOf(plan.documents ?? {}, roomFacts(reach), planReach(reach)),
4708
+ ...filterOf(asked.documents ?? {}, roomFacts(reach), planReach(reach)),
4638
4709
  locked,
4639
4710
  };
4640
4711
  const unlocking = (filter.rooms ?? [])
@@ -4643,65 +4714,41 @@ export function composeRuntime(options) {
4643
4714
  for (;;) {
4644
4715
  const cover = covered(filter);
4645
4716
  awaitPreparation(cover);
4646
- const pending = [
4647
- ...unlocking,
4648
- ...('pending' in cover ? cover.pending : []),
4649
- ];
4650
- const awaited = new Set('pending' in cover ? cover.pending.map(({ room }) => room) : []);
4651
- const held = covered({
4652
- ...filter,
4653
- rooms: reachedRooms(filter).filter((room) => !awaited.has(room)),
4654
- });
4655
- const covering = 'pending' in held ? [] : held.coverage.rooms;
4656
- const unindexed = covering
4657
- .filter(({ room }) => !searchIndex.rooms().has(room))
4658
- .map(({ room }) => room);
4659
- for (const room of unindexed)
4660
- pending.push({ room, awaits: 'index' });
4661
- const listed = conversations.searchable();
4662
- const named = reach !== null && !reach.conversations
4663
- ? []
4664
- : (plan.conversations ?? listed).filter((one) => listed.includes(one) && conversations.acquired(one));
4665
- const waiting = named.filter((one) => !searchIndex.conversations().has(one));
4666
- for (const conversation of waiting)
4667
- pending.push({ conversation, awaits: 'index' });
4668
- if (unindexed.length > 0 || waiting.length > 0)
4669
- index(unindexed, waiting);
4717
+ const pending = [...unlocking, ...('pending' in cover ? cover.pending : [])];
4670
4718
  const complete = pending.length === 0 && !('pending' in cover);
4719
+ const awaited = new Set('pending' in cover ? cover.pending.map(({ room }) => room) : []);
4671
4720
  const answering = complete
4672
4721
  ? filter
4673
- : {
4674
- ...filter,
4675
- rooms: covering
4676
- .filter(({ room }) => searchIndex.rooms().has(room))
4677
- .map(({ room }) => room),
4678
- };
4722
+ : { ...filter, rooms: reachedRooms(filter).filter((room) => !awaited.has(room)) };
4679
4723
  const answered = covered(answering);
4680
4724
  if ('pending' in answered)
4681
4725
  continue;
4682
- const searchedConversations = named.filter((one) => searchIndex.conversations().has(one));
4683
- const seen = searchIndex.changes();
4684
- await searchIndex.caughtUp();
4685
- const selected = roomsAlone(answering) ? null : await everyRow(answering);
4686
- if (selected !== null && 'pending' in selected)
4687
- continue;
4688
- if (searchIndex.changes() !== seen ||
4689
- JSON.stringify(covered(answering)) !== JSON.stringify(answered))
4726
+ const listed = conversations.searchable();
4727
+ const named = reach !== null && !reach.conversations
4728
+ ? []
4729
+ : (asked.conversations ?? listed).filter((one) => listed.includes(one) && conversations.acquired(one));
4730
+ const hits = await (await openLocal()).search({
4731
+ text: asked.text,
4732
+ limit: capped(asked.limit),
4733
+ after: asked.after,
4734
+ total: asked.total,
4735
+ documents: {
4736
+ plan: answering,
4737
+ rooms: answered.rooms,
4738
+ beneath: asked.collection
4739
+ ? answered.rooms.map(({ locator, key }) => ({
4740
+ key,
4741
+ roots: replicas.get(locator)?.record.typedSelection?.roots ?? [],
4742
+ }))
4743
+ : [],
4744
+ },
4745
+ conversations: named,
4746
+ });
4747
+ if (JSON.stringify(covered(answering)) !== JSON.stringify(answered))
4690
4748
  continue;
4691
- const hits = searchIndex.search(plan, selected === null ? null : selected.documents, searchedConversations, answering.rooms !== null
4692
- ? new Set(answering.rooms)
4693
- : locked.length === 0
4694
- ? null
4695
- : new Set(reachedRooms(answering)), plan.collection
4696
- ? new Map(answered.coverage.rooms.map(({ room }) => [
4697
- room,
4698
- replicas.get(room)?.record.typedSelection?.roots ?? [],
4699
- ]))
4700
- : null);
4701
4749
  const coverage = {
4702
4750
  rooms: answered.coverage.rooms,
4703
- conversations: searchedConversations,
4704
- index: searchIndex.generation(),
4751
+ conversations: named,
4705
4752
  overlay: answered.coverage.overlay,
4706
4753
  };
4707
4754
  return complete
@@ -4943,7 +4990,7 @@ export function composeRuntime(options) {
4943
4990
  },
4944
4991
  async retry() {
4945
4992
  journal.record('retry');
4946
- unindexable.clear();
4993
+ unpreparable.clear();
4947
4994
  queueReadiness();
4948
4995
  resume();
4949
4996
  },
@@ -4991,6 +5038,7 @@ export function composeRuntime(options) {
4991
5038
  journal.record(`persistence ${answer}`);
4992
5039
  queueReadiness();
4993
5040
  },
5041
+ shown: screen,
4994
5042
  within: (reach, port) => narrowed(reachable, reach, port),
4995
5043
  };
4996
5044
  }