@agentshouse/kit 0.2.1-ceeaf32286734 → 0.2.1-ceffdba88db37

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/dist/copies.js CHANGED
@@ -1,17 +1,31 @@
1
- import { parse } from '@agentshouse/mdmodel';
2
- import { randomUUID } from 'node:crypto';
1
+ import { HOUSE_ROOM_TYPES, readTypes } from '@agentshouse/mdmodel';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
3
  import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { finalRefusal, HouseRefusal, retrying } from './api.js';
6
+ import { chunkTexts, stagedChunks } from './chunks.js';
6
7
  import { blobId, blobText, git, isText, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
8
+ import { paused } from './files.js';
7
9
  import { kitHome } from './home.js';
8
10
  import { operationId } from './operation.js';
9
11
  import { refusalOf } from './refusals.js';
10
- import { changed } from './translate.js';
11
- import { relayed } from './relay.js';
12
+ import { changed, readDocument } from './translate.js';
12
13
  const BRANCH = 'main';
13
14
  const PROVENANCE = '_provenance';
14
15
  const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
16
+ const ROOMS = 'rooms/';
17
+ const NOT_READY = 'not ready yet; Kit copies it once House has built it';
18
+ const STAGED_CHUNKS = 'house-chunks';
19
+ const SCHEMA = 'settings/schema.yaml';
20
+ // A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
21
+ const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
22
+ class NotReady extends Error {
23
+ rooms;
24
+ constructor(rooms) {
25
+ super(NOT_READY);
26
+ this.rooms = rooms;
27
+ }
28
+ }
15
29
  function causeOf(error) {
16
30
  return error instanceof Error ? error.message : String(error);
17
31
  }
@@ -72,6 +86,9 @@ function sidecar(path) {
72
86
  function recordOf(key) {
73
87
  return join(kitHome(), 'local-copies', `${key}.json`);
74
88
  }
89
+ function stagingPrefix() {
90
+ return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
91
+ }
75
92
  function rootOf(handle) {
76
93
  return `/rooms/${handle}`;
77
94
  }
@@ -99,32 +116,59 @@ async function removeFile(repository, path) {
99
116
  return;
100
117
  }
101
118
  }
102
- async function readOnlyPlaced(repository, held, written, removed) {
119
+ async function readOnlyFiles(repository, held) {
103
120
  const tracked = new Set((await git(repository, ['ls-files', '-z'])).split('\0'));
104
121
  const holding = new Set(held);
105
- for (const path of removed) {
106
- if (!holding.has(path))
107
- continue;
108
- if (!tracked.has(path))
109
- await removeFile(repository, path);
110
- holding.delete(path);
111
- }
112
- for (const [path, content] of written) {
113
- if (tracked.has(path))
114
- continue;
115
- if (!holding.has(path) && (await exists(join(repository, path))))
116
- continue;
117
- await writeText(join(repository, path), content);
118
- holding.add(path);
119
- }
120
- return [...holding].sort();
122
+ return {
123
+ remove: async (path) => {
124
+ if (!holding.has(path))
125
+ return;
126
+ if (!tracked.has(path))
127
+ await removeFile(repository, path);
128
+ holding.delete(path);
129
+ },
130
+ place: async (path, content) => {
131
+ if (tracked.has(path))
132
+ return;
133
+ if (!holding.has(path) && (await exists(join(repository, path))))
134
+ return;
135
+ await writeText(join(repository, path), content);
136
+ holding.add(path);
137
+ },
138
+ held: () => [...holding].sort(),
139
+ };
140
+ }
141
+ async function readOnlyPlaced(repository, held, written, removed) {
142
+ const files = await readOnlyFiles(repository, held);
143
+ for (const path of removed)
144
+ await files.remove(path);
145
+ for (const [path, content] of written)
146
+ await files.place(path, content);
147
+ return files.held();
148
+ }
149
+ async function importedWhole(repository, scope, texts, commit, readOnly) {
150
+ const revisions = {};
151
+ async function* tracked() {
152
+ for await (const { path, content, revision: held, provenance } of texts) {
153
+ if (declaredWritable(path, scope)) {
154
+ revisions[path] = held;
155
+ yield [path, content];
156
+ }
157
+ else {
158
+ await readOnly.place(path, content);
159
+ }
160
+ if (provenance !== undefined)
161
+ await readOnly.place(sidecar(path), provenance);
162
+ }
163
+ }
164
+ return { house: await importCommits(repository, [{ ...commit, files: tracked() }]), revisions };
121
165
  }
122
166
  async function ruled(path, rules) {
123
167
  if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
124
168
  await writeFile(path, rules);
125
169
  }
126
170
  async function ignoreRules(repository, writable, copy) {
127
- const readOnly = ('read_only' in copy ? copy.read_only : [...copy.readOnly.keys()]).filter((path) => !path.split('/').includes(PROVENANCE));
171
+ const readOnly = copy.read_only.filter((path) => !path.split('/').includes(PROVENANCE));
128
172
  const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
129
173
  const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
130
174
  const house = [
@@ -155,6 +199,12 @@ async function ignoreRules(repository, writable, copy) {
155
199
  '',
156
200
  ].join('\n'));
157
201
  }
202
+ async function modelOf(copy, files) {
203
+ const entry = files.get(SCHEMA);
204
+ if (entry !== undefined)
205
+ return readTypes(await blobText(copy.repository, entry), HOUSE_ROOM_TYPES);
206
+ return readTypes(copy.read_only.includes(SCHEMA) ? await readFile(join(copy.repository, SCHEMA), 'utf8') : '', HOUSE_ROOM_TYPES);
207
+ }
158
208
  function revisionAt(copy, commit, path) {
159
209
  let manifest = copy.manifests[commit];
160
210
  while (manifest !== undefined) {
@@ -184,6 +234,7 @@ export class LocalCopies {
184
234
  house;
185
235
  copies = new Map();
186
236
  queues = new Map();
237
+ waiting = new Set();
187
238
  answering = Promise.resolve();
188
239
  constructor(house) {
189
240
  this.house = house;
@@ -191,7 +242,7 @@ export class LocalCopies {
191
242
  key: 'owner',
192
243
  submit: async (operation, changes) => {
193
244
  try {
194
- const outcome = await retrying(() => house.post('/kit/door/batch', { operation_id: operation, changes }));
245
+ const outcome = await retrying(() => house.door('files/batch', { operation_id: operation, changes }));
195
246
  return { accepted: outcome.contents };
196
247
  }
197
248
  catch (error) {
@@ -215,25 +266,21 @@ export class LocalCopies {
215
266
  this.copies.set(copy.key, copy);
216
267
  }
217
268
  }
218
- async mapping() {
219
- await this.answering;
220
- return [...this.copies.values()]
221
- .filter((copy) => copy.unfollowed === undefined)
222
- .map((copy) => `${rootOf(copy.room_handle)}: ${copy.repository}`)
223
- .sort();
269
+ async swept() {
270
+ const rooms = join(localCopies(), 'rooms');
271
+ for (const name of await readdir(rooms).catch(() => [])) {
272
+ if (name.startsWith(stagingPrefix()))
273
+ await rm(join(rooms, name), { recursive: true, force: true });
274
+ }
224
275
  }
225
276
  connected() {
226
277
  return this.selected();
227
278
  }
228
- async changed(rooms) {
229
- if (rooms.length > 0)
230
- await this.selected(rooms);
231
- }
232
279
  fetch() {
233
280
  return this.answered(() => this.fetched(true));
234
281
  }
235
- async selected(rooms) {
236
- const copied = await this.answered(() => this.fetched(false, rooms));
282
+ async selected() {
283
+ const copied = await this.answered(() => this.fetched(false));
237
284
  if (copied.refused)
238
285
  logged(copied.text);
239
286
  }
@@ -242,16 +289,14 @@ export class LocalCopies {
242
289
  this.answering = answered.catch(() => undefined);
243
290
  return answered;
244
291
  }
245
- async fetched(refreshing, rooms) {
246
- const { local_copy: answer } = await this.house.post('/kit/local-copy/selection', rooms === undefined ? {} : { rooms });
247
- const synced = new Map(answer.rooms.map((room) => [`rooms/${room.room_ref}`, room.room_handle]));
248
- const named = rooms === undefined ? null : new Set(rooms.map((room) => `rooms/${room}`));
249
- const held = () => [...this.copies.values()].filter((copy) => named === null || named.has(copy.key));
250
- return this.holding([...new Set([...held().map((copy) => copy.key), ...synced.keys()])].sort(), async () => {
292
+ async fetched(refreshing) {
293
+ const { local_copy: answer } = await this.house.act('local-copies/selection/read', {});
294
+ const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
295
+ return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
251
296
  const said = [];
252
297
  let refused = false;
253
298
  const mark = refreshing ? 'said' : 'unsaid';
254
- for (const copy of held()) {
299
+ for (const copy of [...this.copies.values()]) {
255
300
  if (synced.has(copy.key) || copy.unfollowed === 'said' || copy.unfollowed === mark)
256
301
  continue;
257
302
  await this.save({ ...copy, unfollowed: mark });
@@ -259,11 +304,12 @@ export class LocalCopies {
259
304
  said.push(`${rootOf(copy.room_handle)}: no longer followed; kept as it is`);
260
305
  }
261
306
  const current = [];
307
+ const wanted = new Map();
262
308
  for (const [key, handle] of synced) {
263
309
  try {
264
310
  const held = this.copies.get(key);
265
311
  if (held === undefined || !(await exists(held.repository))) {
266
- said.push(await this.materialize(key, handle));
312
+ wanted.set(key, handle);
267
313
  continue;
268
314
  }
269
315
  const { unfollowed, ...followed } = held;
@@ -274,18 +320,62 @@ export class LocalCopies {
274
320
  said.push(`${rootOf(handle)}: ${causeOf(error)}`);
275
321
  }
276
322
  }
277
- if (refreshing && current.length > 0) {
278
- const failed = await this.refreshed(current).then(() => null, houseCause);
279
- for (const copy of current)
280
- said.push(...moved(copy, this.copies.get(copy.key)));
323
+ const unready = [];
324
+ const unsettled = [];
325
+ if (wanted.size > 0) {
326
+ const { authorities } = await this.discovered([...wanted.values()]);
327
+ for (const [key, handle] of wanted) {
328
+ const authority = authorities.find((entry) => `${ROOMS}${entry.room_ref}` === key);
329
+ try {
330
+ if (authority === undefined)
331
+ throw new Error('House serves nothing of it to copy');
332
+ said.push(await this.copied({ key, handle, authority }));
333
+ }
334
+ catch (error) {
335
+ if (error instanceof NotReady) {
336
+ unready.push(...error.rooms);
337
+ said.push(`${rootOf(handle)}: ${NOT_READY}`);
338
+ }
339
+ else {
340
+ refused = true;
341
+ unsettled.push(key.slice(ROOMS.length));
342
+ said.push(`${rootOf(handle)}: ${causeOf(error)}`);
343
+ }
344
+ }
345
+ }
346
+ }
347
+ const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
348
+ if (reading.length > 0) {
349
+ const failed = await this.refreshed(reading, unready).then(() => null, houseCause);
350
+ for (const copy of reading) {
351
+ said.push(...(unready.includes(copy.room_ref) ? [`${rootOf(copy.room_handle)}: ${NOT_READY}`] : moved(copy, this.copies.get(copy.key))));
352
+ }
281
353
  if (failed !== null) {
282
354
  refused = true;
355
+ unsettled.push(...reading.map((copy) => copy.room_ref));
283
356
  said.push(failed);
284
357
  }
285
358
  }
359
+ for (const room of [...this.waiting]) {
360
+ if (!unready.includes(room) && !unsettled.includes(room))
361
+ this.waiting.delete(room);
362
+ }
363
+ const due = unready.filter((room) => !this.waiting.has(room));
364
+ for (const room of due)
365
+ this.waiting.add(room);
366
+ if (due.length > 0)
367
+ void this.askedAgain(due);
286
368
  return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
287
369
  });
288
370
  }
371
+ async askedAgain(rooms) {
372
+ let asking = rooms;
373
+ for (let step = 0; asking.length > 0; step++) {
374
+ await paused(ASKED_AGAIN_MS[Math.min(step, ASKED_AGAIN_MS.length - 1)]);
375
+ await this.selected().catch(logged);
376
+ asking = asking.filter((room) => this.waiting.has(room));
377
+ }
378
+ }
289
379
  async observed(report) {
290
380
  const key = await this.keyAt(dirname(report.common));
291
381
  if (key === undefined)
@@ -347,132 +437,170 @@ export class LocalCopies {
347
437
  this.copies.set(copy.key, copy);
348
438
  return copy;
349
439
  }
350
- async read(handle) {
351
- const prefixes = [rootOf(handle)];
352
- const discovery = await this.house.post('/kit/door/discover', { prefixes });
353
- let bundle = await this.house.post('/kit/door/bootstrap', { prefixes });
354
- if ('transfer' in bundle)
355
- bundle = (await (await fetch(relayed(bundle.transfer.url))).json());
356
- const [view] = bundle.scopes;
357
- if (view === undefined)
358
- return null;
359
- const authority = discovery.authorities.find((entry) => entry.room_ref === view.manifest.anchor);
360
- const contents = new Map(view.contents.map((entry) => [entry.path, entry.content]));
361
- const read = {
362
- anchor: view.manifest.anchor,
363
- position: view.manifest.position,
364
- log_epoch: view.manifest.log_epoch,
365
- write: authority.operations.write,
366
- protected: authority.protected_paths,
367
- writable: new Map(),
368
- revisions: {},
369
- readOnly: new Map(),
370
- };
371
- for (const file of view.manifest.files) {
372
- const content = contents.get(file.path);
373
- if (file.writable) {
374
- read.writable.set(file.path, content);
375
- read.revisions[file.path] = file.revision;
376
- }
377
- else {
378
- read.readOnly.set(file.path, content);
440
+ async discovered(handles) {
441
+ const discover = (group) => this.house.door('rooms/discover', { prefixes: group.map(rootOf) });
442
+ const [first, ...rest] = handles;
443
+ const { bounds, authorities } = await discover([first]);
444
+ for (const group of grouped(rest, bounds.max_prefixes))
445
+ authorities.push(...(await discover(group)).authorities);
446
+ return { bounds, authorities };
447
+ }
448
+ async chunked(root, room, directory) {
449
+ await mkdir(directory, { recursive: true });
450
+ for (let renewed = false;; renewed = true) {
451
+ const { rooms } = await this.house.door('room-chunks/list', { prefixes: [root] });
452
+ const list = rooms.find((entry) => entry.room_ref === room);
453
+ if (list === undefined)
454
+ throw new Error('House serves nothing of it to copy');
455
+ if (!list.ready)
456
+ throw new NotReady([room]);
457
+ const held = new Set(await readdir(directory));
458
+ const failure = await stagedChunks(directory, list.chunks.filter(({ hash }) => !held.has(hash)));
459
+ if (failure === undefined) {
460
+ return { position: list.position, log_epoch: list.log_epoch, texts: chunkTexts(directory, list.chunks, list.models) };
379
461
  }
380
- if (file.provenance !== undefined)
381
- read.readOnly.set(sidecar(file.path), file.provenance);
462
+ if (renewed && (await readdir(directory)).length === held.size)
463
+ throw failure;
382
464
  }
383
- return read;
384
465
  }
385
- async materialize(key, handle) {
386
- const read = await this.read(handle);
387
- if (read === null)
388
- throw new Error('House serves nothing of it to copy');
389
- const repository = join(localCopies(), 'rooms', read.anchor);
466
+ async materialized({ key, handle, authority }, whole) {
467
+ const repository = join(localCopies(), 'rooms', authority.room_ref);
390
468
  if (await exists(repository))
391
469
  throw new Error(`${repository} already exists and Kit leaves it as it is`);
392
- const staging = join(dirname(repository), `.${randomUUID()}`);
470
+ const staging = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
471
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
393
472
  await mkdir(staging, { recursive: true });
394
- let house;
395
- let readOnly;
473
+ let copy;
396
474
  try {
397
475
  await git(staging, ['init', '-q', '-b', BRANCH]);
398
- house = await importCommits(staging, [
399
- { message: `House ${rootOf(handle)}`, from: null, files: read.writable },
400
- ]);
476
+ const read = await whole(join(staging, '.git', STAGED_CHUNKS));
477
+ const readOnly = await readOnlyFiles(staging, []);
478
+ const { house, revisions } = await importedWhole(staging, scope, read.texts, { message: `House ${rootOf(handle)}`, from: null }, readOnly);
401
479
  await git(staging, ['update-ref', `refs/heads/${BRANCH}`, house]);
402
480
  await git(staging, ['reset', '-q', '--hard']);
403
- readOnly = await readOnlyPlaced(staging, [], read.readOnly, []);
404
- await ignoreRules(staging, [...read.writable.keys()], read);
481
+ copy = {
482
+ key,
483
+ room_ref: authority.room_ref,
484
+ room_handle: handle,
485
+ repository,
486
+ house,
487
+ position: read.position,
488
+ log_epoch: read.log_epoch,
489
+ ...scope,
490
+ read_only: readOnly.held(),
491
+ manifests: { [house]: { parent: null, files: revisions } },
492
+ moves: [],
493
+ };
494
+ await ignoreRules(staging, Object.keys(revisions), copy);
495
+ await rm(join(staging, '.git', STAGED_CHUNKS), { recursive: true, force: true });
405
496
  await rename(staging, repository);
406
497
  }
407
498
  catch (error) {
408
499
  await rm(staging, { recursive: true, force: true });
409
500
  throw error;
410
501
  }
411
- await this.save({
412
- key,
413
- room_ref: read.anchor,
414
- room_handle: handle,
415
- repository,
416
- house,
417
- position: read.position,
418
- log_epoch: read.log_epoch,
419
- write: read.write,
420
- protected: read.protected,
421
- read_only: readOnly,
422
- manifests: { [house]: { parent: null, files: read.revisions } },
423
- moves: [],
424
- });
425
- return `${rootOf(handle)}: copied to ${repository}`;
502
+ return this.save(copy);
426
503
  }
427
- async received(copy) {
428
- const read = await this.read(copy.room_handle);
429
- if (read === null)
430
- return copy;
431
- const house = await importCommits(copy.repository, [
432
- { message: `House ${rootOf(copy.room_handle)} at position ${read.position}`, from: copy.house, replaces: true, files: read.writable },
433
- ]);
434
- const removed = copy.read_only.filter((path) => !read.readOnly.has(path));
435
- const readOnly = await readOnlyPlaced(copy.repository, copy.read_only, read.readOnly, removed);
436
- const next = {
437
- ...copy,
438
- house,
439
- position: read.position,
440
- log_epoch: read.log_epoch,
441
- write: read.write,
442
- protected: read.protected,
443
- read_only: readOnly,
444
- manifests: { ...copy.manifests, [house]: { parent: null, files: read.revisions } },
445
- };
446
- await ignoreRules(copy.repository, [...read.writable.keys()], next);
447
- return next;
504
+ async copied(wanted) {
505
+ const { handle, authority } = wanted;
506
+ const root = rootOf(handle);
507
+ let answer = null;
508
+ try {
509
+ if (!this.waiting.has(authority.room_ref)) {
510
+ answer = await this.house.door('changes/enumerate', {
511
+ prefixes: [root],
512
+ positions: [
513
+ { authority: authority.room_ref, position: '0', log_epoch: authority.log_epoch, page: { through: authority.position, read: 0 } },
514
+ ],
515
+ });
516
+ }
517
+ }
518
+ catch (error) {
519
+ if (expiredAuthority(error) !== authority.room_ref)
520
+ throw error;
521
+ }
522
+ const view = answer?.authorities.find((entry) => entry.authority === authority.room_ref);
523
+ if (answer !== null && view === undefined)
524
+ throw new Error('House serves nothing of it to copy');
525
+ if (view !== undefined && view.next === undefined) {
526
+ const texts = view.changes.filter((change) => !('removed' in change));
527
+ const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, texts }));
528
+ return `${root}: copied to ${copy.repository}`;
529
+ }
530
+ const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
531
+ const [caught] = await this.caughtUp([copy], true);
532
+ await succeeded(copy.repository, ['merge', '--ff-only', '-q', caught.house]);
533
+ return `${root}: copied to ${copy.repository}`;
448
534
  }
449
- async refreshed(copies) {
450
- const discover = (group) => this.house.post('/kit/door/discover', { prefixes: group.map((copy) => rootOf(copy.room_handle)) });
451
- const [first, ...rest] = copies;
452
- const { bounds, authorities } = await discover([first]);
453
- for (const group of grouped(rest, bounds.max_prefixes))
454
- authorities.push(...(await discover(group)).authorities);
535
+ async received(copy, authority) {
536
+ const root = rootOf(copy.room_handle);
537
+ const directory = join(copy.repository, '.git', STAGED_CHUNKS);
538
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
539
+ try {
540
+ const read = await this.chunked(root, copy.room_ref, directory);
541
+ const readOnly = await readOnlyFiles(copy.repository, copy.read_only);
542
+ for (const path of copy.read_only)
543
+ await readOnly.remove(path);
544
+ const commit = { message: `House ${root} at position ${read.position}`, from: copy.house, replaces: true };
545
+ const { house, revisions } = await importedWhole(copy.repository, scope, read.texts, commit, readOnly);
546
+ const next = {
547
+ ...copy,
548
+ ...scope,
549
+ house,
550
+ position: read.position,
551
+ log_epoch: read.log_epoch,
552
+ read_only: readOnly.held(),
553
+ manifests: { ...copy.manifests, [house]: { parent: null, files: revisions } },
554
+ };
555
+ await ignoreRules(copy.repository, Object.keys(revisions), next);
556
+ return next;
557
+ }
558
+ finally {
559
+ await rm(directory, { recursive: true, force: true });
560
+ }
561
+ }
562
+ async reread(copy) {
563
+ const { authorities } = await this.discovered([copy.room_handle]);
564
+ const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
565
+ if (authority === undefined)
566
+ throw new NotReady([copy.room_ref]);
567
+ return this.received(copy, authority);
568
+ }
569
+ async refreshed(copies, unready) {
570
+ const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
571
+ const read = [];
455
572
  const unchanged = [];
456
573
  for (const copy of copies) {
457
574
  const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
458
575
  if (authority !== undefined &&
459
576
  JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
460
- await this.save(await this.received(copy));
577
+ try {
578
+ read.push(await this.save(await this.received(copy, authority)));
579
+ }
580
+ catch (error) {
581
+ if (!(error instanceof NotReady))
582
+ throw error;
583
+ unready.push(...error.rooms);
584
+ }
461
585
  }
462
586
  else {
463
587
  unchanged.push(copy);
464
588
  }
465
589
  }
466
590
  for (const group of grouped(unchanged, bounds.max_prefixes))
467
- await this.caughtUp(group, false);
591
+ await this.caughtUp(group, false, unready);
592
+ for (const group of grouped(read, bounds.max_prefixes))
593
+ await this.caughtUp(group, true, unready);
468
594
  }
469
- async caughtUp(copies, whole) {
595
+ async caughtUp(copies, whole, unready) {
470
596
  const caught = new Map(copies.map((copy) => [copy.key, copy]));
597
+ const read = new Set();
598
+ const waiting = unready ?? [];
471
599
  let asking = [...copies];
472
600
  while (asking.length > 0) {
473
601
  let answer;
474
602
  try {
475
- answer = await this.house.post('/kit/door/enumerate', {
603
+ answer = await this.house.door('changes/enumerate', {
476
604
  prefixes: asking.map((copy) => rootOf(copy.room_handle)),
477
605
  positions: asking.map((copy) => ({ authority: copy.room_ref, position: copy.position, log_epoch: copy.log_epoch })),
478
606
  });
@@ -481,8 +609,22 @@ export class LocalCopies {
481
609
  const expired = asking.find((copy) => copy.room_ref === expiredAuthority(error));
482
610
  if (expired === undefined)
483
611
  throw error;
484
- caught.set(expired.key, await this.save(await this.received(expired)));
485
612
  asking = asking.filter((copy) => copy !== expired);
613
+ if (read.has(expired.key)) {
614
+ waiting.push(expired.room_ref);
615
+ continue;
616
+ }
617
+ read.add(expired.key);
618
+ try {
619
+ const next = await this.save(await this.reread(expired));
620
+ caught.set(expired.key, next);
621
+ asking.push(next);
622
+ }
623
+ catch (refused) {
624
+ if (!(refused instanceof NotReady))
625
+ throw refused;
626
+ waiting.push(expired.room_ref);
627
+ }
486
628
  continue;
487
629
  }
488
630
  const advanced = [];
@@ -494,8 +636,10 @@ export class LocalCopies {
494
636
  caught.set(copy.key, next);
495
637
  advanced.push(next);
496
638
  }
497
- asking = whole ? advanced : [];
639
+ asking = advanced.filter((copy) => whole || read.has(copy.key));
498
640
  }
641
+ if (unready === undefined && waiting.length > 0)
642
+ throw new NotReady(waiting);
499
643
  return copies.map((copy) => caught.get(copy.key));
500
644
  }
501
645
  async incorporated(copy, changes, position, logEpoch) {
@@ -598,7 +742,7 @@ export class LocalCopies {
598
742
  const arrived = new Set(moved.values());
599
743
  const arriving = [...after].filter(([path, entry]) => before.get(path) !== entry);
600
744
  if (arriving.length > 0) {
601
- const { bounds } = await this.house.post('/kit/door/discover', { prefixes: [root] });
745
+ const { bounds } = await this.house.door('rooms/discover', { prefixes: [root] });
602
746
  const binary = [];
603
747
  for (const [path, entry] of arriving) {
604
748
  if (!(await isText(copy.repository, entry, bounds.max_authored_file_bytes)))
@@ -608,6 +752,7 @@ export class LocalCopies {
608
752
  throw new Error(textOnly(binary));
609
753
  }
610
754
  const text = (entry) => blobText(copy.repository, entry);
755
+ const model = await modelOf(copy, after);
611
756
  const changes = [];
612
757
  for (const path of [...new Set([...before.keys(), ...after.keys()])].sort()) {
613
758
  if (arrived.has(path))
@@ -631,7 +776,7 @@ export class LocalCopies {
631
776
  else if (prior === undefined)
632
777
  changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next) });
633
778
  else
634
- changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next)));
779
+ changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next), model));
635
780
  }
636
781
  return changes;
637
782
  }
@@ -687,7 +832,8 @@ export class LocalCopies {
687
832
  const current = (await this.caughtUp([copy], true))[0];
688
833
  const house = await tree(copy.repository, current.house);
689
834
  const root = rootOf(current.room_handle);
690
- if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit)))) {
835
+ const model = await modelOf(current, house);
836
+ if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit), model))) {
691
837
  await this.save(without(current));
692
838
  return {
693
839
  refused: true,
@@ -786,7 +932,7 @@ export class LocalCopies {
786
932
  push: { ...push, acknowledged },
787
933
  };
788
934
  }
789
- async landed(repository, changes, house, selected) {
935
+ async landed(repository, changes, house, selected, model) {
790
936
  const text = async (path) => {
791
937
  const entry = house.get(relative(path));
792
938
  return entry === undefined ? undefined : blobText(repository, entry);
@@ -800,9 +946,8 @@ export class LocalCopies {
800
946
  }
801
947
  if ((change.op === 'create' || change.op === 'replace') && current === change.content)
802
948
  continue;
803
- const parsed = current === undefined ? null : parse(current, change.path);
804
- const document = parsed?.ok === true ? parsed.document : null;
805
- if (document === null)
949
+ const document = current === undefined ? undefined : readDocument(current, relative(change.path), model)?.document;
950
+ if (document === undefined)
806
951
  return false;
807
952
  if (change.op === 'set_field' && JSON.stringify(document.fields[change.field]) === JSON.stringify(change.value))
808
953
  continue;
@@ -815,7 +960,7 @@ export class LocalCopies {
815
960
  return true;
816
961
  }
817
962
  async revisionText(path, wanted) {
818
- const page = await this.house.post('/kit/door/revisions', {
963
+ const page = await this.house.door('files/revisions/list', {
819
964
  path,
820
965
  revision: wanted,
821
966
  });