@agentshouse/kit 0.2.1-ccb257fa8a318 → 0.2.1-ccffb3f637815

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (57) hide show
  1. package/README.md +27 -31
  2. package/dist/acp.js +17 -10
  3. package/dist/afk.js +17 -0
  4. package/dist/agents.js +121 -52
  5. package/dist/api.js +11 -6
  6. package/dist/binaries.js +89 -0
  7. package/dist/bridge.js +28 -27
  8. package/dist/building.js +7 -3
  9. package/dist/calls.js +18 -1
  10. package/dist/chunks.js +49 -0
  11. package/dist/clis.js +52 -63
  12. package/dist/connect.js +108 -0
  13. package/dist/connection.js +24 -20
  14. package/dist/content.js +106 -0
  15. package/dist/conversations.js +255 -134
  16. package/dist/copies.js +748 -248
  17. package/dist/earlier.js +263 -0
  18. package/dist/end.js +9 -6
  19. package/dist/events.js +549 -281
  20. package/dist/files.js +77 -11
  21. package/dist/git-main.js +24 -29
  22. package/dist/git.js +87 -12
  23. package/dist/grok.js +39 -0
  24. package/dist/home.js +80 -29
  25. package/dist/house-cli.js +106 -123
  26. package/dist/instructions.js +2 -4
  27. package/dist/keys.js +238 -0
  28. package/dist/kit-cli.js +14 -4
  29. package/dist/login.js +34 -20
  30. package/dist/logout.js +2 -2
  31. package/dist/mcp.js +38 -3
  32. package/dist/old-references.js +27 -0
  33. package/dist/owner.js +46 -0
  34. package/dist/placement.js +9 -23
  35. package/dist/refusals.js +37 -21
  36. package/dist/resident.js +107 -72
  37. package/dist/scope.js +12 -4
  38. package/dist/scripts.js +59 -11
  39. package/dist/secret-input.js +1 -1
  40. package/dist/settings.js +188 -0
  41. package/dist/sign-in.js +49 -4
  42. package/dist/supervisor.js +163 -0
  43. package/dist/transcription.js +5 -2
  44. package/dist/translate.js +18 -11
  45. package/dist/update.js +115 -160
  46. package/dist/wire.js +19 -9
  47. package/dist/work.js +27 -2
  48. package/dist/workspace.js +103 -0
  49. package/package.json +13 -8
  50. package/dist/authority-main.js +0 -22
  51. package/dist/chatgpt.js +0 -32
  52. package/dist/clis-main.js +0 -23
  53. package/dist/configure-main.js +0 -6
  54. package/dist/enrol-main.js +0 -10
  55. package/dist/parts.js +0 -82
  56. package/dist/service-main.js +0 -9
  57. package/dist/service.js +0 -55
package/dist/copies.js CHANGED
@@ -1,17 +1,35 @@
1
- import { parse } from '@agentshouse/mdmodel';
2
- import { randomUUID } from 'node:crypto';
1
+ import { HOUSE_ROOM_TYPES, readSchemaDocument, readTypes } from '@agentshouse/mdmodel';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
3
  import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { finalRefusal, HouseRefusal, retrying } from './api.js';
6
- import { blobId, blobText, git, isText, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
6
+ import { attachments, eachAtOnce, holds, isDocument, receivedBinary, sentBinary, sweptTransfers, untext, } from './binaries.js';
7
+ import { chunkEntries, stagedChunks } from './chunks.js';
8
+ import { blobId, blobText, git, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
9
+ import { fileInfo, paused } from './files.js';
7
10
  import { kitHome } from './home.js';
11
+ import { ownAgentCall, rpc, toolAnswer, Unsettled } from './mcp.js';
12
+ import { renameOldReferences } from './old-references.js';
8
13
  import { operationId } from './operation.js';
9
- import { refusalOf } from './refusals.js';
10
- import { changed } from './translate.js';
11
- import { relayed } from './relay.js';
14
+ import { firstLine, refusalOf, retryable, stopsPush } from './refusals.js';
15
+ import { changed, readDocument } from './translate.js';
12
16
  const BRANCH = 'main';
13
17
  const PROVENANCE = '_provenance';
14
18
  const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
19
+ const ROOMS = 'rooms/';
20
+ const NOT_READY = 'not ready yet; Kit copies it once House has built it';
21
+ const STAGED_CHUNKS = 'house-chunks';
22
+ const SCHEMA = 'settings/schema.md';
23
+ const SCHEMA_YAML = 'settings/schema.yaml';
24
+ // A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
25
+ const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
26
+ class NotReady extends Error {
27
+ rooms;
28
+ constructor(rooms) {
29
+ super(NOT_READY);
30
+ this.rooms = rooms;
31
+ }
32
+ }
15
33
  function causeOf(error) {
16
34
  return error instanceof Error ? error.message : String(error);
17
35
  }
@@ -28,13 +46,37 @@ const REBASED = new Set([
28
46
  'base_revision_stale',
29
47
  'path_exists',
30
48
  'path_absent',
49
+ 'base_required',
50
+ 'text_file',
31
51
  ]);
32
52
  function rebaseFixes(refusal) {
33
- return refusal.code === 'edit_conflict' && refusal.conflicts.some((code) => REBASED.has(code));
53
+ return refusal.code === 'edit_conflict' && refusal.conflicts.some(({ code }) => REBASED.has(code));
34
54
  }
35
- function textOnly(paths) {
55
+ function markdownOnly(paths, maxBytes) {
36
56
  const [named, them] = paths.length === 1 ? [`${paths[0]} is`, 'it'] : [`${paths.join(', ')} are`, 'them'];
37
- return `${named} not text, and house push sends text only; remove ${them} from the commit and place ${them} with house upload`;
57
+ return `${named} Markdown, which holds text only: UTF-8 without NUL, at most ${maxBytes} bytes; commit ${them} as text and push again`;
58
+ }
59
+ async function stopping(sent) {
60
+ try {
61
+ return await sent;
62
+ }
63
+ catch (error) {
64
+ const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
65
+ if (refusal === null || !stopsPush(refusal))
66
+ throw error;
67
+ return { refused: refusal };
68
+ }
69
+ }
70
+ function changesKind(changes) {
71
+ const removed = new Set(changes.flatMap((change) => (change.op === 'remove' ? [change.path] : [])));
72
+ return changes.some((change) => (change.op === 'create' || change.op === 'upload') && removed.has(change.path));
73
+ }
74
+ function unstagedUploads(refusal) {
75
+ const named = refusal.detail.upload;
76
+ return Array.isArray(named) ? named : [];
77
+ }
78
+ function heldKey(file) {
79
+ return `${file.sha256}:${file.bytes}`;
38
80
  }
39
81
  function outside(path) {
40
82
  return { refused: true, text: `${path} is not a House Local copy; run house push inside one` };
@@ -72,6 +114,9 @@ function sidecar(path) {
72
114
  function recordOf(key) {
73
115
  return join(kitHome(), 'local-copies', `${key}.json`);
74
116
  }
117
+ function stagingPrefix() {
118
+ return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
119
+ }
75
120
  function rootOf(handle) {
76
121
  return `/rooms/${handle}`;
77
122
  }
@@ -99,32 +144,76 @@ async function removeFile(repository, path) {
99
144
  return;
100
145
  }
101
146
  }
102
- async function readOnlyPlaced(repository, held, written, removed) {
147
+ async function readOnlyFiles(repository, held) {
103
148
  const tracked = new Set((await git(repository, ['ls-files', '-z'])).split('\0'));
104
149
  const holding = new Set(held);
105
- for (const path of removed) {
106
- if (!holding.has(path))
107
- continue;
108
- if (!tracked.has(path))
109
- await removeFile(repository, path);
110
- holding.delete(path);
111
- }
112
- for (const [path, content] of written) {
113
- if (tracked.has(path))
114
- continue;
115
- if (!holding.has(path) && (await exists(join(repository, path))))
116
- continue;
117
- await writeText(join(repository, path), content);
118
- holding.add(path);
119
- }
120
- return [...holding].sort();
150
+ return {
151
+ remove: async (path) => {
152
+ if (!holding.has(path))
153
+ return;
154
+ if (!tracked.has(path))
155
+ await removeFile(repository, path);
156
+ holding.delete(path);
157
+ },
158
+ place: async (path, content) => {
159
+ if (tracked.has(path))
160
+ return;
161
+ if (!holding.has(path) && (await exists(join(repository, path))))
162
+ return;
163
+ await writeText(join(repository, path), content);
164
+ holding.add(path);
165
+ },
166
+ held: () => [...holding].sort(),
167
+ };
168
+ }
169
+ async function readOnlyPlaced(repository, held, written, removed) {
170
+ const files = await readOnlyFiles(repository, held);
171
+ for (const path of removed)
172
+ await files.remove(path);
173
+ for (const [path, content] of written)
174
+ await files.place(path, content);
175
+ return files.held();
176
+ }
177
+ async function importedWhole(repository, scope, entries, commit, readOnly, kept) {
178
+ const revisions = {};
179
+ const binary = {};
180
+ async function* tracked() {
181
+ const binaries = [];
182
+ for await (const entry of entries) {
183
+ if (!('content' in entry)) {
184
+ if (declaredWritable(entry.path, scope))
185
+ binaries.push(entry);
186
+ continue;
187
+ }
188
+ const { path, content, revision: held, provenance } = entry;
189
+ if (declaredWritable(path, scope)) {
190
+ revisions[path] = held;
191
+ yield [path, content];
192
+ }
193
+ else {
194
+ await readOnly.place(path, content);
195
+ }
196
+ if (provenance !== undefined)
197
+ await readOnly.place(sidecar(path), provenance);
198
+ }
199
+ const blobs = await kept(binaries);
200
+ for (const entry of binaries) {
201
+ const blob = blobs.get(entry.path);
202
+ if (blob === undefined)
203
+ continue;
204
+ revisions[entry.path] = entry.revision;
205
+ binary[entry.path] = { bytes: entry.bytes, sha256: entry.sha256 };
206
+ yield [entry.path, { blob }];
207
+ }
208
+ }
209
+ return { house: await importCommits(repository, [{ ...commit, files: tracked() }]), revisions, binary };
121
210
  }
122
211
  async function ruled(path, rules) {
123
212
  if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
124
213
  await writeFile(path, rules);
125
214
  }
126
215
  async function ignoreRules(repository, writable, copy) {
127
- const readOnly = ('read_only' in copy ? copy.read_only : [...copy.readOnly.keys()]).filter((path) => !path.split('/').includes(PROVENANCE));
216
+ const readOnly = copy.read_only.filter((path) => !path.split('/').includes(PROVENANCE));
128
217
  const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
129
218
  const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
130
219
  const house = [
@@ -155,15 +244,71 @@ async function ignoreRules(repository, writable, copy) {
155
244
  '',
156
245
  ].join('\n'));
157
246
  }
158
- function revisionAt(copy, commit, path) {
247
+ async function schemaText(copy, files, path) {
248
+ const entry = files.get(path);
249
+ if (entry !== undefined)
250
+ return blobText(copy.repository, entry);
251
+ return copy.read_only.includes(path) ? readFile(join(copy.repository, path), 'utf8') : undefined;
252
+ }
253
+ async function modelOf(copy, files) {
254
+ const document = await schemaText(copy, files, SCHEMA);
255
+ if (document !== undefined)
256
+ return readSchemaDocument(document, HOUSE_ROOM_TYPES);
257
+ return readTypes((await schemaText(copy, files, SCHEMA_YAML)) ?? '', HOUSE_ROOM_TYPES);
258
+ }
259
+ async function attachedText(repository, houseTree, change) {
260
+ const blob = houseTree.get(change.path);
261
+ return blob !== undefined && (await holds(repository, blob, change)) ? blob : null;
262
+ }
263
+ function entryAt(copy, commit, path) {
159
264
  let manifest = copy.manifests[commit];
160
265
  while (manifest !== undefined) {
161
- if (path in manifest.files)
162
- return manifest.files[path] ?? undefined;
266
+ if (path in manifest.files) {
267
+ const revision = manifest.files[path];
268
+ if (revision === null || revision === undefined)
269
+ return undefined;
270
+ const held = manifest.binary?.[path];
271
+ return held === undefined ? { revision } : { revision, held };
272
+ }
163
273
  manifest = manifest.parent === null ? undefined : copy.manifests[manifest.parent];
164
274
  }
165
275
  return undefined;
166
276
  }
277
+ function revisionAt(copy, commit, path) {
278
+ return entryAt(copy, commit, path)?.revision;
279
+ }
280
+ function heldAt(copy, commit, path) {
281
+ return entryAt(copy, commit, path)?.held;
282
+ }
283
+ function manifest(parent, files, binary) {
284
+ return Object.keys(binary).length === 0 ? { parent, files } : { parent, files, binary };
285
+ }
286
+ function binaryParts(changes) {
287
+ return changes.filter((change) => change.op === 'upload');
288
+ }
289
+ function pushedBinary(copy, push) {
290
+ const sent = new Map();
291
+ for (const change of push.changes) {
292
+ if (change.op === 'upload')
293
+ sent.set(relative(change.path), { bytes: change.bytes, sha256: change.sha256 });
294
+ else if (change.op === 'rename') {
295
+ const held = heldAt(copy, push.base, relative(change.path));
296
+ if (held !== undefined)
297
+ sent.set(relative(change.to), held);
298
+ }
299
+ }
300
+ return sent;
301
+ }
302
+ async function writtenAt(copy, house, path) {
303
+ const named = `${rootOf(copy.room_handle)}/${path}`;
304
+ const entry = house.get(path);
305
+ if (entry === undefined)
306
+ return { path: named, revision: null, content: null };
307
+ const revision = revisionAt(copy, copy.house, path) ?? null;
308
+ return heldAt(copy, copy.house, path) === undefined
309
+ ? { path: named, revision, content: await blobText(copy.repository, entry) }
310
+ : { path: named, revision };
311
+ }
167
312
  function without(copy) {
168
313
  const { push, ...settled } = copy;
169
314
  return settled;
@@ -179,11 +324,36 @@ function movedPaths(moves, before, after) {
179
324
  .filter((pair) => pairs.filter((other) => other.from === pair.from || other.to === pair.to).length === 1)
180
325
  .map(({ from, to }) => [from, to]));
181
326
  }
327
+ function sameBytesMoved(before, after, held, binary, intended) {
328
+ const arriving = new Set(intended.values());
329
+ const leaving = new Map();
330
+ const added = new Map();
331
+ const listed = (paths, file, path) => {
332
+ paths.set(heldKey(file), [...(paths.get(heldKey(file)) ?? []), path]);
333
+ };
334
+ for (const [path, entry] of before) {
335
+ const file = held(path);
336
+ if (file !== undefined && !intended.has(path) && !binary.has(path) && after.get(path) !== entry)
337
+ listed(leaving, file, path);
338
+ }
339
+ for (const [path, file] of binary) {
340
+ if (!arriving.has(path) && held(path) === undefined)
341
+ listed(added, file, path);
342
+ }
343
+ const moved = new Map();
344
+ for (const [key, [from, ...others]] of leaving) {
345
+ const [to, ...more] = added.get(key) ?? [];
346
+ if (others.length === 0 && more.length === 0 && to !== undefined && !after.has(from) && !before.has(to))
347
+ moved.set(from, to);
348
+ }
349
+ return moved;
350
+ }
182
351
  export class LocalCopies {
183
352
  owner;
184
353
  house;
185
354
  copies = new Map();
186
355
  queues = new Map();
356
+ waiting = new Set();
187
357
  answering = Promise.resolve();
188
358
  constructor(house) {
189
359
  this.house = house;
@@ -191,17 +361,36 @@ export class LocalCopies {
191
361
  key: 'owner',
192
362
  submit: async (operation, changes) => {
193
363
  try {
194
- const outcome = await retrying(() => house.post('/kit/door/batch', { operation_id: operation, changes }));
195
- return { accepted: outcome.contents };
364
+ const outcome = await retrying(() => stopping(house.door('update', { operation_id: operation, changes })));
365
+ return 'refused' in outcome ? outcome : { accepted: outcome.contents };
196
366
  }
197
367
  catch (error) {
198
368
  if (!finalRefusal(error))
199
369
  throw error;
200
370
  return {
201
- refused: refusalOf(error.text) ?? { code: '', text: `House refused it with ${error.status}`, conflicts: [] },
371
+ refused: refusalOf(error.text) ?? { code: '', text: `House refused it with ${error.status}`, conflicts: [], detail: {} },
202
372
  };
203
373
  }
204
374
  },
375
+ finish: async (upload, _room, partSha256) => {
376
+ try {
377
+ return (await house.act('uploads/finish', { upload, part_sha256: partSha256 }));
378
+ }
379
+ catch (error) {
380
+ if (!(error instanceof HouseRefusal))
381
+ throw new Unsettled(causeOf(error));
382
+ throw retryable(error.text) === false ? new Error(houseCause(error)) : new Unsettled(houseCause(error));
383
+ }
384
+ },
385
+ info: async (path, revision) => {
386
+ const read = toolAnswer({
387
+ status: 200,
388
+ text: await ownAgentCall(rpc('tools/call', { name: 'info', arguments: { path, ...(revision === undefined ? {} : { ref: revision }) } })),
389
+ });
390
+ if ('refused' in read)
391
+ throw new Error(read.refused.text);
392
+ return fileInfo(read.text);
393
+ },
205
394
  };
206
395
  }
207
396
  async load() {
@@ -212,28 +401,29 @@ export class LocalCopies {
212
401
  if (text === null)
213
402
  continue;
214
403
  const copy = JSON.parse(text);
215
- this.copies.set(copy.key, copy);
404
+ if (renameOldReferences(copy))
405
+ await this.save(copy);
406
+ else
407
+ this.copies.set(copy.key, copy);
216
408
  }
217
409
  }
218
- async mapping() {
219
- await this.answering;
220
- return [...this.copies.values()]
221
- .filter((copy) => copy.unfollowed === undefined)
222
- .map((copy) => `${rootOf(copy.room_handle)}: ${copy.repository}`)
223
- .sort();
410
+ async swept() {
411
+ const rooms = join(localCopies(), 'rooms');
412
+ for (const name of await readdir(rooms).catch(() => [])) {
413
+ if (name.startsWith(stagingPrefix()))
414
+ await rm(join(rooms, name), { recursive: true, force: true });
415
+ }
416
+ for (const copy of this.copies.values())
417
+ await sweptTransfers(copy.repository);
224
418
  }
225
419
  connected() {
226
420
  return this.selected();
227
421
  }
228
- async changed(rooms) {
229
- if (rooms.length > 0)
230
- await this.selected(rooms);
422
+ fetch(caller) {
423
+ return this.answered(() => this.fetched(true, caller));
231
424
  }
232
- fetch() {
233
- return this.answered(() => this.fetched(true));
234
- }
235
- async selected(rooms) {
236
- const copied = await this.answered(() => this.fetched(false, rooms));
425
+ async selected() {
426
+ const copied = await this.answered(() => this.fetched(false, this.owner));
237
427
  if (copied.refused)
238
428
  logged(copied.text);
239
429
  }
@@ -242,16 +432,14 @@ export class LocalCopies {
242
432
  this.answering = answered.catch(() => undefined);
243
433
  return answered;
244
434
  }
245
- async fetched(refreshing, rooms) {
246
- const { local_copy: answer } = await this.house.post('/kit/local-copy/selection', rooms === undefined ? {} : { rooms });
247
- const synced = new Map(answer.rooms.map((room) => [`rooms/${room.room_ref}`, room.room_handle]));
248
- const named = rooms === undefined ? null : new Set(rooms.map((room) => `rooms/${room}`));
249
- const held = () => [...this.copies.values()].filter((copy) => named === null || named.has(copy.key));
250
- return this.holding([...new Set([...held().map((copy) => copy.key), ...synced.keys()])].sort(), async () => {
435
+ async fetched(refreshing, caller) {
436
+ const { local_copy: answer } = await this.house.act('local-copies/selection/read', {});
437
+ const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
438
+ return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
251
439
  const said = [];
252
440
  let refused = false;
253
441
  const mark = refreshing ? 'said' : 'unsaid';
254
- for (const copy of held()) {
442
+ for (const copy of [...this.copies.values()]) {
255
443
  if (synced.has(copy.key) || copy.unfollowed === 'said' || copy.unfollowed === mark)
256
444
  continue;
257
445
  await this.save({ ...copy, unfollowed: mark });
@@ -259,11 +447,12 @@ export class LocalCopies {
259
447
  said.push(`${rootOf(copy.room_handle)}: no longer followed; kept as it is`);
260
448
  }
261
449
  const current = [];
450
+ const wanted = new Map();
262
451
  for (const [key, handle] of synced) {
263
452
  try {
264
453
  const held = this.copies.get(key);
265
454
  if (held === undefined || !(await exists(held.repository))) {
266
- said.push(await this.materialize(key, handle));
455
+ wanted.set(key, handle);
267
456
  continue;
268
457
  }
269
458
  const { unfollowed, ...followed } = held;
@@ -274,18 +463,62 @@ export class LocalCopies {
274
463
  said.push(`${rootOf(handle)}: ${causeOf(error)}`);
275
464
  }
276
465
  }
277
- if (refreshing && current.length > 0) {
278
- const failed = await this.refreshed(current).then(() => null, houseCause);
279
- for (const copy of current)
280
- said.push(...moved(copy, this.copies.get(copy.key)));
466
+ const unready = [];
467
+ const unsettled = [];
468
+ if (wanted.size > 0) {
469
+ const { authorities } = await this.discovered([...wanted.values()]);
470
+ for (const [key, handle] of wanted) {
471
+ const authority = authorities.find((entry) => `${ROOMS}${entry.room_ref}` === key);
472
+ try {
473
+ if (authority === undefined)
474
+ throw new Error('House serves nothing of it to copy');
475
+ said.push(await this.copied({ key, handle, authority }));
476
+ }
477
+ catch (error) {
478
+ if (error instanceof NotReady) {
479
+ unready.push(...error.rooms);
480
+ said.push(`${rootOf(handle)}: ${NOT_READY}`);
481
+ }
482
+ else {
483
+ refused = true;
484
+ unsettled.push(key.slice(ROOMS.length));
485
+ said.push(`${rootOf(handle)}: ${causeOf(error)}`);
486
+ }
487
+ }
488
+ }
489
+ }
490
+ const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
491
+ if (reading.length > 0) {
492
+ const failed = await this.refreshed(reading, unready, caller).then(() => null, houseCause);
493
+ for (const copy of reading) {
494
+ said.push(...(unready.includes(copy.room_ref) ? [`${rootOf(copy.room_handle)}: ${NOT_READY}`] : moved(copy, this.copies.get(copy.key))));
495
+ }
281
496
  if (failed !== null) {
282
497
  refused = true;
498
+ unsettled.push(...reading.map((copy) => copy.room_ref));
283
499
  said.push(failed);
284
500
  }
285
501
  }
502
+ for (const room of [...this.waiting]) {
503
+ if (!unready.includes(room) && !unsettled.includes(room))
504
+ this.waiting.delete(room);
505
+ }
506
+ const due = unready.filter((room) => !this.waiting.has(room));
507
+ for (const room of due)
508
+ this.waiting.add(room);
509
+ if (due.length > 0)
510
+ void this.askedAgain(due);
286
511
  return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
287
512
  });
288
513
  }
514
+ async askedAgain(rooms) {
515
+ let asking = rooms;
516
+ for (let step = 0; asking.length > 0; step++) {
517
+ await paused(ASKED_AGAIN_MS[Math.min(step, ASKED_AGAIN_MS.length - 1)]);
518
+ await this.selected().catch(logged);
519
+ asking = asking.filter((room) => this.waiting.has(room));
520
+ }
521
+ }
289
522
  async observed(report) {
290
523
  const key = await this.keyAt(dirname(report.common));
291
524
  if (key === undefined)
@@ -347,132 +580,188 @@ export class LocalCopies {
347
580
  this.copies.set(copy.key, copy);
348
581
  return copy;
349
582
  }
350
- async read(handle) {
351
- const prefixes = [rootOf(handle)];
352
- const discovery = await this.house.post('/kit/door/discover', { prefixes });
353
- let bundle = await this.house.post('/kit/door/bootstrap', { prefixes });
354
- if ('transfer' in bundle)
355
- bundle = (await (await fetch(relayed(bundle.transfer.url))).json());
356
- const [view] = bundle.scopes;
357
- if (view === undefined)
358
- return null;
359
- const authority = discovery.authorities.find((entry) => entry.room_ref === view.manifest.anchor);
360
- const contents = new Map(view.contents.map((entry) => [entry.path, entry.content]));
361
- const read = {
362
- anchor: view.manifest.anchor,
363
- position: view.manifest.position,
364
- log_epoch: view.manifest.log_epoch,
365
- write: authority.operations.write,
366
- protected: authority.protected_paths,
367
- writable: new Map(),
368
- revisions: {},
369
- readOnly: new Map(),
370
- };
371
- for (const file of view.manifest.files) {
372
- const content = contents.get(file.path);
373
- if (file.writable) {
374
- read.writable.set(file.path, content);
375
- read.revisions[file.path] = file.revision;
376
- }
377
- else {
378
- read.readOnly.set(file.path, content);
583
+ async discovered(handles) {
584
+ const discover = (group) => this.house.door('rooms/discover', { prefixes: group.map(rootOf) });
585
+ const [first, ...rest] = handles;
586
+ const { bounds, authorities } = await discover([first]);
587
+ for (const group of grouped(rest, bounds.max_prefixes))
588
+ authorities.push(...(await discover(group)).authorities);
589
+ return { bounds, authorities };
590
+ }
591
+ async chunked(root, room, directory) {
592
+ await mkdir(directory, { recursive: true });
593
+ for (let renewed = false;; renewed = true) {
594
+ const { rooms } = await this.house.door('room-chunks/list', { prefixes: [root] });
595
+ const list = rooms.find((entry) => entry.room_ref === room);
596
+ if (list === undefined)
597
+ throw new Error('House serves nothing of it to copy');
598
+ if (!list.ready)
599
+ throw new NotReady([room]);
600
+ const held = new Set(await readdir(directory));
601
+ const failure = await stagedChunks(directory, list.chunks.filter(({ hash }) => !held.has(hash)));
602
+ if (failure === undefined) {
603
+ return { position: list.position, log_epoch: list.log_epoch, entries: chunkEntries(directory, list.chunks, list.models) };
379
604
  }
380
- if (file.provenance !== undefined)
381
- read.readOnly.set(sidecar(file.path), file.provenance);
605
+ if (renewed && (await readdir(directory)).length === held.size)
606
+ throw failure;
382
607
  }
383
- return read;
384
608
  }
385
- async materialize(key, handle) {
386
- const read = await this.read(handle);
387
- if (read === null)
388
- throw new Error('House serves nothing of it to copy');
389
- const repository = join(localCopies(), 'rooms', read.anchor);
609
+ async materialized({ key, handle, authority }, whole) {
610
+ const repository = join(localCopies(), 'rooms', authority.room_ref);
390
611
  if (await exists(repository))
391
612
  throw new Error(`${repository} already exists and Kit leaves it as it is`);
392
- const staging = join(dirname(repository), `.${randomUUID()}`);
613
+ const staging = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
614
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
393
615
  await mkdir(staging, { recursive: true });
394
- let house;
395
- let readOnly;
616
+ let copy;
396
617
  try {
397
618
  await git(staging, ['init', '-q', '-b', BRANCH]);
398
- house = await importCommits(staging, [
399
- { message: `House ${rootOf(handle)}`, from: null, files: read.writable },
400
- ]);
619
+ await git(staging, ['config', 'core.precomposeUnicode', 'true']);
620
+ const read = await whole(join(staging, '.git', STAGED_CHUNKS));
621
+ const readOnly = await readOnlyFiles(staging, []);
622
+ const commit = { message: `House ${rootOf(handle)}`, from: null };
623
+ const { house, revisions } = await importedWhole(staging, scope, read.entries, commit, readOnly, async () => new Map());
401
624
  await git(staging, ['update-ref', `refs/heads/${BRANCH}`, house]);
402
625
  await git(staging, ['reset', '-q', '--hard']);
403
- readOnly = await readOnlyPlaced(staging, [], read.readOnly, []);
404
- await ignoreRules(staging, [...read.writable.keys()], read);
626
+ copy = {
627
+ key,
628
+ room_ref: authority.room_ref,
629
+ room_handle: handle,
630
+ repository,
631
+ house,
632
+ position: read.position,
633
+ log_epoch: read.log_epoch,
634
+ ...scope,
635
+ read_only: readOnly.held(),
636
+ manifests: { [house]: { parent: null, files: revisions } },
637
+ moves: [],
638
+ };
639
+ await ignoreRules(staging, Object.keys(revisions), copy);
640
+ await rm(join(staging, '.git', STAGED_CHUNKS), { recursive: true, force: true });
405
641
  await rename(staging, repository);
406
642
  }
407
643
  catch (error) {
408
644
  await rm(staging, { recursive: true, force: true });
409
645
  throw error;
410
646
  }
411
- await this.save({
412
- key,
413
- room_ref: read.anchor,
414
- room_handle: handle,
415
- repository,
416
- house,
417
- position: read.position,
418
- log_epoch: read.log_epoch,
419
- write: read.write,
420
- protected: read.protected,
421
- read_only: readOnly,
422
- manifests: { [house]: { parent: null, files: read.revisions } },
423
- moves: [],
424
- });
425
- return `${rootOf(handle)}: copied to ${repository}`;
647
+ return this.save(copy);
426
648
  }
427
- async received(copy) {
428
- const read = await this.read(copy.room_handle);
429
- if (read === null)
430
- return copy;
431
- const house = await importCommits(copy.repository, [
432
- { message: `House ${rootOf(copy.room_handle)} at position ${read.position}`, from: copy.house, replaces: true, files: read.writable },
433
- ]);
434
- const removed = copy.read_only.filter((path) => !read.readOnly.has(path));
435
- const readOnly = await readOnlyPlaced(copy.repository, copy.read_only, read.readOnly, removed);
436
- const next = {
437
- ...copy,
438
- house,
439
- position: read.position,
440
- log_epoch: read.log_epoch,
441
- write: read.write,
442
- protected: read.protected,
443
- read_only: readOnly,
444
- manifests: { ...copy.manifests, [house]: { parent: null, files: read.revisions } },
445
- };
446
- await ignoreRules(copy.repository, [...read.writable.keys()], next);
447
- return next;
649
+ async copied(wanted) {
650
+ const { handle, authority } = wanted;
651
+ const root = rootOf(handle);
652
+ let answer = null;
653
+ try {
654
+ if (!this.waiting.has(authority.room_ref)) {
655
+ answer = await this.house.door('changes/enumerate', {
656
+ prefixes: [root],
657
+ positions: [
658
+ { authority: authority.room_ref, position: '0', log_epoch: authority.log_epoch, page: { through: authority.position, read: 0 } },
659
+ ],
660
+ });
661
+ }
662
+ }
663
+ catch (error) {
664
+ if (expiredAuthority(error) !== authority.room_ref)
665
+ throw error;
666
+ }
667
+ const view = answer?.authorities.find((entry) => entry.authority === authority.room_ref);
668
+ if (answer !== null && view === undefined)
669
+ throw new Error('House serves nothing of it to copy');
670
+ if (view !== undefined && view.next === undefined) {
671
+ const entries = view.changes.filter((change) => 'content' in change);
672
+ const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, entries }));
673
+ return `${root}: copied to ${copy.repository}`;
674
+ }
675
+ const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
676
+ const [caught] = await this.caughtUp([copy], true, this.owner);
677
+ await succeeded(copy.repository, ['merge', '--ff-only', '-q', caught.house]);
678
+ return `${root}: copied to ${copy.repository}`;
448
679
  }
449
- async refreshed(copies) {
450
- const discover = (group) => this.house.post('/kit/door/discover', { prefixes: group.map((copy) => rootOf(copy.room_handle)) });
451
- const [first, ...rest] = copies;
452
- const { bounds, authorities } = await discover([first]);
453
- for (const group of grouped(rest, bounds.max_prefixes))
454
- authorities.push(...(await discover(group)).authorities);
680
+ async received(copy, authority, caller) {
681
+ const root = rootOf(copy.room_handle);
682
+ const directory = join(copy.repository, '.git', STAGED_CHUNKS);
683
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
684
+ try {
685
+ const read = await this.chunked(root, copy.room_ref, directory);
686
+ const readOnly = await readOnlyFiles(copy.repository, copy.read_only);
687
+ for (const path of copy.read_only)
688
+ await readOnly.remove(path);
689
+ const commit = { message: `House ${root} at position ${read.position}`, from: copy.house, replaces: true };
690
+ const houseTree = await tree(copy.repository, copy.house);
691
+ const pushing = await this.pushing(copy);
692
+ const kept = async (entries) => {
693
+ const present = new Set(entries.map((entry) => entry.path));
694
+ const leaving = new Map([...houseTree].flatMap(([path, blob]) => {
695
+ const held = present.has(path) ? undefined : heldAt(copy, copy.house, path);
696
+ return held === undefined ? [] : [[heldKey(held), blob]];
697
+ }));
698
+ const blobs = new Map();
699
+ for (const entry of entries) {
700
+ const blob = await this.binaryIn(copy, houseTree, entry, pushing, leaving, caller);
701
+ if (blob !== null)
702
+ blobs.set(entry.path, blob);
703
+ }
704
+ return blobs;
705
+ };
706
+ const { house, revisions, binary } = await importedWhole(copy.repository, scope, read.entries, commit, readOnly, kept);
707
+ const next = {
708
+ ...copy,
709
+ ...scope,
710
+ house,
711
+ position: read.position,
712
+ log_epoch: read.log_epoch,
713
+ read_only: readOnly.held(),
714
+ manifests: { ...copy.manifests, [house]: manifest(null, revisions, binary) },
715
+ };
716
+ await ignoreRules(copy.repository, Object.keys(revisions), next);
717
+ return next;
718
+ }
719
+ finally {
720
+ await rm(directory, { recursive: true, force: true });
721
+ }
722
+ }
723
+ async reread(copy, caller) {
724
+ const { authorities } = await this.discovered([copy.room_handle]);
725
+ const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
726
+ if (authority === undefined)
727
+ throw new NotReady([copy.room_ref]);
728
+ return this.received(copy, authority, caller);
729
+ }
730
+ async refreshed(copies, unready, caller) {
731
+ const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
732
+ const read = [];
455
733
  const unchanged = [];
456
734
  for (const copy of copies) {
457
735
  const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
458
736
  if (authority !== undefined &&
459
737
  JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
460
- await this.save(await this.received(copy));
738
+ try {
739
+ read.push(await this.save(await this.received(copy, authority, caller)));
740
+ }
741
+ catch (error) {
742
+ if (!(error instanceof NotReady))
743
+ throw error;
744
+ unready.push(...error.rooms);
745
+ }
461
746
  }
462
747
  else {
463
748
  unchanged.push(copy);
464
749
  }
465
750
  }
466
751
  for (const group of grouped(unchanged, bounds.max_prefixes))
467
- await this.caughtUp(group, false);
752
+ await this.caughtUp(group, false, caller, unready);
753
+ for (const group of grouped(read, bounds.max_prefixes))
754
+ await this.caughtUp(group, true, caller, unready);
468
755
  }
469
- async caughtUp(copies, whole) {
756
+ async caughtUp(copies, whole, caller, unready) {
470
757
  const caught = new Map(copies.map((copy) => [copy.key, copy]));
758
+ const read = new Set();
759
+ const waiting = unready ?? [];
471
760
  let asking = [...copies];
472
761
  while (asking.length > 0) {
473
762
  let answer;
474
763
  try {
475
- answer = await this.house.post('/kit/door/enumerate', {
764
+ answer = await this.house.door('changes/enumerate', {
476
765
  prefixes: asking.map((copy) => rootOf(copy.room_handle)),
477
766
  positions: asking.map((copy) => ({ authority: copy.room_ref, position: copy.position, log_epoch: copy.log_epoch })),
478
767
  });
@@ -481,8 +770,22 @@ export class LocalCopies {
481
770
  const expired = asking.find((copy) => copy.room_ref === expiredAuthority(error));
482
771
  if (expired === undefined)
483
772
  throw error;
484
- caught.set(expired.key, await this.save(await this.received(expired)));
485
773
  asking = asking.filter((copy) => copy !== expired);
774
+ if (read.has(expired.key)) {
775
+ waiting.push(expired.room_ref);
776
+ continue;
777
+ }
778
+ read.add(expired.key);
779
+ try {
780
+ const next = await this.save(await this.reread(expired, caller));
781
+ caught.set(expired.key, next);
782
+ asking.push(next);
783
+ }
784
+ catch (refused) {
785
+ if (!(refused instanceof NotReady))
786
+ throw refused;
787
+ waiting.push(expired.room_ref);
788
+ }
486
789
  continue;
487
790
  }
488
791
  const advanced = [];
@@ -490,22 +793,83 @@ export class LocalCopies {
490
793
  const view = answer.authorities.find((entry) => entry.authority === copy.room_ref);
491
794
  if (view === undefined || (view.position === copy.position && view.log_epoch === copy.log_epoch))
492
795
  continue;
493
- const next = await this.save(await this.incorporated(copy, view.changes, view.position, view.log_epoch));
796
+ const next = await this.save(await this.incorporated(copy, view.changes, view.position, view.log_epoch, caller));
494
797
  caught.set(copy.key, next);
495
798
  advanced.push(next);
496
799
  }
497
- asking = whole ? advanced : [];
800
+ asking = advanced.filter((copy) => whole || read.has(copy.key));
498
801
  }
802
+ if (unready === undefined && waiting.length > 0)
803
+ throw new NotReady(waiting);
499
804
  return copies.map((copy) => caught.get(copy.key));
500
805
  }
501
- async incorporated(copy, changes, position, logEpoch) {
806
+ async pushing(copy) {
807
+ const sent = copy.push === undefined ? new Map() : pushedBinary(copy, copy.push);
808
+ if (sent.size === 0)
809
+ return new Map();
810
+ const commit = await tree(copy.repository, copy.push.commit);
811
+ return new Map([...sent].map(([path, held]) => [path, { ...held, blob: commit.get(path) }]));
812
+ }
813
+ async binaryIn(copy, houseTree, change, pushing, leaving, caller) {
814
+ const pushed = pushing.get(change.path);
815
+ if (pushed !== undefined && heldKey(pushed) === heldKey(change))
816
+ return pushed.blob;
817
+ const moved = leaving.get(heldKey(change));
818
+ const kept = heldAt(copy, copy.house, change.path) ?? pushed;
819
+ if (kept === undefined)
820
+ return moved ?? (await attachedText(copy.repository, houseTree, change));
821
+ if (heldKey(kept) === heldKey(change))
822
+ return houseTree.get(change.path);
823
+ return moved ?? this.revisionBlob(copy, change.path, change.revision, caller);
824
+ }
825
+ revisionBlob(copy, path, revision, caller, first = null) {
826
+ const named = `${rootOf(copy.room_handle)}/${path}`;
827
+ let answered = first;
828
+ return receivedBinary(copy.repository, named, async () => {
829
+ const file = answered ?? (await caller.info(named, revision));
830
+ answered = null;
831
+ if (file === null)
832
+ throw new Error(`House holds no attachment at ${named}`);
833
+ return file;
834
+ });
835
+ }
836
+ async incorporated(copy, changes, position, logEpoch, caller) {
502
837
  const houseTree = await tree(copy.repository, copy.house);
503
838
  const held = new Set(copy.read_only);
504
839
  const files = new Map();
505
840
  const revisions = {};
841
+ const binary = {};
506
842
  const written = new Map();
507
843
  const removed = [];
844
+ const pushing = await this.pushing(copy);
845
+ const leaving = new Map(changes.flatMap((change) => {
846
+ const kept = 'removed' in change ? heldAt(copy, copy.house, change.path) : undefined;
847
+ return kept === undefined ? [] : [[heldKey(kept), houseTree.get(change.path)]];
848
+ }));
508
849
  for (const change of changes) {
850
+ if (!('removed' in change) && !('content' in change)) {
851
+ const blob = await this.binaryIn(copy, houseTree, change, pushing, leaving, caller);
852
+ if (blob === null) {
853
+ if (houseTree.has(change.path)) {
854
+ files.set(change.path, null);
855
+ revisions[change.path] = null;
856
+ }
857
+ else if (held.has(change.path))
858
+ removed.push(change.path, sidecar(change.path));
859
+ continue;
860
+ }
861
+ const kept = heldAt(copy, copy.house, change.path);
862
+ const unchanged = houseTree.get(change.path) === blob &&
863
+ kept !== undefined &&
864
+ heldKey(kept) === heldKey(change) &&
865
+ revisionAt(copy, copy.house, change.path) === change.revision;
866
+ if (unchanged)
867
+ continue;
868
+ files.set(change.path, { blob });
869
+ revisions[change.path] = change.revision;
870
+ binary[change.path] = { bytes: change.bytes, sha256: change.sha256 };
871
+ continue;
872
+ }
509
873
  const gone = 'removed' in change;
510
874
  if (houseTree.has(change.path) || (!gone && !held.has(change.path) && declaredWritable(change.path, copy))) {
511
875
  const unchanged = gone
@@ -531,7 +895,7 @@ export class LocalCopies {
531
895
  house = await importCommits(copy.repository, [
532
896
  { message: `House ${rootOf(copy.room_handle)} at position ${position}`, from: copy.house, files },
533
897
  ]);
534
- manifests = { ...manifests, [house]: { parent: copy.house, files: revisions } };
898
+ manifests = { ...manifests, [house]: manifest(copy.house, revisions, binary) };
535
899
  }
536
900
  const next = {
537
901
  ...copy,
@@ -546,16 +910,25 @@ export class LocalCopies {
546
910
  }
547
911
  async pushed(key, worktree, selector, caller) {
548
912
  let copy = this.copies.get(key);
549
- if (copy.push?.state === 'submitted')
550
- return this.settle(copy, caller);
551
913
  const selected = await run(worktree, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${selector}^{commit}`]);
552
914
  if (selected.status !== 0)
553
915
  return { refused: true, text: `${selector} names no commit` };
554
916
  const commit = selected.stdout.toString('utf8').trim();
917
+ if (copy.push?.state === 'submitted') {
918
+ if (copy.push.commit === commit)
919
+ return this.settle(copy, caller);
920
+ const replaced = await this.recovered(copy, caller, async (current) => {
921
+ await this.save(without(current));
922
+ return { refused: false, text: '' };
923
+ });
924
+ if (replaced.refused)
925
+ return replaced;
926
+ copy = this.copies.get(key);
927
+ }
555
928
  if (copy.push !== undefined) {
556
929
  const held = copy.push.acknowledged;
557
930
  if (held === undefined || !(await succeeded(worktree, ['merge-base', '--is-ancestor', held, commit]))) {
558
- return this.finished(copy);
931
+ return this.finished(copy, caller);
559
932
  }
560
933
  copy = await this.save(without(copy));
561
934
  }
@@ -567,6 +940,12 @@ export class LocalCopies {
567
940
  let changes;
568
941
  try {
569
942
  changes = await this.derived(copy, base, commit);
943
+ if (changesKind(changes)) {
944
+ const read = (await this.caughtUp([copy], true, caller))[0];
945
+ if (read.house !== copy.house)
946
+ changes = await this.derived(read, base, commit);
947
+ copy = read;
948
+ }
570
949
  }
571
950
  catch (error) {
572
951
  return { refused: true, text: houseCause(error) };
@@ -590,24 +969,36 @@ export class LocalCopies {
590
969
  });
591
970
  return this.settle(submitted, caller);
592
971
  }
972
+ async attachmentsOf(copy, arriving) {
973
+ const documents = arriving.filter(([path]) => isDocument(path));
974
+ if (documents.length > 0) {
975
+ const { bounds } = await this.house.door('rooms/discover', { prefixes: [rootOf(copy.room_handle)] });
976
+ const refused = await untext(copy.repository, documents, bounds.max_authored_file_bytes);
977
+ if (refused.length > 0)
978
+ throw new Error(markdownOnly(refused, bounds.max_authored_file_bytes));
979
+ }
980
+ return attachments(copy.repository, arriving.filter(([path]) => !isDocument(path)));
981
+ }
593
982
  async derived(copy, base, commit) {
594
983
  const root = rootOf(copy.room_handle);
595
984
  const before = await tree(copy.repository, base);
596
985
  const after = await tree(copy.repository, commit);
597
- const moved = movedPaths(copy.moves, before, after);
986
+ const intended = movedPaths(copy.moves, before, after);
987
+ const renamedAsIs = new Set([...intended].filter(([from, to]) => before.get(from) === after.get(to)).map(([, to]) => to));
988
+ const binary = await this.attachmentsOf(copy, [...after].filter(([path, entry]) => before.get(path) !== entry && !renamedAsIs.has(path)));
989
+ const held = (path) => heldAt(copy, base, path);
990
+ const houseTree = base === copy.house ? before : await tree(copy.repository, copy.house);
991
+ const houseText = (path) => houseTree.has(path) && heldAt(copy, copy.house, path) === undefined;
992
+ const moved = new Map([...intended, ...sameBytesMoved(before, after, held, binary, intended)]);
598
993
  const arrived = new Set(moved.values());
599
- const arriving = [...after].filter(([path, entry]) => before.get(path) !== entry);
600
- if (arriving.length > 0) {
601
- const { bounds } = await this.house.post('/kit/door/discover', { prefixes: [root] });
602
- const binary = [];
603
- for (const [path, entry] of arriving) {
604
- if (!(await isText(copy.repository, entry, bounds.max_authored_file_bytes)))
605
- binary.push(path);
606
- }
607
- if (binary.length > 0)
608
- throw new Error(textOnly(binary));
609
- }
610
994
  const text = (entry) => blobText(copy.repository, entry);
995
+ const created = async (path, entry) => {
996
+ const part = binary.get(path);
997
+ return part === undefined
998
+ ? { op: 'create', path: `${root}/${path}`, content: await text(entry) }
999
+ : { op: 'upload', path: `${root}/${path}`, ...part };
1000
+ };
1001
+ const model = await modelOf(copy, after);
611
1002
  const changes = [];
612
1003
  for (const path of [...new Set([...before.keys(), ...after.keys()])].sort()) {
613
1004
  if (arrived.has(path))
@@ -619,58 +1010,89 @@ export class LocalCopies {
619
1010
  if (to !== undefined) {
620
1011
  changes.push({ op: 'rename', path: `${root}/${path}`, to: `${root}/${to}`, base: known });
621
1012
  const moving = after.get(to);
1013
+ const part = binary.get(to);
622
1014
  if (moving !== prior) {
623
- changes.push({ op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) });
1015
+ changes.push(part === undefined
1016
+ ? { op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) }
1017
+ : { op: 'upload', path: `${root}/${to}`, base: known, ...part });
624
1018
  }
625
1019
  continue;
626
1020
  }
627
1021
  if (prior === next)
628
1022
  continue;
1023
+ const part = binary.get(path);
629
1024
  if (next === undefined)
630
1025
  changes.push({ op: 'remove', path: `${root}/${path}`, base: known });
631
1026
  else if (prior === undefined)
632
- changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next) });
1027
+ changes.push(await created(path, next));
1028
+ else if (part !== undefined && houseText(path)) {
1029
+ changes.push({ op: 'remove', path: `${root}/${path}`, base: known }, { op: 'upload', path: `${root}/${path}`, ...part });
1030
+ }
1031
+ else if (part !== undefined)
1032
+ changes.push({ op: 'upload', path: `${root}/${path}`, base: known, ...part });
633
1033
  else
634
- changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next)));
1034
+ changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next), model));
635
1035
  }
636
1036
  return changes;
637
1037
  }
638
1038
  async settle(start, caller, restaged = false) {
639
1039
  let copy = start;
640
- const push = copy.push;
641
- if (push.caller !== caller.key)
642
- return this.recovered(copy);
643
- let outcome;
644
- try {
645
- outcome = await caller.submit(push.operation_id, push.changes, push.upload, async (upload) => {
646
- copy = await this.save({ ...copy, push: { ...push, upload } });
647
- });
1040
+ if (copy.push.caller !== caller.key) {
1041
+ return this.recovered(copy, caller, async (current) => this.settle(await this.save({ ...current, push: { ...current.push, caller: caller.key } }), caller));
648
1042
  }
649
- catch (error) {
650
- const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
651
- return {
652
- refused: true,
653
- text: refusal?.text ?? `House's answer to ${short(push.commit)} did not arrive; run house push again`,
654
- };
1043
+ const finished = new Set();
1044
+ let outcome;
1045
+ for (;;) {
1046
+ const push = copy.push;
1047
+ try {
1048
+ outcome = await caller.submit(push.operation_id, push.changes, push.upload, async (upload) => {
1049
+ copy = await this.save({ ...copy, push: { ...copy.push, upload } });
1050
+ });
1051
+ }
1052
+ catch (error) {
1053
+ const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
1054
+ return {
1055
+ refused: true,
1056
+ text: refusal?.text ?? `House's answer to ${short(push.commit)} did not arrive; run house push again`,
1057
+ };
1058
+ }
1059
+ if (!('refused' in outcome) || outcome.refused.code !== 'files_unstaged')
1060
+ break;
1061
+ const named = unstagedUploads(outcome.refused).filter(({ id }) => !finished.has(id));
1062
+ if (named.length === 0)
1063
+ break;
1064
+ for (const { id } of named)
1065
+ finished.add(id);
1066
+ const failed = await this.sentUploads(copy, named, caller);
1067
+ copy = this.copies.get(copy.key);
1068
+ if (failed.size > 0)
1069
+ return this.stopped(copy, failed);
655
1070
  }
656
1071
  if ('refused' in outcome) {
1072
+ if (stopsPush(outcome.refused))
1073
+ return this.stopped(copy, firstLine(outcome.refused));
657
1074
  if (outcome.refused.code === 'upload_unavailable' && copy.push.upload !== undefined && !restaged) {
658
1075
  const { upload, ...unstaged } = copy.push;
659
1076
  return this.settle(await this.save({ ...copy, push: unstaged }), caller, true);
660
1077
  }
661
- if (outcome.refused.code === 'expired')
662
- return this.recovered(copy);
1078
+ if (outcome.refused.code === 'expired') {
1079
+ return this.recovered(copy, caller, async (current) => {
1080
+ await this.save(without(current));
1081
+ return { refused: true, text: `House does not hold ${short(current.push.commit)}; run house push again` };
1082
+ });
1083
+ }
663
1084
  let settled = await this.save(without(copy));
664
1085
  if (settled.unfollowed !== undefined)
665
1086
  return { refused: true, text: outcome.refused.text };
666
1087
  const rebased = rebaseFixes(outcome.refused);
667
1088
  try {
668
- settled = (await this.caughtUp([settled], true))[0];
1089
+ settled = (await this.caughtUp([settled], true, caller))[0];
1090
+ await this.save(await this.conflicted(settled, outcome.refused, copy.push, caller));
669
1091
  }
670
- catch {
1092
+ catch (error) {
671
1093
  return {
672
1094
  refused: true,
673
- text: rebased ? `${outcome.refused.text}\nHouse's current state did not arrive; push again later` : outcome.refused.text,
1095
+ text: rebased ? `${outcome.refused.text}\nHouse's current state did not arrive: ${houseCause(error)}` : outcome.refused.text,
674
1096
  };
675
1097
  }
676
1098
  return {
@@ -680,37 +1102,91 @@ export class LocalCopies {
680
1102
  : outcome.refused.text,
681
1103
  };
682
1104
  }
683
- return this.finished(await this.save({ ...copy, push: { ...copy.push, state: 'accepted', written: outcome.accepted } }));
1105
+ return this.finished(await this.save({ ...copy, push: { ...copy.push, state: 'accepted', written: outcome.accepted } }), caller);
1106
+ }
1107
+ async sentUploads(start, named, caller) {
1108
+ let copy = start;
1109
+ const parts = new Map(binaryParts(copy.push.changes).map((part) => [part.sha256, part]));
1110
+ const commit = await tree(copy.repository, copy.push.commit);
1111
+ const staged = new Set(copy.push.staged ?? []);
1112
+ const failed = new Map();
1113
+ let saving = Promise.resolve();
1114
+ await eachAtOnce(named, async (upload) => {
1115
+ const part = parts.get(upload.sha256);
1116
+ const path = relative(part.path);
1117
+ try {
1118
+ await sentBinary(copy.repository, commit.get(path), path, part, upload, (id, partSha256) => caller.finish(id, copy.room_ref, partSha256));
1119
+ }
1120
+ catch (error) {
1121
+ failed.set(upload.sha256, `its transfer failed: ${causeOf(error)}`);
1122
+ return;
1123
+ }
1124
+ staged.add(upload.sha256);
1125
+ saving = saving.then(async () => {
1126
+ copy = await this.save({ ...copy, push: { ...copy.push, staged: [...staged] } });
1127
+ });
1128
+ await saving;
1129
+ });
1130
+ return failed;
1131
+ }
1132
+ stopped(copy, reasons) {
1133
+ const push = copy.push;
1134
+ const staged = new Set(push.staged ?? []);
1135
+ const unsent = binaryParts(push.changes).filter(({ sha256 }) => !staged.has(sha256));
1136
+ const unchanged = `House holds nothing of ${short(push.commit)} yet, so ${rootOf(copy.room_handle)} is unchanged`;
1137
+ const again = 'house push again resumes this push and sends only what it has not sent';
1138
+ if (typeof reasons === 'string' && unsent.length === 0)
1139
+ return { refused: true, text: `${unchanged}: ${reasons}\n${again}` };
1140
+ const why = (sha256) => (typeof reasons === 'string' ? reasons : (reasons.get(sha256) ?? 'not sent yet'));
1141
+ return {
1142
+ refused: true,
1143
+ text: [`${unchanged}; these attachments are not sent:`, ...unsent.map(({ path, sha256 }) => `${relative(path)}: ${why(sha256)}`), again].join('\n'),
1144
+ };
1145
+ }
1146
+ async conflicted(copy, refusal, push, caller) {
1147
+ const root = rootOf(copy.room_handle);
1148
+ const touched = new Set(changedPaths(push.changes));
1149
+ const houseTree = await tree(copy.repository, copy.house);
1150
+ const paths = [
1151
+ ...new Set(refusal.conflicts.filter(({ code, path }) => code !== 'path_absent' && touched.has(path)).map(({ path }) => relative(path))),
1152
+ ].filter((path) => !houseTree.has(path));
1153
+ const files = new Map();
1154
+ const revisions = {};
1155
+ const binary = {};
1156
+ for (const path of paths) {
1157
+ const file = await caller.info(`${root}/${path}`);
1158
+ if (file === null)
1159
+ continue;
1160
+ files.set(path, { blob: await this.revisionBlob(copy, path, file.revision, caller, file) });
1161
+ revisions[path] = file.revision;
1162
+ binary[path] = { bytes: file.bytes, sha256: file.sha256 };
1163
+ }
1164
+ if (files.size === 0)
1165
+ return copy;
1166
+ const house = await importCommits(copy.repository, [
1167
+ { message: `House ${root}: the attachments ${short(push.commit)} conflicts with`, from: copy.house, files },
1168
+ ]);
1169
+ return { ...copy, house, manifests: { ...copy.manifests, [house]: manifest(copy.house, revisions, binary) } };
684
1170
  }
685
- async recovered(copy) {
1171
+ async recovered(copy, caller, unheld) {
686
1172
  const push = copy.push;
687
- const current = (await this.caughtUp([copy], true))[0];
1173
+ const current = (await this.caughtUp([copy], true, caller))[0];
688
1174
  const house = await tree(copy.repository, current.house);
689
- const root = rootOf(current.room_handle);
690
- if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit)))) {
691
- await this.save(without(current));
692
- return {
693
- refused: true,
694
- text: `House does not hold ${short(push.commit)}; run house push again`,
695
- };
1175
+ const model = await modelOf(current, house);
1176
+ if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit), model))) {
1177
+ return unheld(current);
696
1178
  }
697
1179
  const written = [];
698
- for (const path of changedPaths(push.changes).map(relative)) {
699
- const entry = house.get(path);
700
- written.push({
701
- path: `${root}/${path}`,
702
- revision: entry === undefined ? null : (revisionAt(current, current.house, path) ?? null),
703
- content: entry === undefined ? null : await blobText(copy.repository, entry),
704
- });
705
- }
706
- return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }));
1180
+ for (const path of changedPaths(push.changes).map(relative))
1181
+ written.push(await writtenAt(current, house, path));
1182
+ return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }), caller);
707
1183
  }
708
- async finished(start) {
1184
+ async finished(start, caller) {
709
1185
  let copy = start;
710
1186
  const { commit } = copy.push;
711
1187
  try {
712
1188
  if (copy.push.acknowledged === undefined)
713
- copy = await this.save(await this.acknowledged(copy));
1189
+ copy = await this.save(await this.acknowledged(copy, caller));
714
1190
  const integrated = await this.integrated(copy.push);
715
1191
  if (integrated.done)
716
1192
  await this.save(without(copy));
@@ -723,45 +1199,61 @@ export class LocalCopies {
723
1199
  };
724
1200
  }
725
1201
  }
726
- async acknowledged(start) {
1202
+ async acknowledged(start, caller) {
727
1203
  let copy = start;
728
1204
  const push = copy.push;
729
1205
  let written = push.written;
730
- if (push.changes.some((change) => change.op === 'rename')) {
731
- copy = (await this.caughtUp([copy], true))[0];
1206
+ const read = push.changes.some((change) => change.op === 'rename');
1207
+ if (read) {
1208
+ copy = (await this.caughtUp([copy], true, caller))[0];
732
1209
  const house = await tree(copy.repository, copy.house);
733
1210
  const sent = await tree(copy.repository, push.house);
734
- if (push.changes.some((change) => change.op === 'rename' && !house.has(relative(change.to)))) {
1211
+ const unread = push.changes.some((change) => change.op === 'rename' &&
1212
+ house.get(relative(change.path)) === sent.get(relative(change.path)) &&
1213
+ !house.has(relative(change.to)));
1214
+ if (unread)
735
1215
  throw new Error(`House's result for ${short(push.commit)} is not readable yet`);
736
- }
737
1216
  const known = new Set(written.map((entry) => relative(entry.path)));
738
1217
  for (const path of new Set([...sent.keys(), ...house.keys()])) {
739
- const entry = house.get(path);
740
- if (known.has(path) || sent.get(path) === entry)
741
- continue;
742
- written = [
743
- ...written,
744
- {
745
- path: `${rootOf(copy.room_handle)}/${path}`,
746
- revision: entry === undefined ? null : (revisionAt(copy, copy.house, path) ?? null),
747
- content: entry === undefined ? null : await blobText(copy.repository, entry),
748
- },
749
- ];
1218
+ if (!known.has(path) && sent.get(path) !== house.get(path))
1219
+ written = [...written, await writtenAt(copy, house, path)];
750
1220
  }
751
1221
  }
752
1222
  const selected = await tree(copy.repository, push.commit);
753
1223
  const houseTree = await tree(copy.repository, copy.house);
1224
+ const parts = pushedBinary(copy, push);
754
1225
  const files = new Map();
755
1226
  const revisions = {};
1227
+ const binary = {};
1228
+ const settled = new Map();
756
1229
  for (const entry of written) {
757
1230
  const path = relative(entry.path);
1231
+ if (entry.revision !== null || !settled.has(path))
1232
+ settled.set(path, entry);
1233
+ }
1234
+ for (const [path, entry] of settled) {
758
1235
  if (!selected.has(path) && !houseTree.has(path) && !declaredWritable(path, copy))
759
1236
  continue;
760
- files.set(path, entry.revision === null ? null : (entry.content ?? (await this.revisionText(entry.path, entry.revision))));
761
1237
  revisions[path] = entry.revision;
1238
+ const part = parts.get(path);
1239
+ const kept = heldAt(copy, copy.house, path);
1240
+ if (entry.revision === null)
1241
+ files.set(path, null);
1242
+ else if (part !== undefined) {
1243
+ files.set(path, { blob: selected.get(path) });
1244
+ binary[path] = part;
1245
+ }
1246
+ else if (kept !== undefined && revisionAt(copy, copy.house, path) === entry.revision) {
1247
+ files.set(path, { blob: houseTree.get(path) });
1248
+ binary[path] = kept;
1249
+ }
1250
+ else
1251
+ files.set(path, entry.content ?? (await this.revisionText(entry.path, entry.revision)));
762
1252
  }
763
1253
  const edits = new Map([...files].filter(([path, content]) => (content === null ? selected.has(path) : selected.get(path) !== blobId(content))));
764
- const received = [...files.keys()].filter((path) => revisionAt(copy, copy.house, path) === revisionAt(copy, push.house, path));
1254
+ const received = read
1255
+ ? []
1256
+ : [...files.keys()].filter((path) => revisionAt(copy, copy.house, path) === revisionAt(copy, push.house, path));
765
1257
  const accepted = `House ${rootOf(copy.room_handle)} with ${push.commit} accepted`;
766
1258
  const house = await importCommits(copy.repository, [
767
1259
  ...(edits.size === 0 ? [] : [{ message: accepted, from: push.commit, files: edits }]),
@@ -779,30 +1271,38 @@ export class LocalCopies {
779
1271
  house,
780
1272
  manifests: {
781
1273
  ...copy.manifests,
782
- [acknowledged]: { parent: push.base, files: revisions },
783
- [house]: { parent: copy.house, files: Object.fromEntries(received.map((path) => [path, revisions[path] ?? null])) },
1274
+ [acknowledged]: manifest(push.base, revisions, binary),
1275
+ [house]: manifest(copy.house, Object.fromEntries(received.map((path) => [path, revisions[path] ?? null])), Object.fromEntries(received.flatMap((path) => (binary[path] === undefined ? [] : [[path, binary[path]]])))),
784
1276
  },
785
1277
  moves: copy.moves.filter(({ from, to }) => !(after.has(to) && !after.has(from))),
786
1278
  push: { ...push, acknowledged },
787
1279
  };
788
1280
  }
789
- async landed(repository, changes, house, selected) {
790
- const text = async (path) => {
791
- const entry = house.get(relative(path));
792
- return entry === undefined ? undefined : blobText(repository, entry);
793
- };
1281
+ async landed(repository, changes, house, selected, model) {
1282
+ const created = new Set(changes.flatMap((change) => (change.op === 'create' || change.op === 'upload' ? [change.path] : [])));
794
1283
  for (const change of changes) {
795
- const current = await text(change.path);
796
- if (change.op === 'remove' && current === undefined)
797
- continue;
798
- if (change.op === 'rename' && current === undefined && house.get(relative(change.to)) === selected.get(relative(change.to))) {
799
- continue;
1284
+ const path = relative(change.path);
1285
+ if (change.op === 'upload') {
1286
+ if (house.get(path) === selected.get(path))
1287
+ continue;
1288
+ return false;
1289
+ }
1290
+ if (change.op === 'remove') {
1291
+ if (created.has(change.path) || !house.has(path))
1292
+ continue;
1293
+ return false;
1294
+ }
1295
+ if (change.op === 'rename') {
1296
+ if (!house.has(path) && house.get(relative(change.to)) === selected.get(relative(change.to)))
1297
+ continue;
1298
+ return false;
800
1299
  }
1300
+ const entry = house.get(path);
1301
+ const current = entry === undefined ? undefined : await blobText(repository, entry);
801
1302
  if ((change.op === 'create' || change.op === 'replace') && current === change.content)
802
1303
  continue;
803
- const parsed = current === undefined ? null : parse(current, change.path);
804
- const document = parsed?.ok === true ? parsed.document : null;
805
- if (document === null)
1304
+ const document = current === undefined ? undefined : readDocument(current, path, model)?.document;
1305
+ if (document === undefined)
806
1306
  return false;
807
1307
  if (change.op === 'set_field' && JSON.stringify(document.fields[change.field]) === JSON.stringify(change.value))
808
1308
  continue;
@@ -815,7 +1315,7 @@ export class LocalCopies {
815
1315
  return true;
816
1316
  }
817
1317
  async revisionText(path, wanted) {
818
- const page = await this.house.post('/kit/door/revisions', {
1318
+ const page = await this.house.door('files/revisions/list', {
819
1319
  path,
820
1320
  revision: wanted,
821
1321
  });