@agentshouse/kit 0.2.1-ce3d2bb67177c → 0.2.1-ce46b2b55463f

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (161) hide show
  1. package/README.md +31 -35
  2. package/dist/acp.js +56 -6
  3. package/dist/afk.js +17 -0
  4. package/dist/agents.js +120 -51
  5. package/dist/api.js +11 -6
  6. package/dist/binaries.js +79 -0
  7. package/dist/bridge.js +30 -26
  8. package/dist/building.js +50 -0
  9. package/dist/calls.js +29 -4
  10. package/dist/chunks.js +49 -0
  11. package/dist/clis.js +61 -49
  12. package/dist/connect.js +108 -0
  13. package/dist/connection.js +33 -38
  14. package/dist/content.js +106 -0
  15. package/dist/conversations.js +325 -141
  16. package/dist/copies.js +720 -247
  17. package/dist/earlier.js +263 -0
  18. package/dist/end.js +4 -5
  19. package/dist/events.js +549 -281
  20. package/dist/files.js +110 -28
  21. package/dist/git-main.js +24 -29
  22. package/dist/git.js +89 -14
  23. package/dist/grok.js +39 -0
  24. package/dist/home.js +65 -29
  25. package/dist/house-cli.js +114 -159
  26. package/dist/house-main.js +3 -3
  27. package/dist/instructions.js +29 -11
  28. package/dist/keys.js +238 -0
  29. package/dist/kit-cli.js +16 -6
  30. package/dist/kit-main.js +3 -3
  31. package/dist/login.js +36 -20
  32. package/dist/logout.js +2 -2
  33. package/dist/mcp.js +38 -1
  34. package/dist/owner.js +46 -0
  35. package/dist/placement.js +9 -23
  36. package/dist/refusals.js +37 -21
  37. package/dist/relay.js +21 -0
  38. package/dist/resident.js +109 -65
  39. package/dist/scope.js +12 -4
  40. package/dist/scripts.js +303 -0
  41. package/dist/secret-input.js +1 -1
  42. package/dist/settings.js +188 -0
  43. package/dist/sign-in.js +49 -3
  44. package/dist/silence.js +28 -0
  45. package/dist/skills.js +41 -96
  46. package/dist/supervisor.js +163 -0
  47. package/dist/transcription.js +37 -25
  48. package/dist/translate.js +18 -9
  49. package/dist/update.js +115 -117
  50. package/dist/wire.js +19 -9
  51. package/dist/work.js +9 -2
  52. package/dist/workspace.js +103 -0
  53. package/package.json +25 -21
  54. package/dist/authority-main.js +0 -22
  55. package/dist/bootstrap.json +0 -24
  56. package/dist/chatgpt.js +0 -32
  57. package/dist/clis-main.js +0 -23
  58. package/dist/configure-main.js +0 -6
  59. package/dist/enrol-main.js +0 -10
  60. package/dist/parts.js +0 -82
  61. package/dist/service-main.js +0 -9
  62. package/dist/service.js +0 -55
  63. package/how-we-work.md +0 -33
  64. package/skills/ask-matt/LICENSE +0 -21
  65. package/skills/ask-matt/PHASE-BOUNDARIES.md +0 -55
  66. package/skills/ask-matt/SKILL.md +0 -97
  67. package/skills/ask-matt/agents/openai.yaml +0 -5
  68. package/skills/code-review/LICENSE +0 -21
  69. package/skills/code-review/SKILL.md +0 -89
  70. package/skills/code-review/agents/openai.yaml +0 -3
  71. package/skills/codebase-design/DEEPENING.md +0 -37
  72. package/skills/codebase-design/DESIGN-IT-TWICE.md +0 -44
  73. package/skills/codebase-design/LICENSE +0 -21
  74. package/skills/codebase-design/SKILL.md +0 -114
  75. package/skills/codebase-design/agents/openai.yaml +0 -3
  76. package/skills/diagnosing-bugs/LICENSE +0 -21
  77. package/skills/diagnosing-bugs/SKILL.md +0 -138
  78. package/skills/diagnosing-bugs/agents/openai.yaml +0 -3
  79. package/skills/diagnosing-bugs/scripts/hitl-loop.template.sh +0 -44
  80. package/skills/domain-modeling/ADR-FORMAT.md +0 -47
  81. package/skills/domain-modeling/GLOSSARY-FORMAT.md +0 -60
  82. package/skills/domain-modeling/LICENSE +0 -21
  83. package/skills/domain-modeling/SKILL.md +0 -74
  84. package/skills/domain-modeling/agents/openai.yaml +0 -3
  85. package/skills/grill-me/LICENSE +0 -21
  86. package/skills/grill-me/SKILL.md +0 -7
  87. package/skills/grill-me/agents/openai.yaml +0 -5
  88. package/skills/grill-with-docs/LICENSE +0 -21
  89. package/skills/grill-with-docs/SKILL.md +0 -7
  90. package/skills/grill-with-docs/agents/openai.yaml +0 -5
  91. package/skills/grilling/LICENSE +0 -21
  92. package/skills/grilling/SKILL.md +0 -30
  93. package/skills/grilling/agents/openai.yaml +0 -3
  94. package/skills/handoff/LICENSE +0 -21
  95. package/skills/handoff/SKILL.md +0 -16
  96. package/skills/handoff/agents/openai.yaml +0 -5
  97. package/skills/implement/LICENSE +0 -21
  98. package/skills/implement/SKILL.md +0 -17
  99. package/skills/implement/agents/openai.yaml +0 -5
  100. package/skills/implement-spec/LICENSE +0 -21
  101. package/skills/implement-spec/SKILL.md +0 -40
  102. package/skills/implement-spec/agents/openai.yaml +0 -5
  103. package/skills/improve-codebase-architecture/HTML-REPORT.md +0 -123
  104. package/skills/improve-codebase-architecture/LICENSE +0 -21
  105. package/skills/improve-codebase-architecture/SKILL.md +0 -71
  106. package/skills/improve-codebase-architecture/agents/openai.yaml +0 -5
  107. package/skills/pr/CREDITS.md +0 -3
  108. package/skills/pr/LICENSE +0 -21
  109. package/skills/pr/SKILL.md +0 -170
  110. package/skills/pr/agents/openai.yaml +0 -3
  111. package/skills/prototype/LICENSE +0 -21
  112. package/skills/prototype/LOGIC.md +0 -67
  113. package/skills/prototype/SKILL.md +0 -26
  114. package/skills/prototype/UI.md +0 -112
  115. package/skills/prototype/agents/openai.yaml +0 -3
  116. package/skills/research/LICENSE +0 -21
  117. package/skills/research/SKILL.md +0 -12
  118. package/skills/research/agents/openai.yaml +0 -3
  119. package/skills/retro/LICENSE +0 -21
  120. package/skills/retro/SKILL.md +0 -44
  121. package/skills/retro/agents/openai.yaml +0 -5
  122. package/skills/tdd/LICENSE +0 -21
  123. package/skills/tdd/SKILL.md +0 -38
  124. package/skills/tdd/agents/openai.yaml +0 -3
  125. package/skills/tdd/mocking.md +0 -59
  126. package/skills/tdd/tests.md +0 -77
  127. package/skills/teach/GLOSSARY-FORMAT.md +0 -35
  128. package/skills/teach/LEARNING-RECORD-FORMAT.md +0 -46
  129. package/skills/teach/LICENSE +0 -21
  130. package/skills/teach/MISSION-FORMAT.md +0 -31
  131. package/skills/teach/RESOURCES-FORMAT.md +0 -32
  132. package/skills/teach/SKILL.md +0 -140
  133. package/skills/teach/agents/openai.yaml +0 -5
  134. package/skills/to-questionnaire/LICENSE +0 -21
  135. package/skills/to-questionnaire/SKILL.md +0 -54
  136. package/skills/to-questionnaire/agents/openai.yaml +0 -5
  137. package/skills/to-spec/LICENSE +0 -21
  138. package/skills/to-spec/SKILL.md +0 -75
  139. package/skills/to-spec/agents/openai.yaml +0 -5
  140. package/skills/to-tickets/LICENSE +0 -21
  141. package/skills/to-tickets/SKILL.md +0 -105
  142. package/skills/to-tickets/agents/openai.yaml +0 -5
  143. package/skills/triage/AGENT-BRIEF.md +0 -207
  144. package/skills/triage/LICENSE +0 -21
  145. package/skills/triage/OUT-OF-SCOPE.md +0 -105
  146. package/skills/triage/SKILL.md +0 -112
  147. package/skills/triage/agents/openai.yaml +0 -5
  148. package/skills/wait-what/LICENSE +0 -21
  149. package/skills/wait-what/SKILL.md +0 -7
  150. package/skills/wait-what/agents/openai.yaml +0 -5
  151. package/skills/wayfinder/LICENSE +0 -21
  152. package/skills/wayfinder/SKILL.md +0 -128
  153. package/skills/wayfinder/agents/openai.yaml +0 -5
  154. package/skills/wizard/LICENSE +0 -21
  155. package/skills/wizard/SKILL.md +0 -44
  156. package/skills/wizard/agents/openai.yaml +0 -3
  157. package/skills/wizard/template.sh +0 -223
  158. package/skills/writing-for-agents/LICENSE +0 -21
  159. package/skills/writing-for-agents/SKILL-MECHANICS.md +0 -22
  160. package/skills/writing-for-agents/SKILL.md +0 -81
  161. package/skills/writing-for-agents/agents/openai.yaml +0 -3
package/dist/copies.js CHANGED
@@ -1,17 +1,33 @@
1
- import { parse } from '@agentshouse/mdmodel';
2
- import { randomUUID } from 'node:crypto';
1
+ import { HOUSE_ROOM_TYPES, readTypes } from '@agentshouse/mdmodel';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
3
  import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { finalRefusal, HouseRefusal, retrying } from './api.js';
6
- import { blobId, blobText, git, isText, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
6
+ import { eachAtOnce, receivedBinary, sentBinary, sorted, sweptTransfers, } from './binaries.js';
7
+ import { chunkEntries, stagedChunks } from './chunks.js';
8
+ import { blobId, blobText, git, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
9
+ import { fileInfo, paused } from './files.js';
7
10
  import { kitHome } from './home.js';
11
+ import { ownAgentCall, rpc, toolAnswer, Unsettled } from './mcp.js';
8
12
  import { operationId } from './operation.js';
9
- import { refusalOf } from './refusals.js';
10
- import { changed } from './translate.js';
11
- import { relayed } from './relay.js';
13
+ import { firstLine, refusalOf, retryable, stopsPush } from './refusals.js';
14
+ import { changed, readDocument } from './translate.js';
12
15
  const BRANCH = 'main';
13
16
  const PROVENANCE = '_provenance';
14
17
  const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
18
+ const ROOMS = 'rooms/';
19
+ const NOT_READY = 'not ready yet; Kit copies it once House has built it';
20
+ const STAGED_CHUNKS = 'house-chunks';
21
+ const SCHEMA = 'settings/schema.yaml';
22
+ // A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
23
+ const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
24
+ class NotReady extends Error {
25
+ rooms;
26
+ constructor(rooms) {
27
+ super(NOT_READY);
28
+ this.rooms = rooms;
29
+ }
30
+ }
15
31
  function causeOf(error) {
16
32
  return error instanceof Error ? error.message : String(error);
17
33
  }
@@ -28,13 +44,34 @@ const REBASED = new Set([
28
44
  'base_revision_stale',
29
45
  'path_exists',
30
46
  'path_absent',
47
+ 'base_required',
48
+ 'binary_file',
49
+ 'text_file',
31
50
  ]);
32
51
  function rebaseFixes(refusal) {
33
- return refusal.code === 'edit_conflict' && refusal.conflicts.some((code) => REBASED.has(code));
52
+ return refusal.code === 'edit_conflict' && refusal.conflicts.some(({ code }) => REBASED.has(code));
34
53
  }
35
- function textOnly(paths) {
54
+ function markdownOnly(paths, maxBytes) {
36
55
  const [named, them] = paths.length === 1 ? [`${paths[0]} is`, 'it'] : [`${paths.join(', ')} are`, 'them'];
37
- return `${named} not text, and house push sends text only; remove ${them} from the commit and place ${them} with house upload`;
56
+ return `${named} Markdown, which holds text only: UTF-8 without NUL, at most ${maxBytes} bytes; commit ${them} as text and push again`;
57
+ }
58
+ async function stopping(sent) {
59
+ try {
60
+ return await sent;
61
+ }
62
+ catch (error) {
63
+ const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
64
+ if (refusal === null || !stopsPush(refusal))
65
+ throw error;
66
+ return { refused: refusal };
67
+ }
68
+ }
69
+ function unstagedUploads(refusal) {
70
+ const named = refusal.detail.upload;
71
+ return Array.isArray(named) ? named : [];
72
+ }
73
+ function heldKey(file) {
74
+ return `${file.sha256}:${file.bytes}`;
38
75
  }
39
76
  function outside(path) {
40
77
  return { refused: true, text: `${path} is not a House Local copy; run house push inside one` };
@@ -72,6 +109,9 @@ function sidecar(path) {
72
109
  function recordOf(key) {
73
110
  return join(kitHome(), 'local-copies', `${key}.json`);
74
111
  }
112
+ function stagingPrefix() {
113
+ return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
114
+ }
75
115
  function rootOf(handle) {
76
116
  return `/rooms/${handle}`;
77
117
  }
@@ -99,32 +139,76 @@ async function removeFile(repository, path) {
99
139
  return;
100
140
  }
101
141
  }
102
- async function readOnlyPlaced(repository, held, written, removed) {
142
+ async function readOnlyFiles(repository, held) {
103
143
  const tracked = new Set((await git(repository, ['ls-files', '-z'])).split('\0'));
104
144
  const holding = new Set(held);
105
- for (const path of removed) {
106
- if (!holding.has(path))
107
- continue;
108
- if (!tracked.has(path))
109
- await removeFile(repository, path);
110
- holding.delete(path);
111
- }
112
- for (const [path, content] of written) {
113
- if (tracked.has(path))
114
- continue;
115
- if (!holding.has(path) && (await exists(join(repository, path))))
116
- continue;
117
- await writeText(join(repository, path), content);
118
- holding.add(path);
119
- }
120
- return [...holding].sort();
145
+ return {
146
+ remove: async (path) => {
147
+ if (!holding.has(path))
148
+ return;
149
+ if (!tracked.has(path))
150
+ await removeFile(repository, path);
151
+ holding.delete(path);
152
+ },
153
+ place: async (path, content) => {
154
+ if (tracked.has(path))
155
+ return;
156
+ if (!holding.has(path) && (await exists(join(repository, path))))
157
+ return;
158
+ await writeText(join(repository, path), content);
159
+ holding.add(path);
160
+ },
161
+ held: () => [...holding].sort(),
162
+ };
163
+ }
164
+ async function readOnlyPlaced(repository, held, written, removed) {
165
+ const files = await readOnlyFiles(repository, held);
166
+ for (const path of removed)
167
+ await files.remove(path);
168
+ for (const [path, content] of written)
169
+ await files.place(path, content);
170
+ return files.held();
171
+ }
172
+ async function importedWhole(repository, scope, entries, commit, readOnly, kept) {
173
+ const revisions = {};
174
+ const binary = {};
175
+ async function* tracked() {
176
+ const binaries = [];
177
+ for await (const entry of entries) {
178
+ if (!('content' in entry)) {
179
+ if (declaredWritable(entry.path, scope))
180
+ binaries.push(entry);
181
+ continue;
182
+ }
183
+ const { path, content, revision: held, provenance } = entry;
184
+ if (declaredWritable(path, scope)) {
185
+ revisions[path] = held;
186
+ yield [path, content];
187
+ }
188
+ else {
189
+ await readOnly.place(path, content);
190
+ }
191
+ if (provenance !== undefined)
192
+ await readOnly.place(sidecar(path), provenance);
193
+ }
194
+ const blobs = await kept(binaries);
195
+ for (const entry of binaries) {
196
+ const blob = blobs.get(entry.path);
197
+ if (blob === undefined)
198
+ continue;
199
+ revisions[entry.path] = entry.revision;
200
+ binary[entry.path] = { bytes: entry.bytes, sha256: entry.sha256 };
201
+ yield [entry.path, { blob }];
202
+ }
203
+ }
204
+ return { house: await importCommits(repository, [{ ...commit, files: tracked() }]), revisions, binary };
121
205
  }
122
206
  async function ruled(path, rules) {
123
207
  if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
124
208
  await writeFile(path, rules);
125
209
  }
126
210
  async function ignoreRules(repository, writable, copy) {
127
- const readOnly = ('read_only' in copy ? copy.read_only : [...copy.readOnly.keys()]).filter((path) => !path.split('/').includes(PROVENANCE));
211
+ const readOnly = copy.read_only.filter((path) => !path.split('/').includes(PROVENANCE));
128
212
  const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
129
213
  const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
130
214
  const house = [
@@ -155,15 +239,61 @@ async function ignoreRules(repository, writable, copy) {
155
239
  '',
156
240
  ].join('\n'));
157
241
  }
158
- function revisionAt(copy, commit, path) {
242
+ async function modelOf(copy, files) {
243
+ const entry = files.get(SCHEMA);
244
+ if (entry !== undefined)
245
+ return readTypes(await blobText(copy.repository, entry), HOUSE_ROOM_TYPES);
246
+ return readTypes(copy.read_only.includes(SCHEMA) ? await readFile(join(copy.repository, SCHEMA), 'utf8') : '', HOUSE_ROOM_TYPES);
247
+ }
248
+ function entryAt(copy, commit, path) {
159
249
  let manifest = copy.manifests[commit];
160
250
  while (manifest !== undefined) {
161
- if (path in manifest.files)
162
- return manifest.files[path] ?? undefined;
251
+ if (path in manifest.files) {
252
+ const revision = manifest.files[path];
253
+ if (revision === null || revision === undefined)
254
+ return undefined;
255
+ const held = manifest.binary?.[path];
256
+ return held === undefined ? { revision } : { revision, held };
257
+ }
163
258
  manifest = manifest.parent === null ? undefined : copy.manifests[manifest.parent];
164
259
  }
165
260
  return undefined;
166
261
  }
262
+ function revisionAt(copy, commit, path) {
263
+ return entryAt(copy, commit, path)?.revision;
264
+ }
265
+ function heldAt(copy, commit, path) {
266
+ return entryAt(copy, commit, path)?.held;
267
+ }
268
+ function manifest(parent, files, binary) {
269
+ return Object.keys(binary).length === 0 ? { parent, files } : { parent, files, binary };
270
+ }
271
+ function binaryParts(changes) {
272
+ return changes.filter((change) => change.op === 'upload');
273
+ }
274
+ function pushedBinary(copy, push) {
275
+ const sent = new Map();
276
+ for (const change of push.changes) {
277
+ if (change.op === 'upload')
278
+ sent.set(relative(change.path), { bytes: change.bytes, sha256: change.sha256 });
279
+ else if (change.op === 'rename') {
280
+ const held = heldAt(copy, push.base, relative(change.path));
281
+ if (held !== undefined)
282
+ sent.set(relative(change.to), held);
283
+ }
284
+ }
285
+ return sent;
286
+ }
287
+ async function writtenAt(copy, house, path) {
288
+ const named = `${rootOf(copy.room_handle)}/${path}`;
289
+ const entry = house.get(path);
290
+ if (entry === undefined)
291
+ return { path: named, revision: null, content: null };
292
+ const revision = revisionAt(copy, copy.house, path) ?? null;
293
+ return heldAt(copy, copy.house, path) === undefined
294
+ ? { path: named, revision, content: await blobText(copy.repository, entry) }
295
+ : { path: named, revision };
296
+ }
167
297
  function without(copy) {
168
298
  const { push, ...settled } = copy;
169
299
  return settled;
@@ -179,11 +309,36 @@ function movedPaths(moves, before, after) {
179
309
  .filter((pair) => pairs.filter((other) => other.from === pair.from || other.to === pair.to).length === 1)
180
310
  .map(({ from, to }) => [from, to]));
181
311
  }
312
+ function sameBytesMoved(before, after, held, binary, intended) {
313
+ const arriving = new Set(intended.values());
314
+ const leaving = new Map();
315
+ const added = new Map();
316
+ const listed = (paths, file, path) => {
317
+ paths.set(heldKey(file), [...(paths.get(heldKey(file)) ?? []), path]);
318
+ };
319
+ for (const [path, entry] of before) {
320
+ const file = held(path);
321
+ if (file !== undefined && !intended.has(path) && !binary.has(path) && after.get(path) !== entry)
322
+ listed(leaving, file, path);
323
+ }
324
+ for (const [path, file] of binary) {
325
+ if (!arriving.has(path) && held(path) === undefined)
326
+ listed(added, file, path);
327
+ }
328
+ const moved = new Map();
329
+ for (const [key, [from, ...others]] of leaving) {
330
+ const [to, ...more] = added.get(key) ?? [];
331
+ if (others.length === 0 && more.length === 0 && to !== undefined && !after.has(from) && !before.has(to))
332
+ moved.set(from, to);
333
+ }
334
+ return moved;
335
+ }
182
336
  export class LocalCopies {
183
337
  owner;
184
338
  house;
185
339
  copies = new Map();
186
340
  queues = new Map();
341
+ waiting = new Set();
187
342
  answering = Promise.resolve();
188
343
  constructor(house) {
189
344
  this.house = house;
@@ -191,17 +346,36 @@ export class LocalCopies {
191
346
  key: 'owner',
192
347
  submit: async (operation, changes) => {
193
348
  try {
194
- const outcome = await retrying(() => house.post('/kit/door/batch', { operation_id: operation, changes }));
195
- return { accepted: outcome.contents };
349
+ const outcome = await retrying(() => stopping(house.door('files/batch', { operation_id: operation, changes })));
350
+ return 'refused' in outcome ? outcome : { accepted: outcome.contents };
196
351
  }
197
352
  catch (error) {
198
353
  if (!finalRefusal(error))
199
354
  throw error;
200
355
  return {
201
- refused: refusalOf(error.text) ?? { code: '', text: `House refused it with ${error.status}`, conflicts: [] },
356
+ refused: refusalOf(error.text) ?? { code: '', text: `House refused it with ${error.status}`, conflicts: [], detail: {} },
202
357
  };
203
358
  }
204
359
  },
360
+ finish: async (upload, _room, partSha256) => {
361
+ try {
362
+ return (await house.act('uploads/finish', { upload, part_sha256: partSha256 }));
363
+ }
364
+ catch (error) {
365
+ if (!(error instanceof HouseRefusal))
366
+ throw new Unsettled(causeOf(error));
367
+ throw retryable(error.text) === false ? new Error(houseCause(error)) : new Unsettled(houseCause(error));
368
+ }
369
+ },
370
+ info: async (path, revision) => {
371
+ const read = toolAnswer({
372
+ status: 200,
373
+ text: await ownAgentCall(rpc('tools/call', { name: 'info', arguments: { path, ...(revision === undefined ? {} : { ref: revision }) } })),
374
+ });
375
+ if ('refused' in read)
376
+ throw new Error(read.refused.text);
377
+ return fileInfo(read.text);
378
+ },
205
379
  };
206
380
  }
207
381
  async load() {
@@ -215,25 +389,23 @@ export class LocalCopies {
215
389
  this.copies.set(copy.key, copy);
216
390
  }
217
391
  }
218
- async mapping() {
219
- await this.answering;
220
- return [...this.copies.values()]
221
- .filter((copy) => copy.unfollowed === undefined)
222
- .map((copy) => `${rootOf(copy.room_handle)}: ${copy.repository}`)
223
- .sort();
392
+ async swept() {
393
+ const rooms = join(localCopies(), 'rooms');
394
+ for (const name of await readdir(rooms).catch(() => [])) {
395
+ if (name.startsWith(stagingPrefix()))
396
+ await rm(join(rooms, name), { recursive: true, force: true });
397
+ }
398
+ for (const copy of this.copies.values())
399
+ await sweptTransfers(copy.repository);
224
400
  }
225
401
  connected() {
226
402
  return this.selected();
227
403
  }
228
- async changed(rooms) {
229
- if (rooms.length > 0)
230
- await this.selected(rooms);
231
- }
232
- fetch() {
233
- return this.answered(() => this.fetched(true));
404
+ fetch(caller) {
405
+ return this.answered(() => this.fetched(true, caller));
234
406
  }
235
- async selected(rooms) {
236
- const copied = await this.answered(() => this.fetched(false, rooms));
407
+ async selected() {
408
+ const copied = await this.answered(() => this.fetched(false, this.owner));
237
409
  if (copied.refused)
238
410
  logged(copied.text);
239
411
  }
@@ -242,16 +414,14 @@ export class LocalCopies {
242
414
  this.answering = answered.catch(() => undefined);
243
415
  return answered;
244
416
  }
245
- async fetched(refreshing, rooms) {
246
- const { local_copy: answer } = await this.house.post('/kit/local-copy/selection', rooms === undefined ? {} : { rooms });
247
- const synced = new Map(answer.rooms.map((room) => [`rooms/${room.room_ref}`, room.room_handle]));
248
- const named = rooms === undefined ? null : new Set(rooms.map((room) => `rooms/${room}`));
249
- const held = () => [...this.copies.values()].filter((copy) => named === null || named.has(copy.key));
250
- return this.holding([...new Set([...held().map((copy) => copy.key), ...synced.keys()])].sort(), async () => {
417
+ async fetched(refreshing, caller) {
418
+ const { local_copy: answer } = await this.house.act('local-copies/selection/read', {});
419
+ const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
420
+ return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
251
421
  const said = [];
252
422
  let refused = false;
253
423
  const mark = refreshing ? 'said' : 'unsaid';
254
- for (const copy of held()) {
424
+ for (const copy of [...this.copies.values()]) {
255
425
  if (synced.has(copy.key) || copy.unfollowed === 'said' || copy.unfollowed === mark)
256
426
  continue;
257
427
  await this.save({ ...copy, unfollowed: mark });
@@ -259,11 +429,12 @@ export class LocalCopies {
259
429
  said.push(`${rootOf(copy.room_handle)}: no longer followed; kept as it is`);
260
430
  }
261
431
  const current = [];
432
+ const wanted = new Map();
262
433
  for (const [key, handle] of synced) {
263
434
  try {
264
435
  const held = this.copies.get(key);
265
436
  if (held === undefined || !(await exists(held.repository))) {
266
- said.push(await this.materialize(key, handle));
437
+ wanted.set(key, handle);
267
438
  continue;
268
439
  }
269
440
  const { unfollowed, ...followed } = held;
@@ -274,18 +445,62 @@ export class LocalCopies {
274
445
  said.push(`${rootOf(handle)}: ${causeOf(error)}`);
275
446
  }
276
447
  }
277
- if (refreshing && current.length > 0) {
278
- const failed = await this.refreshed(current).then(() => null, houseCause);
279
- for (const copy of current)
280
- said.push(...moved(copy, this.copies.get(copy.key)));
448
+ const unready = [];
449
+ const unsettled = [];
450
+ if (wanted.size > 0) {
451
+ const { authorities } = await this.discovered([...wanted.values()]);
452
+ for (const [key, handle] of wanted) {
453
+ const authority = authorities.find((entry) => `${ROOMS}${entry.room_ref}` === key);
454
+ try {
455
+ if (authority === undefined)
456
+ throw new Error('House serves nothing of it to copy');
457
+ said.push(await this.copied({ key, handle, authority }));
458
+ }
459
+ catch (error) {
460
+ if (error instanceof NotReady) {
461
+ unready.push(...error.rooms);
462
+ said.push(`${rootOf(handle)}: ${NOT_READY}`);
463
+ }
464
+ else {
465
+ refused = true;
466
+ unsettled.push(key.slice(ROOMS.length));
467
+ said.push(`${rootOf(handle)}: ${causeOf(error)}`);
468
+ }
469
+ }
470
+ }
471
+ }
472
+ const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
473
+ if (reading.length > 0) {
474
+ const failed = await this.refreshed(reading, unready, caller).then(() => null, houseCause);
475
+ for (const copy of reading) {
476
+ said.push(...(unready.includes(copy.room_ref) ? [`${rootOf(copy.room_handle)}: ${NOT_READY}`] : moved(copy, this.copies.get(copy.key))));
477
+ }
281
478
  if (failed !== null) {
282
479
  refused = true;
480
+ unsettled.push(...reading.map((copy) => copy.room_ref));
283
481
  said.push(failed);
284
482
  }
285
483
  }
484
+ for (const room of [...this.waiting]) {
485
+ if (!unready.includes(room) && !unsettled.includes(room))
486
+ this.waiting.delete(room);
487
+ }
488
+ const due = unready.filter((room) => !this.waiting.has(room));
489
+ for (const room of due)
490
+ this.waiting.add(room);
491
+ if (due.length > 0)
492
+ void this.askedAgain(due);
286
493
  return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
287
494
  });
288
495
  }
496
+ async askedAgain(rooms) {
497
+ let asking = rooms;
498
+ for (let step = 0; asking.length > 0; step++) {
499
+ await paused(ASKED_AGAIN_MS[Math.min(step, ASKED_AGAIN_MS.length - 1)]);
500
+ await this.selected().catch(logged);
501
+ asking = asking.filter((room) => this.waiting.has(room));
502
+ }
503
+ }
289
504
  async observed(report) {
290
505
  const key = await this.keyAt(dirname(report.common));
291
506
  if (key === undefined)
@@ -347,132 +562,188 @@ export class LocalCopies {
347
562
  this.copies.set(copy.key, copy);
348
563
  return copy;
349
564
  }
350
- async read(handle) {
351
- const prefixes = [rootOf(handle)];
352
- const discovery = await this.house.post('/kit/door/discover', { prefixes });
353
- let bundle = await this.house.post('/kit/door/bootstrap', { prefixes });
354
- if ('transfer' in bundle)
355
- bundle = (await (await fetch(relayed(bundle.transfer.url))).json());
356
- const [view] = bundle.scopes;
357
- if (view === undefined)
358
- return null;
359
- const authority = discovery.authorities.find((entry) => entry.room_ref === view.manifest.anchor);
360
- const contents = new Map(view.contents.map((entry) => [entry.path, entry.content]));
361
- const read = {
362
- anchor: view.manifest.anchor,
363
- position: view.manifest.position,
364
- log_epoch: view.manifest.log_epoch,
365
- write: authority.operations.write,
366
- protected: authority.protected_paths,
367
- writable: new Map(),
368
- revisions: {},
369
- readOnly: new Map(),
370
- };
371
- for (const file of view.manifest.files) {
372
- const content = contents.get(file.path);
373
- if (file.writable) {
374
- read.writable.set(file.path, content);
375
- read.revisions[file.path] = file.revision;
376
- }
377
- else {
378
- read.readOnly.set(file.path, content);
565
+ async discovered(handles) {
566
+ const discover = (group) => this.house.door('rooms/discover', { prefixes: group.map(rootOf) });
567
+ const [first, ...rest] = handles;
568
+ const { bounds, authorities } = await discover([first]);
569
+ for (const group of grouped(rest, bounds.max_prefixes))
570
+ authorities.push(...(await discover(group)).authorities);
571
+ return { bounds, authorities };
572
+ }
573
+ async chunked(root, room, directory) {
574
+ await mkdir(directory, { recursive: true });
575
+ for (let renewed = false;; renewed = true) {
576
+ const { rooms } = await this.house.door('room-chunks/list', { prefixes: [root] });
577
+ const list = rooms.find((entry) => entry.room_ref === room);
578
+ if (list === undefined)
579
+ throw new Error('House serves nothing of it to copy');
580
+ if (!list.ready)
581
+ throw new NotReady([room]);
582
+ const held = new Set(await readdir(directory));
583
+ const failure = await stagedChunks(directory, list.chunks.filter(({ hash }) => !held.has(hash)));
584
+ if (failure === undefined) {
585
+ return { position: list.position, log_epoch: list.log_epoch, entries: chunkEntries(directory, list.chunks, list.models) };
379
586
  }
380
- if (file.provenance !== undefined)
381
- read.readOnly.set(sidecar(file.path), file.provenance);
587
+ if (renewed && (await readdir(directory)).length === held.size)
588
+ throw failure;
382
589
  }
383
- return read;
384
590
  }
385
- async materialize(key, handle) {
386
- const read = await this.read(handle);
387
- if (read === null)
388
- throw new Error('House serves nothing of it to copy');
389
- const repository = join(localCopies(), 'rooms', read.anchor);
591
+ async materialized({ key, handle, authority }, whole) {
592
+ const repository = join(localCopies(), 'rooms', authority.room_ref);
390
593
  if (await exists(repository))
391
594
  throw new Error(`${repository} already exists and Kit leaves it as it is`);
392
- const staging = join(dirname(repository), `.${randomUUID()}`);
595
+ const staging = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
596
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
393
597
  await mkdir(staging, { recursive: true });
394
- let house;
395
- let readOnly;
598
+ let copy;
396
599
  try {
397
600
  await git(staging, ['init', '-q', '-b', BRANCH]);
398
- house = await importCommits(staging, [
399
- { message: `House ${rootOf(handle)}`, from: null, files: read.writable },
400
- ]);
601
+ await git(staging, ['config', 'core.precomposeUnicode', 'true']);
602
+ const read = await whole(join(staging, '.git', STAGED_CHUNKS));
603
+ const readOnly = await readOnlyFiles(staging, []);
604
+ const commit = { message: `House ${rootOf(handle)}`, from: null };
605
+ const { house, revisions } = await importedWhole(staging, scope, read.entries, commit, readOnly, async () => new Map());
401
606
  await git(staging, ['update-ref', `refs/heads/${BRANCH}`, house]);
402
607
  await git(staging, ['reset', '-q', '--hard']);
403
- readOnly = await readOnlyPlaced(staging, [], read.readOnly, []);
404
- await ignoreRules(staging, [...read.writable.keys()], read);
608
+ copy = {
609
+ key,
610
+ room_ref: authority.room_ref,
611
+ room_handle: handle,
612
+ repository,
613
+ house,
614
+ position: read.position,
615
+ log_epoch: read.log_epoch,
616
+ ...scope,
617
+ read_only: readOnly.held(),
618
+ manifests: { [house]: { parent: null, files: revisions } },
619
+ moves: [],
620
+ };
621
+ await ignoreRules(staging, Object.keys(revisions), copy);
622
+ await rm(join(staging, '.git', STAGED_CHUNKS), { recursive: true, force: true });
405
623
  await rename(staging, repository);
406
624
  }
407
625
  catch (error) {
408
626
  await rm(staging, { recursive: true, force: true });
409
627
  throw error;
410
628
  }
411
- await this.save({
412
- key,
413
- room_ref: read.anchor,
414
- room_handle: handle,
415
- repository,
416
- house,
417
- position: read.position,
418
- log_epoch: read.log_epoch,
419
- write: read.write,
420
- protected: read.protected,
421
- read_only: readOnly,
422
- manifests: { [house]: { parent: null, files: read.revisions } },
423
- moves: [],
424
- });
425
- return `${rootOf(handle)}: copied to ${repository}`;
629
+ return this.save(copy);
426
630
  }
427
- async received(copy) {
428
- const read = await this.read(copy.room_handle);
429
- if (read === null)
430
- return copy;
431
- const house = await importCommits(copy.repository, [
432
- { message: `House ${rootOf(copy.room_handle)} at position ${read.position}`, from: copy.house, replaces: true, files: read.writable },
433
- ]);
434
- const removed = copy.read_only.filter((path) => !read.readOnly.has(path));
435
- const readOnly = await readOnlyPlaced(copy.repository, copy.read_only, read.readOnly, removed);
436
- const next = {
437
- ...copy,
438
- house,
439
- position: read.position,
440
- log_epoch: read.log_epoch,
441
- write: read.write,
442
- protected: read.protected,
443
- read_only: readOnly,
444
- manifests: { ...copy.manifests, [house]: { parent: null, files: read.revisions } },
445
- };
446
- await ignoreRules(copy.repository, [...read.writable.keys()], next);
447
- return next;
631
+ async copied(wanted) {
632
+ const { handle, authority } = wanted;
633
+ const root = rootOf(handle);
634
+ let answer = null;
635
+ try {
636
+ if (!this.waiting.has(authority.room_ref)) {
637
+ answer = await this.house.door('changes/enumerate', {
638
+ prefixes: [root],
639
+ positions: [
640
+ { authority: authority.room_ref, position: '0', log_epoch: authority.log_epoch, page: { through: authority.position, read: 0 } },
641
+ ],
642
+ });
643
+ }
644
+ }
645
+ catch (error) {
646
+ if (expiredAuthority(error) !== authority.room_ref)
647
+ throw error;
648
+ }
649
+ const view = answer?.authorities.find((entry) => entry.authority === authority.room_ref);
650
+ if (answer !== null && view === undefined)
651
+ throw new Error('House serves nothing of it to copy');
652
+ if (view !== undefined && view.next === undefined) {
653
+ const entries = view.changes.filter((change) => 'content' in change);
654
+ const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, entries }));
655
+ return `${root}: copied to ${copy.repository}`;
656
+ }
657
+ const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
658
+ const [caught] = await this.caughtUp([copy], true, this.owner);
659
+ await succeeded(copy.repository, ['merge', '--ff-only', '-q', caught.house]);
660
+ return `${root}: copied to ${copy.repository}`;
448
661
  }
449
- async refreshed(copies) {
450
- const discover = (group) => this.house.post('/kit/door/discover', { prefixes: group.map((copy) => rootOf(copy.room_handle)) });
451
- const [first, ...rest] = copies;
452
- const { bounds, authorities } = await discover([first]);
453
- for (const group of grouped(rest, bounds.max_prefixes))
454
- authorities.push(...(await discover(group)).authorities);
662
+ async received(copy, authority, caller) {
663
+ const root = rootOf(copy.room_handle);
664
+ const directory = join(copy.repository, '.git', STAGED_CHUNKS);
665
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
666
+ try {
667
+ const read = await this.chunked(root, copy.room_ref, directory);
668
+ const readOnly = await readOnlyFiles(copy.repository, copy.read_only);
669
+ for (const path of copy.read_only)
670
+ await readOnly.remove(path);
671
+ const commit = { message: `House ${root} at position ${read.position}`, from: copy.house, replaces: true };
672
+ const houseTree = await tree(copy.repository, copy.house);
673
+ const pushing = await this.pushing(copy);
674
+ const kept = async (entries) => {
675
+ const present = new Set(entries.map((entry) => entry.path));
676
+ const leaving = new Map([...houseTree].flatMap(([path, blob]) => {
677
+ const held = present.has(path) ? undefined : heldAt(copy, copy.house, path);
678
+ return held === undefined ? [] : [[heldKey(held), blob]];
679
+ }));
680
+ const blobs = new Map();
681
+ for (const entry of entries) {
682
+ const blob = await this.binaryIn(copy, houseTree, entry, pushing, leaving, caller);
683
+ if (blob !== null)
684
+ blobs.set(entry.path, blob);
685
+ }
686
+ return blobs;
687
+ };
688
+ const { house, revisions, binary } = await importedWhole(copy.repository, scope, read.entries, commit, readOnly, kept);
689
+ const next = {
690
+ ...copy,
691
+ ...scope,
692
+ house,
693
+ position: read.position,
694
+ log_epoch: read.log_epoch,
695
+ read_only: readOnly.held(),
696
+ manifests: { ...copy.manifests, [house]: manifest(null, revisions, binary) },
697
+ };
698
+ await ignoreRules(copy.repository, Object.keys(revisions), next);
699
+ return next;
700
+ }
701
+ finally {
702
+ await rm(directory, { recursive: true, force: true });
703
+ }
704
+ }
705
+ async reread(copy, caller) {
706
+ const { authorities } = await this.discovered([copy.room_handle]);
707
+ const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
708
+ if (authority === undefined)
709
+ throw new NotReady([copy.room_ref]);
710
+ return this.received(copy, authority, caller);
711
+ }
712
+ async refreshed(copies, unready, caller) {
713
+ const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
714
+ const read = [];
455
715
  const unchanged = [];
456
716
  for (const copy of copies) {
457
717
  const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
458
718
  if (authority !== undefined &&
459
719
  JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
460
- await this.save(await this.received(copy));
720
+ try {
721
+ read.push(await this.save(await this.received(copy, authority, caller)));
722
+ }
723
+ catch (error) {
724
+ if (!(error instanceof NotReady))
725
+ throw error;
726
+ unready.push(...error.rooms);
727
+ }
461
728
  }
462
729
  else {
463
730
  unchanged.push(copy);
464
731
  }
465
732
  }
466
733
  for (const group of grouped(unchanged, bounds.max_prefixes))
467
- await this.caughtUp(group, false);
734
+ await this.caughtUp(group, false, caller, unready);
735
+ for (const group of grouped(read, bounds.max_prefixes))
736
+ await this.caughtUp(group, true, caller, unready);
468
737
  }
469
- async caughtUp(copies, whole) {
738
+ async caughtUp(copies, whole, caller, unready) {
470
739
  const caught = new Map(copies.map((copy) => [copy.key, copy]));
740
+ const read = new Set();
741
+ const waiting = unready ?? [];
471
742
  let asking = [...copies];
472
743
  while (asking.length > 0) {
473
744
  let answer;
474
745
  try {
475
- answer = await this.house.post('/kit/door/enumerate', {
746
+ answer = await this.house.door('changes/enumerate', {
476
747
  prefixes: asking.map((copy) => rootOf(copy.room_handle)),
477
748
  positions: asking.map((copy) => ({ authority: copy.room_ref, position: copy.position, log_epoch: copy.log_epoch })),
478
749
  });
@@ -481,8 +752,22 @@ export class LocalCopies {
481
752
  const expired = asking.find((copy) => copy.room_ref === expiredAuthority(error));
482
753
  if (expired === undefined)
483
754
  throw error;
484
- caught.set(expired.key, await this.save(await this.received(expired)));
485
755
  asking = asking.filter((copy) => copy !== expired);
756
+ if (read.has(expired.key)) {
757
+ waiting.push(expired.room_ref);
758
+ continue;
759
+ }
760
+ read.add(expired.key);
761
+ try {
762
+ const next = await this.save(await this.reread(expired, caller));
763
+ caught.set(expired.key, next);
764
+ asking.push(next);
765
+ }
766
+ catch (refused) {
767
+ if (!(refused instanceof NotReady))
768
+ throw refused;
769
+ waiting.push(expired.room_ref);
770
+ }
486
771
  continue;
487
772
  }
488
773
  const advanced = [];
@@ -490,22 +775,83 @@ export class LocalCopies {
490
775
  const view = answer.authorities.find((entry) => entry.authority === copy.room_ref);
491
776
  if (view === undefined || (view.position === copy.position && view.log_epoch === copy.log_epoch))
492
777
  continue;
493
- const next = await this.save(await this.incorporated(copy, view.changes, view.position, view.log_epoch));
778
+ const next = await this.save(await this.incorporated(copy, view.changes, view.position, view.log_epoch, caller));
494
779
  caught.set(copy.key, next);
495
780
  advanced.push(next);
496
781
  }
497
- asking = whole ? advanced : [];
782
+ asking = advanced.filter((copy) => whole || read.has(copy.key));
498
783
  }
784
+ if (unready === undefined && waiting.length > 0)
785
+ throw new NotReady(waiting);
499
786
  return copies.map((copy) => caught.get(copy.key));
500
787
  }
501
- async incorporated(copy, changes, position, logEpoch) {
788
+ async pushing(copy) {
789
+ const sent = copy.push === undefined ? new Map() : pushedBinary(copy, copy.push);
790
+ if (sent.size === 0)
791
+ return new Map();
792
+ const commit = await tree(copy.repository, copy.push.commit);
793
+ return new Map([...sent].map(([path, held]) => [path, { ...held, blob: commit.get(path) }]));
794
+ }
795
+ async binaryIn(copy, houseTree, change, pushing, leaving, caller) {
796
+ const pushed = pushing.get(change.path);
797
+ if (pushed !== undefined && heldKey(pushed) === heldKey(change))
798
+ return pushed.blob;
799
+ const moved = leaving.get(heldKey(change));
800
+ const kept = heldAt(copy, copy.house, change.path) ?? pushed;
801
+ if (kept === undefined)
802
+ return moved ?? null;
803
+ if (heldKey(kept) === heldKey(change))
804
+ return houseTree.get(change.path);
805
+ return moved ?? this.revisionBlob(copy, change.path, change.revision, caller);
806
+ }
807
+ revisionBlob(copy, path, revision, caller, first = null) {
808
+ const named = `${rootOf(copy.room_handle)}/${path}`;
809
+ let answered = first;
810
+ return receivedBinary(copy.repository, named, async () => {
811
+ const file = answered ?? (await caller.info(named, revision));
812
+ answered = null;
813
+ if (file === null)
814
+ throw new Error(`House holds no binary file at ${named}`);
815
+ return file;
816
+ });
817
+ }
818
+ async incorporated(copy, changes, position, logEpoch, caller) {
502
819
  const houseTree = await tree(copy.repository, copy.house);
503
820
  const held = new Set(copy.read_only);
504
821
  const files = new Map();
505
822
  const revisions = {};
823
+ const binary = {};
506
824
  const written = new Map();
507
825
  const removed = [];
826
+ const pushing = await this.pushing(copy);
827
+ const leaving = new Map(changes.flatMap((change) => {
828
+ const kept = 'removed' in change ? heldAt(copy, copy.house, change.path) : undefined;
829
+ return kept === undefined ? [] : [[heldKey(kept), houseTree.get(change.path)]];
830
+ }));
508
831
  for (const change of changes) {
832
+ if (!('removed' in change) && !('content' in change)) {
833
+ const blob = await this.binaryIn(copy, houseTree, change, pushing, leaving, caller);
834
+ if (blob === null) {
835
+ if (houseTree.has(change.path)) {
836
+ files.set(change.path, null);
837
+ revisions[change.path] = null;
838
+ }
839
+ else if (held.has(change.path))
840
+ removed.push(change.path, sidecar(change.path));
841
+ continue;
842
+ }
843
+ const kept = heldAt(copy, copy.house, change.path);
844
+ const unchanged = houseTree.get(change.path) === blob &&
845
+ kept !== undefined &&
846
+ heldKey(kept) === heldKey(change) &&
847
+ revisionAt(copy, copy.house, change.path) === change.revision;
848
+ if (unchanged)
849
+ continue;
850
+ files.set(change.path, { blob });
851
+ revisions[change.path] = change.revision;
852
+ binary[change.path] = { bytes: change.bytes, sha256: change.sha256 };
853
+ continue;
854
+ }
509
855
  const gone = 'removed' in change;
510
856
  if (houseTree.has(change.path) || (!gone && !held.has(change.path) && declaredWritable(change.path, copy))) {
511
857
  const unchanged = gone
@@ -531,7 +877,7 @@ export class LocalCopies {
531
877
  house = await importCommits(copy.repository, [
532
878
  { message: `House ${rootOf(copy.room_handle)} at position ${position}`, from: copy.house, files },
533
879
  ]);
534
- manifests = { ...manifests, [house]: { parent: copy.house, files: revisions } };
880
+ manifests = { ...manifests, [house]: manifest(copy.house, revisions, binary) };
535
881
  }
536
882
  const next = {
537
883
  ...copy,
@@ -546,16 +892,25 @@ export class LocalCopies {
546
892
  }
547
893
  async pushed(key, worktree, selector, caller) {
548
894
  let copy = this.copies.get(key);
549
- if (copy.push?.state === 'submitted')
550
- return this.settle(copy, caller);
551
895
  const selected = await run(worktree, ['rev-parse', '--verify', '--quiet', '--end-of-options', `${selector}^{commit}`]);
552
896
  if (selected.status !== 0)
553
897
  return { refused: true, text: `${selector} names no commit` };
554
898
  const commit = selected.stdout.toString('utf8').trim();
899
+ if (copy.push?.state === 'submitted') {
900
+ if (copy.push.commit === commit)
901
+ return this.settle(copy, caller);
902
+ const replaced = await this.recovered(copy, caller, async (current) => {
903
+ await this.save(without(current));
904
+ return { refused: false, text: '' };
905
+ });
906
+ if (replaced.refused)
907
+ return replaced;
908
+ copy = this.copies.get(key);
909
+ }
555
910
  if (copy.push !== undefined) {
556
911
  const held = copy.push.acknowledged;
557
912
  if (held === undefined || !(await succeeded(worktree, ['merge-base', '--is-ancestor', held, commit]))) {
558
- return this.finished(copy);
913
+ return this.finished(copy, caller);
559
914
  }
560
915
  copy = await this.save(without(copy));
561
916
  }
@@ -590,24 +945,33 @@ export class LocalCopies {
590
945
  });
591
946
  return this.settle(submitted, caller);
592
947
  }
948
+ async binaryFiles(copy, arriving) {
949
+ if (arriving.length === 0)
950
+ return new Map();
951
+ const { bounds } = await this.house.door('rooms/discover', { prefixes: [rootOf(copy.room_handle)] });
952
+ const { binary, markdown } = await sorted(copy.repository, arriving, bounds.max_authored_file_bytes);
953
+ if (markdown.length > 0)
954
+ throw new Error(markdownOnly(markdown, bounds.max_authored_file_bytes));
955
+ return binary;
956
+ }
593
957
  async derived(copy, base, commit) {
594
958
  const root = rootOf(copy.room_handle);
595
959
  const before = await tree(copy.repository, base);
596
960
  const after = await tree(copy.repository, commit);
597
- const moved = movedPaths(copy.moves, before, after);
961
+ const binary = await this.binaryFiles(copy, [...after].filter(([path, entry]) => before.get(path) !== entry));
962
+ const held = (path) => heldAt(copy, base, path);
963
+ const heldBinary = (path) => held(path) !== undefined;
964
+ const intended = new Map([...movedPaths(copy.moves, before, after)].filter(([from, to]) => heldBinary(from) === binary.has(to)));
965
+ const moved = new Map([...intended, ...sameBytesMoved(before, after, held, binary, intended)]);
598
966
  const arrived = new Set(moved.values());
599
- const arriving = [...after].filter(([path, entry]) => before.get(path) !== entry);
600
- if (arriving.length > 0) {
601
- const { bounds } = await this.house.post('/kit/door/discover', { prefixes: [root] });
602
- const binary = [];
603
- for (const [path, entry] of arriving) {
604
- if (!(await isText(copy.repository, entry, bounds.max_authored_file_bytes)))
605
- binary.push(path);
606
- }
607
- if (binary.length > 0)
608
- throw new Error(textOnly(binary));
609
- }
610
967
  const text = (entry) => blobText(copy.repository, entry);
968
+ const created = async (path, entry) => {
969
+ const part = binary.get(path);
970
+ return part === undefined
971
+ ? { op: 'create', path: `${root}/${path}`, content: await text(entry) }
972
+ : { op: 'upload', path: `${root}/${path}`, ...part };
973
+ };
974
+ const model = await modelOf(copy, after);
611
975
  const changes = [];
612
976
  for (const path of [...new Set([...before.keys(), ...after.keys()])].sort()) {
613
977
  if (arrived.has(path))
@@ -619,58 +983,89 @@ export class LocalCopies {
619
983
  if (to !== undefined) {
620
984
  changes.push({ op: 'rename', path: `${root}/${path}`, to: `${root}/${to}`, base: known });
621
985
  const moving = after.get(to);
986
+ const part = binary.get(to);
622
987
  if (moving !== prior) {
623
- changes.push({ op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) });
988
+ changes.push(part === undefined
989
+ ? { op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) }
990
+ : { op: 'upload', path: `${root}/${to}`, base: known, ...part });
624
991
  }
625
992
  continue;
626
993
  }
627
994
  if (prior === next)
628
995
  continue;
996
+ const part = binary.get(path);
629
997
  if (next === undefined)
630
998
  changes.push({ op: 'remove', path: `${root}/${path}`, base: known });
631
999
  else if (prior === undefined)
632
- changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next) });
1000
+ changes.push(await created(path, next));
1001
+ else if (heldBinary(path) !== (part !== undefined)) {
1002
+ changes.push({ op: 'remove', path: `${root}/${path}`, base: known }, await created(path, next));
1003
+ }
1004
+ else if (part !== undefined)
1005
+ changes.push({ op: 'upload', path: `${root}/${path}`, base: known, ...part });
633
1006
  else
634
- changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next)));
1007
+ changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next), model));
635
1008
  }
636
1009
  return changes;
637
1010
  }
638
1011
  async settle(start, caller, restaged = false) {
639
1012
  let copy = start;
640
- const push = copy.push;
641
- if (push.caller !== caller.key)
642
- return this.recovered(copy);
643
- let outcome;
644
- try {
645
- outcome = await caller.submit(push.operation_id, push.changes, push.upload, async (upload) => {
646
- copy = await this.save({ ...copy, push: { ...push, upload } });
647
- });
1013
+ if (copy.push.caller !== caller.key) {
1014
+ return this.recovered(copy, caller, async (current) => this.settle(await this.save({ ...current, push: { ...current.push, caller: caller.key } }), caller));
648
1015
  }
649
- catch (error) {
650
- const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
651
- return {
652
- refused: true,
653
- text: refusal?.text ?? `House's answer to ${short(push.commit)} did not arrive; run house push again`,
654
- };
1016
+ const finished = new Set();
1017
+ let outcome;
1018
+ for (;;) {
1019
+ const push = copy.push;
1020
+ try {
1021
+ outcome = await caller.submit(push.operation_id, push.changes, push.upload, async (upload) => {
1022
+ copy = await this.save({ ...copy, push: { ...copy.push, upload } });
1023
+ });
1024
+ }
1025
+ catch (error) {
1026
+ const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
1027
+ return {
1028
+ refused: true,
1029
+ text: refusal?.text ?? `House's answer to ${short(push.commit)} did not arrive; run house push again`,
1030
+ };
1031
+ }
1032
+ if (!('refused' in outcome) || outcome.refused.code !== 'files_unstaged')
1033
+ break;
1034
+ const named = unstagedUploads(outcome.refused).filter(({ id }) => !finished.has(id));
1035
+ if (named.length === 0)
1036
+ break;
1037
+ for (const { id } of named)
1038
+ finished.add(id);
1039
+ const failed = await this.sentUploads(copy, named, caller);
1040
+ copy = this.copies.get(copy.key);
1041
+ if (failed.size > 0)
1042
+ return this.stopped(copy, failed);
655
1043
  }
656
1044
  if ('refused' in outcome) {
1045
+ if (stopsPush(outcome.refused))
1046
+ return this.stopped(copy, firstLine(outcome.refused));
657
1047
  if (outcome.refused.code === 'upload_unavailable' && copy.push.upload !== undefined && !restaged) {
658
1048
  const { upload, ...unstaged } = copy.push;
659
1049
  return this.settle(await this.save({ ...copy, push: unstaged }), caller, true);
660
1050
  }
661
- if (outcome.refused.code === 'expired')
662
- return this.recovered(copy);
1051
+ if (outcome.refused.code === 'expired') {
1052
+ return this.recovered(copy, caller, async (current) => {
1053
+ await this.save(without(current));
1054
+ return { refused: true, text: `House does not hold ${short(current.push.commit)}; run house push again` };
1055
+ });
1056
+ }
663
1057
  let settled = await this.save(without(copy));
664
1058
  if (settled.unfollowed !== undefined)
665
1059
  return { refused: true, text: outcome.refused.text };
666
1060
  const rebased = rebaseFixes(outcome.refused);
667
1061
  try {
668
- settled = (await this.caughtUp([settled], true))[0];
1062
+ settled = (await this.caughtUp([settled], true, caller))[0];
1063
+ await this.save(await this.conflicted(settled, outcome.refused, copy.push, caller));
669
1064
  }
670
- catch {
1065
+ catch (error) {
671
1066
  return {
672
1067
  refused: true,
673
- text: rebased ? `${outcome.refused.text}\nHouse's current state did not arrive; push again later` : outcome.refused.text,
1068
+ text: rebased ? `${outcome.refused.text}\nHouse's current state did not arrive: ${houseCause(error)}` : outcome.refused.text,
674
1069
  };
675
1070
  }
676
1071
  return {
@@ -680,37 +1075,91 @@ export class LocalCopies {
680
1075
  : outcome.refused.text,
681
1076
  };
682
1077
  }
683
- return this.finished(await this.save({ ...copy, push: { ...copy.push, state: 'accepted', written: outcome.accepted } }));
1078
+ return this.finished(await this.save({ ...copy, push: { ...copy.push, state: 'accepted', written: outcome.accepted } }), caller);
1079
+ }
1080
+ async sentUploads(start, named, caller) {
1081
+ let copy = start;
1082
+ const parts = new Map(binaryParts(copy.push.changes).map((part) => [part.sha256, part]));
1083
+ const commit = await tree(copy.repository, copy.push.commit);
1084
+ const staged = new Set(copy.push.staged ?? []);
1085
+ const failed = new Map();
1086
+ let saving = Promise.resolve();
1087
+ await eachAtOnce(named, async (upload) => {
1088
+ const part = parts.get(upload.sha256);
1089
+ const path = relative(part.path);
1090
+ try {
1091
+ await sentBinary(copy.repository, commit.get(path), path, part, upload, (id, partSha256) => caller.finish(id, copy.room_ref, partSha256));
1092
+ }
1093
+ catch (error) {
1094
+ failed.set(upload.sha256, `its transfer failed: ${causeOf(error)}`);
1095
+ return;
1096
+ }
1097
+ staged.add(upload.sha256);
1098
+ saving = saving.then(async () => {
1099
+ copy = await this.save({ ...copy, push: { ...copy.push, staged: [...staged] } });
1100
+ });
1101
+ await saving;
1102
+ });
1103
+ return failed;
1104
+ }
1105
+ stopped(copy, reasons) {
1106
+ const push = copy.push;
1107
+ const staged = new Set(push.staged ?? []);
1108
+ const unsent = binaryParts(push.changes).filter(({ sha256 }) => !staged.has(sha256));
1109
+ const unchanged = `House holds nothing of ${short(push.commit)} yet, so ${rootOf(copy.room_handle)} is unchanged`;
1110
+ const again = 'house push again resumes this push and sends only what it has not sent';
1111
+ if (typeof reasons === 'string' && unsent.length === 0)
1112
+ return { refused: true, text: `${unchanged}: ${reasons}\n${again}` };
1113
+ const why = (sha256) => (typeof reasons === 'string' ? reasons : (reasons.get(sha256) ?? 'not sent yet'));
1114
+ return {
1115
+ refused: true,
1116
+ text: [`${unchanged}; these binary files are not sent:`, ...unsent.map(({ path, sha256 }) => `${relative(path)}: ${why(sha256)}`), again].join('\n'),
1117
+ };
1118
+ }
1119
+ async conflicted(copy, refusal, push, caller) {
1120
+ const root = rootOf(copy.room_handle);
1121
+ const touched = new Set(changedPaths(push.changes));
1122
+ const houseTree = await tree(copy.repository, copy.house);
1123
+ const paths = [
1124
+ ...new Set(refusal.conflicts.filter(({ code, path }) => code !== 'path_absent' && touched.has(path)).map(({ path }) => relative(path))),
1125
+ ].filter((path) => !houseTree.has(path));
1126
+ const files = new Map();
1127
+ const revisions = {};
1128
+ const binary = {};
1129
+ for (const path of paths) {
1130
+ const file = await caller.info(`${root}/${path}`);
1131
+ if (file === null)
1132
+ continue;
1133
+ files.set(path, { blob: await this.revisionBlob(copy, path, file.revision, caller, file) });
1134
+ revisions[path] = file.revision;
1135
+ binary[path] = { bytes: file.bytes, sha256: file.sha256 };
1136
+ }
1137
+ if (files.size === 0)
1138
+ return copy;
1139
+ const house = await importCommits(copy.repository, [
1140
+ { message: `House ${root}: the binary files ${short(push.commit)} conflicts with`, from: copy.house, files },
1141
+ ]);
1142
+ return { ...copy, house, manifests: { ...copy.manifests, [house]: manifest(copy.house, revisions, binary) } };
684
1143
  }
685
- async recovered(copy) {
1144
+ async recovered(copy, caller, unheld) {
686
1145
  const push = copy.push;
687
- const current = (await this.caughtUp([copy], true))[0];
1146
+ const current = (await this.caughtUp([copy], true, caller))[0];
688
1147
  const house = await tree(copy.repository, current.house);
689
- const root = rootOf(current.room_handle);
690
- if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit)))) {
691
- await this.save(without(current));
692
- return {
693
- refused: true,
694
- text: `House does not hold ${short(push.commit)}; run house push again`,
695
- };
1148
+ const model = await modelOf(current, house);
1149
+ if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit), model))) {
1150
+ return unheld(current);
696
1151
  }
697
1152
  const written = [];
698
- for (const path of changedPaths(push.changes).map(relative)) {
699
- const entry = house.get(path);
700
- written.push({
701
- path: `${root}/${path}`,
702
- revision: entry === undefined ? null : (revisionAt(current, current.house, path) ?? null),
703
- content: entry === undefined ? null : await blobText(copy.repository, entry),
704
- });
705
- }
706
- return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }));
1153
+ for (const path of changedPaths(push.changes).map(relative))
1154
+ written.push(await writtenAt(current, house, path));
1155
+ return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }), caller);
707
1156
  }
708
- async finished(start) {
1157
+ async finished(start, caller) {
709
1158
  let copy = start;
710
1159
  const { commit } = copy.push;
711
1160
  try {
712
1161
  if (copy.push.acknowledged === undefined)
713
- copy = await this.save(await this.acknowledged(copy));
1162
+ copy = await this.save(await this.acknowledged(copy, caller));
714
1163
  const integrated = await this.integrated(copy.push);
715
1164
  if (integrated.done)
716
1165
  await this.save(without(copy));
@@ -723,45 +1172,61 @@ export class LocalCopies {
723
1172
  };
724
1173
  }
725
1174
  }
726
- async acknowledged(start) {
1175
+ async acknowledged(start, caller) {
727
1176
  let copy = start;
728
1177
  const push = copy.push;
729
1178
  let written = push.written;
730
- if (push.changes.some((change) => change.op === 'rename')) {
731
- copy = (await this.caughtUp([copy], true))[0];
1179
+ const read = push.changes.some((change) => change.op === 'rename');
1180
+ if (read) {
1181
+ copy = (await this.caughtUp([copy], true, caller))[0];
732
1182
  const house = await tree(copy.repository, copy.house);
733
1183
  const sent = await tree(copy.repository, push.house);
734
- if (push.changes.some((change) => change.op === 'rename' && !house.has(relative(change.to)))) {
1184
+ const unread = push.changes.some((change) => change.op === 'rename' &&
1185
+ house.get(relative(change.path)) === sent.get(relative(change.path)) &&
1186
+ !house.has(relative(change.to)));
1187
+ if (unread)
735
1188
  throw new Error(`House's result for ${short(push.commit)} is not readable yet`);
736
- }
737
1189
  const known = new Set(written.map((entry) => relative(entry.path)));
738
1190
  for (const path of new Set([...sent.keys(), ...house.keys()])) {
739
- const entry = house.get(path);
740
- if (known.has(path) || sent.get(path) === entry)
741
- continue;
742
- written = [
743
- ...written,
744
- {
745
- path: `${rootOf(copy.room_handle)}/${path}`,
746
- revision: entry === undefined ? null : (revisionAt(copy, copy.house, path) ?? null),
747
- content: entry === undefined ? null : await blobText(copy.repository, entry),
748
- },
749
- ];
1191
+ if (!known.has(path) && sent.get(path) !== house.get(path))
1192
+ written = [...written, await writtenAt(copy, house, path)];
750
1193
  }
751
1194
  }
752
1195
  const selected = await tree(copy.repository, push.commit);
753
1196
  const houseTree = await tree(copy.repository, copy.house);
1197
+ const parts = pushedBinary(copy, push);
754
1198
  const files = new Map();
755
1199
  const revisions = {};
1200
+ const binary = {};
1201
+ const settled = new Map();
756
1202
  for (const entry of written) {
757
1203
  const path = relative(entry.path);
1204
+ if (entry.revision !== null || !settled.has(path))
1205
+ settled.set(path, entry);
1206
+ }
1207
+ for (const [path, entry] of settled) {
758
1208
  if (!selected.has(path) && !houseTree.has(path) && !declaredWritable(path, copy))
759
1209
  continue;
760
- files.set(path, entry.revision === null ? null : (entry.content ?? (await this.revisionText(entry.path, entry.revision))));
761
1210
  revisions[path] = entry.revision;
1211
+ const part = parts.get(path);
1212
+ const kept = heldAt(copy, copy.house, path);
1213
+ if (entry.revision === null)
1214
+ files.set(path, null);
1215
+ else if (part !== undefined) {
1216
+ files.set(path, { blob: selected.get(path) });
1217
+ binary[path] = part;
1218
+ }
1219
+ else if (kept !== undefined && revisionAt(copy, copy.house, path) === entry.revision) {
1220
+ files.set(path, { blob: houseTree.get(path) });
1221
+ binary[path] = kept;
1222
+ }
1223
+ else
1224
+ files.set(path, entry.content ?? (await this.revisionText(entry.path, entry.revision)));
762
1225
  }
763
1226
  const edits = new Map([...files].filter(([path, content]) => (content === null ? selected.has(path) : selected.get(path) !== blobId(content))));
764
- const received = [...files.keys()].filter((path) => revisionAt(copy, copy.house, path) === revisionAt(copy, push.house, path));
1227
+ const received = read
1228
+ ? []
1229
+ : [...files.keys()].filter((path) => revisionAt(copy, copy.house, path) === revisionAt(copy, push.house, path));
765
1230
  const accepted = `House ${rootOf(copy.room_handle)} with ${push.commit} accepted`;
766
1231
  const house = await importCommits(copy.repository, [
767
1232
  ...(edits.size === 0 ? [] : [{ message: accepted, from: push.commit, files: edits }]),
@@ -779,30 +1244,38 @@ export class LocalCopies {
779
1244
  house,
780
1245
  manifests: {
781
1246
  ...copy.manifests,
782
- [acknowledged]: { parent: push.base, files: revisions },
783
- [house]: { parent: copy.house, files: Object.fromEntries(received.map((path) => [path, revisions[path] ?? null])) },
1247
+ [acknowledged]: manifest(push.base, revisions, binary),
1248
+ [house]: manifest(copy.house, Object.fromEntries(received.map((path) => [path, revisions[path] ?? null])), Object.fromEntries(received.flatMap((path) => (binary[path] === undefined ? [] : [[path, binary[path]]])))),
784
1249
  },
785
1250
  moves: copy.moves.filter(({ from, to }) => !(after.has(to) && !after.has(from))),
786
1251
  push: { ...push, acknowledged },
787
1252
  };
788
1253
  }
789
- async landed(repository, changes, house, selected) {
790
- const text = async (path) => {
791
- const entry = house.get(relative(path));
792
- return entry === undefined ? undefined : blobText(repository, entry);
793
- };
1254
+ async landed(repository, changes, house, selected, model) {
1255
+ const created = new Set(changes.flatMap((change) => (change.op === 'create' || change.op === 'upload' ? [change.path] : [])));
794
1256
  for (const change of changes) {
795
- const current = await text(change.path);
796
- if (change.op === 'remove' && current === undefined)
797
- continue;
798
- if (change.op === 'rename' && current === undefined && house.get(relative(change.to)) === selected.get(relative(change.to))) {
799
- continue;
1257
+ const path = relative(change.path);
1258
+ if (change.op === 'upload') {
1259
+ if (house.get(path) === selected.get(path))
1260
+ continue;
1261
+ return false;
800
1262
  }
1263
+ if (change.op === 'remove') {
1264
+ if (created.has(change.path) || !house.has(path))
1265
+ continue;
1266
+ return false;
1267
+ }
1268
+ if (change.op === 'rename') {
1269
+ if (!house.has(path) && house.get(relative(change.to)) === selected.get(relative(change.to)))
1270
+ continue;
1271
+ return false;
1272
+ }
1273
+ const entry = house.get(path);
1274
+ const current = entry === undefined ? undefined : await blobText(repository, entry);
801
1275
  if ((change.op === 'create' || change.op === 'replace') && current === change.content)
802
1276
  continue;
803
- const parsed = current === undefined ? null : parse(current, change.path);
804
- const document = parsed?.ok === true ? parsed.document : null;
805
- if (document === null)
1277
+ const document = current === undefined ? undefined : readDocument(current, path, model)?.document;
1278
+ if (document === undefined)
806
1279
  return false;
807
1280
  if (change.op === 'set_field' && JSON.stringify(document.fields[change.field]) === JSON.stringify(change.value))
808
1281
  continue;
@@ -815,7 +1288,7 @@ export class LocalCopies {
815
1288
  return true;
816
1289
  }
817
1290
  async revisionText(path, wanted) {
818
- const page = await this.house.post('/kit/door/revisions', {
1291
+ const page = await this.house.door('files/revisions/list', {
819
1292
  path,
820
1293
  revision: wanted,
821
1294
  });