@agentshouse/kit 0.2.1-cfb7b543f2af6 → 0.2.1-cfbadfb5b5e28

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +33 -34
  2. package/dist/acp.js +91 -20
  3. package/dist/afk.js +17 -0
  4. package/dist/agents.js +91 -44
  5. package/dist/api.js +5 -5
  6. package/dist/app-git.js +10 -26
  7. package/dist/authority-main.js +3 -3
  8. package/dist/bridge.js +7 -10
  9. package/dist/building.js +50 -0
  10. package/dist/calls.js +71 -0
  11. package/dist/chunks.js +49 -0
  12. package/dist/clis.js +56 -45
  13. package/dist/connection.js +120 -0
  14. package/dist/content.js +63 -0
  15. package/dist/conversations.js +703 -244
  16. package/dist/copies.js +344 -171
  17. package/dist/end.js +15 -4
  18. package/dist/events.js +746 -210
  19. package/dist/files.js +139 -37
  20. package/dist/git.js +49 -16
  21. package/dist/grok.js +39 -0
  22. package/dist/home.js +27 -20
  23. package/dist/house-cli.js +98 -90
  24. package/dist/house-main.js +3 -3
  25. package/dist/instructions.js +29 -10
  26. package/dist/kit-cli.js +5 -8
  27. package/dist/kit-main.js +3 -3
  28. package/dist/login.js +20 -11
  29. package/dist/logout.js +1 -3
  30. package/dist/mcp.js +6 -2
  31. package/dist/placement.js +19 -36
  32. package/dist/relay.js +21 -0
  33. package/dist/resident.js +118 -52
  34. package/dist/scope.js +15 -4
  35. package/dist/scripts.js +255 -0
  36. package/dist/secret-input.js +1 -1
  37. package/dist/service.js +23 -1
  38. package/dist/sign-in.js +22 -39
  39. package/dist/silence.js +28 -0
  40. package/dist/skills.js +42 -97
  41. package/dist/transcription.js +88 -45
  42. package/dist/translate.js +19 -16
  43. package/dist/update.js +113 -127
  44. package/dist/wire.js +37 -3
  45. package/dist/work.js +12 -7
  46. package/package.json +22 -21
  47. package/dist/bootstrap.json +0 -24
  48. package/dist/chatgpt.js +0 -32
  49. package/dist/clis-main.js +0 -23
  50. package/dist/parts.js +0 -82
  51. package/dist/stream.js +0 -123
  52. package/how-we-work.md +0 -33
  53. package/skills/ask-matt/LICENSE +0 -21
  54. package/skills/ask-matt/PHASE-BOUNDARIES.md +0 -55
  55. package/skills/ask-matt/SKILL.md +0 -97
  56. package/skills/ask-matt/agents/openai.yaml +0 -5
  57. package/skills/code-review/LICENSE +0 -21
  58. package/skills/code-review/SKILL.md +0 -89
  59. package/skills/code-review/agents/openai.yaml +0 -3
  60. package/skills/codebase-design/DEEPENING.md +0 -37
  61. package/skills/codebase-design/DESIGN-IT-TWICE.md +0 -44
  62. package/skills/codebase-design/LICENSE +0 -21
  63. package/skills/codebase-design/SKILL.md +0 -114
  64. package/skills/codebase-design/agents/openai.yaml +0 -3
  65. package/skills/diagnosing-bugs/LICENSE +0 -21
  66. package/skills/diagnosing-bugs/SKILL.md +0 -138
  67. package/skills/diagnosing-bugs/agents/openai.yaml +0 -3
  68. package/skills/diagnosing-bugs/scripts/hitl-loop.template.sh +0 -44
  69. package/skills/domain-modeling/ADR-FORMAT.md +0 -47
  70. package/skills/domain-modeling/GLOSSARY-FORMAT.md +0 -60
  71. package/skills/domain-modeling/LICENSE +0 -21
  72. package/skills/domain-modeling/SKILL.md +0 -74
  73. package/skills/domain-modeling/agents/openai.yaml +0 -3
  74. package/skills/grill-me/LICENSE +0 -21
  75. package/skills/grill-me/SKILL.md +0 -7
  76. package/skills/grill-me/agents/openai.yaml +0 -5
  77. package/skills/grill-with-docs/LICENSE +0 -21
  78. package/skills/grill-with-docs/SKILL.md +0 -7
  79. package/skills/grill-with-docs/agents/openai.yaml +0 -5
  80. package/skills/grilling/LICENSE +0 -21
  81. package/skills/grilling/SKILL.md +0 -30
  82. package/skills/grilling/agents/openai.yaml +0 -3
  83. package/skills/handoff/LICENSE +0 -21
  84. package/skills/handoff/SKILL.md +0 -16
  85. package/skills/handoff/agents/openai.yaml +0 -5
  86. package/skills/implement/LICENSE +0 -21
  87. package/skills/implement/SKILL.md +0 -17
  88. package/skills/implement/agents/openai.yaml +0 -5
  89. package/skills/implement-spec/LICENSE +0 -21
  90. package/skills/implement-spec/SKILL.md +0 -40
  91. package/skills/implement-spec/agents/openai.yaml +0 -5
  92. package/skills/improve-codebase-architecture/HTML-REPORT.md +0 -123
  93. package/skills/improve-codebase-architecture/LICENSE +0 -21
  94. package/skills/improve-codebase-architecture/SKILL.md +0 -71
  95. package/skills/improve-codebase-architecture/agents/openai.yaml +0 -5
  96. package/skills/pr/CREDITS.md +0 -3
  97. package/skills/pr/LICENSE +0 -21
  98. package/skills/pr/SKILL.md +0 -170
  99. package/skills/pr/agents/openai.yaml +0 -3
  100. package/skills/prototype/LICENSE +0 -21
  101. package/skills/prototype/LOGIC.md +0 -67
  102. package/skills/prototype/SKILL.md +0 -26
  103. package/skills/prototype/UI.md +0 -112
  104. package/skills/prototype/agents/openai.yaml +0 -3
  105. package/skills/research/LICENSE +0 -21
  106. package/skills/research/SKILL.md +0 -12
  107. package/skills/research/agents/openai.yaml +0 -3
  108. package/skills/retro/LICENSE +0 -21
  109. package/skills/retro/SKILL.md +0 -44
  110. package/skills/retro/agents/openai.yaml +0 -5
  111. package/skills/tdd/LICENSE +0 -21
  112. package/skills/tdd/SKILL.md +0 -38
  113. package/skills/tdd/agents/openai.yaml +0 -3
  114. package/skills/tdd/mocking.md +0 -59
  115. package/skills/tdd/tests.md +0 -77
  116. package/skills/teach/GLOSSARY-FORMAT.md +0 -35
  117. package/skills/teach/LEARNING-RECORD-FORMAT.md +0 -46
  118. package/skills/teach/LICENSE +0 -21
  119. package/skills/teach/MISSION-FORMAT.md +0 -31
  120. package/skills/teach/RESOURCES-FORMAT.md +0 -32
  121. package/skills/teach/SKILL.md +0 -140
  122. package/skills/teach/agents/openai.yaml +0 -5
  123. package/skills/to-questionnaire/LICENSE +0 -21
  124. package/skills/to-questionnaire/SKILL.md +0 -54
  125. package/skills/to-questionnaire/agents/openai.yaml +0 -5
  126. package/skills/to-spec/LICENSE +0 -21
  127. package/skills/to-spec/SKILL.md +0 -75
  128. package/skills/to-spec/agents/openai.yaml +0 -5
  129. package/skills/to-tickets/LICENSE +0 -21
  130. package/skills/to-tickets/SKILL.md +0 -105
  131. package/skills/to-tickets/agents/openai.yaml +0 -5
  132. package/skills/triage/AGENT-BRIEF.md +0 -207
  133. package/skills/triage/LICENSE +0 -21
  134. package/skills/triage/OUT-OF-SCOPE.md +0 -105
  135. package/skills/triage/SKILL.md +0 -112
  136. package/skills/triage/agents/openai.yaml +0 -5
  137. package/skills/wait-what/LICENSE +0 -21
  138. package/skills/wait-what/SKILL.md +0 -7
  139. package/skills/wait-what/agents/openai.yaml +0 -5
  140. package/skills/wayfinder/LICENSE +0 -21
  141. package/skills/wayfinder/SKILL.md +0 -128
  142. package/skills/wayfinder/agents/openai.yaml +0 -5
  143. package/skills/wizard/LICENSE +0 -21
  144. package/skills/wizard/SKILL.md +0 -44
  145. package/skills/wizard/agents/openai.yaml +0 -3
  146. package/skills/wizard/template.sh +0 -223
  147. package/skills/writing-for-agents/LICENSE +0 -21
  148. package/skills/writing-for-agents/SKILL-MECHANICS.md +0 -22
  149. package/skills/writing-for-agents/SKILL.md +0 -81
  150. package/skills/writing-for-agents/agents/openai.yaml +0 -3
package/dist/copies.js CHANGED
@@ -1,20 +1,38 @@
1
- import { parse } from '@agentshouse/mdmodel';
2
- import { randomUUID } from 'node:crypto';
1
+ import { HOUSE_ROOM_TYPES, readTypes } from '@agentshouse/mdmodel';
2
+ import { createHash, randomUUID } from 'node:crypto';
3
3
  import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
4
4
  import { dirname, join } from 'node:path';
5
5
  import { finalRefusal, HouseRefusal, retrying } from './api.js';
6
+ import { chunkTexts, stagedChunks } from './chunks.js';
6
7
  import { blobId, blobText, git, isText, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
8
+ import { paused } from './files.js';
7
9
  import { kitHome } from './home.js';
8
10
  import { operationId } from './operation.js';
9
11
  import { refusalOf } from './refusals.js';
10
- import { changed } from './translate.js';
11
- import { relayed } from './relay.js';
12
+ import { changed, readDocument } from './translate.js';
12
13
  const BRANCH = 'main';
13
14
  const PROVENANCE = '_provenance';
14
- const IDENTITY = { GIT_COMMITTER_NAME: 'House Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
15
+ const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
16
+ const ROOMS = 'rooms/';
17
+ const ENUMERATE = '/kit/door/changes/enumerate';
18
+ const NOT_READY = 'not ready yet; Kit copies it once House has built it';
19
+ const STAGED_CHUNKS = 'house-chunks';
20
+ const SCHEMA = 'settings/schema.yaml';
21
+ // A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
22
+ const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
23
+ class NotReady extends Error {
24
+ rooms;
25
+ constructor(rooms) {
26
+ super(NOT_READY);
27
+ this.rooms = rooms;
28
+ }
29
+ }
15
30
  function causeOf(error) {
16
31
  return error instanceof Error ? error.message : String(error);
17
32
  }
33
+ function houseCause(error) {
34
+ return error instanceof HouseRefusal ? (refusalOf(error.text)?.text ?? causeOf(error)) : causeOf(error);
35
+ }
18
36
  function logged(error) {
19
37
  process.stderr.write(`kit: ${causeOf(error)}\n`);
20
38
  }
@@ -33,6 +51,9 @@ function textOnly(paths) {
33
51
  const [named, them] = paths.length === 1 ? [`${paths[0]} is`, 'it'] : [`${paths.join(', ')} are`, 'them'];
34
52
  return `${named} not text, and house push sends text only; remove ${them} from the commit and place ${them} with house upload`;
35
53
  }
54
+ function outside(path) {
55
+ return { refused: true, text: `${path} is not a House Local copy; run house push inside one` };
56
+ }
36
57
  function short(commit) {
37
58
  return commit.slice(0, 12);
38
59
  }
@@ -52,6 +73,9 @@ function expiredAuthority(error) {
52
73
  return undefined;
53
74
  }
54
75
  }
76
+ function grouped(items, size) {
77
+ return items.flatMap((_, index) => (index % size === 0 ? [items.slice(index, index + size)] : []));
78
+ }
55
79
  function covers(path, prefix) {
56
80
  return prefix === '' || path === prefix || path.startsWith(`${prefix}/`);
57
81
  }
@@ -60,6 +84,12 @@ function sidecar(path) {
60
84
  segments.splice(-1, 0, PROVENANCE);
61
85
  return segments.join('/');
62
86
  }
87
+ function recordOf(key) {
88
+ return join(kitHome(), 'local-copies', `${key}.json`);
89
+ }
90
+ function stagingPrefix() {
91
+ return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
92
+ }
63
93
  function rootOf(handle) {
64
94
  return `/rooms/${handle}`;
65
95
  }
@@ -87,32 +117,59 @@ async function removeFile(repository, path) {
87
117
  return;
88
118
  }
89
119
  }
90
- async function readOnlyPlaced(repository, held, written, removed) {
120
+ async function readOnlyFiles(repository, held) {
91
121
  const tracked = new Set((await git(repository, ['ls-files', '-z'])).split('\0'));
92
122
  const holding = new Set(held);
93
- for (const path of removed) {
94
- if (!holding.has(path))
95
- continue;
96
- if (!tracked.has(path))
97
- await removeFile(repository, path);
98
- holding.delete(path);
99
- }
100
- for (const [path, content] of written) {
101
- if (tracked.has(path))
102
- continue;
103
- if (!holding.has(path) && (await exists(join(repository, path))))
104
- continue;
105
- await writeText(join(repository, path), content);
106
- holding.add(path);
107
- }
108
- return [...holding].sort();
123
+ return {
124
+ remove: async (path) => {
125
+ if (!holding.has(path))
126
+ return;
127
+ if (!tracked.has(path))
128
+ await removeFile(repository, path);
129
+ holding.delete(path);
130
+ },
131
+ place: async (path, content) => {
132
+ if (tracked.has(path))
133
+ return;
134
+ if (!holding.has(path) && (await exists(join(repository, path))))
135
+ return;
136
+ await writeText(join(repository, path), content);
137
+ holding.add(path);
138
+ },
139
+ held: () => [...holding].sort(),
140
+ };
141
+ }
142
+ async function readOnlyPlaced(repository, held, written, removed) {
143
+ const files = await readOnlyFiles(repository, held);
144
+ for (const path of removed)
145
+ await files.remove(path);
146
+ for (const [path, content] of written)
147
+ await files.place(path, content);
148
+ return files.held();
149
+ }
150
+ async function importedWhole(repository, scope, texts, commit, readOnly) {
151
+ const revisions = {};
152
+ async function* tracked() {
153
+ for await (const { path, content, revision: held, provenance } of texts) {
154
+ if (declaredWritable(path, scope)) {
155
+ revisions[path] = held;
156
+ yield [path, content];
157
+ }
158
+ else {
159
+ await readOnly.place(path, content);
160
+ }
161
+ if (provenance !== undefined)
162
+ await readOnly.place(sidecar(path), provenance);
163
+ }
164
+ }
165
+ return { house: await importCommits(repository, [{ ...commit, files: tracked() }]), revisions };
109
166
  }
110
167
  async function ruled(path, rules) {
111
168
  if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
112
169
  await writeFile(path, rules);
113
170
  }
114
171
  async function ignoreRules(repository, writable, copy) {
115
- const readOnly = ('read_only' in copy ? copy.read_only : [...copy.readOnly.keys()]).filter((path) => !path.split('/').includes(PROVENANCE));
172
+ const readOnly = copy.read_only.filter((path) => !path.split('/').includes(PROVENANCE));
116
173
  const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
117
174
  const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
118
175
  const house = [
@@ -121,7 +178,7 @@ async function ignoreRules(repository, writable, copy) {
121
178
  ].map(pattern);
122
179
  const granted = readOnly.filter((path) => !protectedPath(path)).map(pattern);
123
180
  await ruled(join(repository, '.gitignore'), [
124
- '# House Kit writes this file and .ignore. The paths below are read-only House',
181
+ '# Kit writes this file and .ignore. The paths below are read-only House',
125
182
  '# content: Git does not track them, and House refuses a change to them even',
126
183
  '# when this file is edited or a path is force-added.',
127
184
  '/.gitignore',
@@ -136,13 +193,19 @@ async function ignoreRules(repository, writable, copy) {
136
193
  '',
137
194
  ].join('\n'));
138
195
  await ruled(join(repository, '.ignore'), [
139
- '# House Kit writes this file so native search still reads the read-only House',
196
+ '# Kit writes this file so native search still reads the read-only House',
140
197
  '# content that .gitignore keeps out of Git.',
141
198
  `!${PROVENANCE}`,
142
199
  ...[...house, ...granted].map((rule) => `!${rule}`),
143
200
  '',
144
201
  ].join('\n'));
145
202
  }
203
+ async function modelOf(copy, files) {
204
+ const entry = files.get(SCHEMA);
205
+ if (entry !== undefined)
206
+ return readTypes(await blobText(copy.repository, entry), HOUSE_ROOM_TYPES);
207
+ return readTypes(copy.read_only.includes(SCHEMA) ? await readFile(join(copy.repository, SCHEMA), 'utf8') : '', HOUSE_ROOM_TYPES);
208
+ }
146
209
  function revisionAt(copy, commit, path) {
147
210
  let manifest = copy.manifests[commit];
148
211
  while (manifest !== undefined) {
@@ -172,6 +235,7 @@ export class LocalCopies {
172
235
  house;
173
236
  copies = new Map();
174
237
  queues = new Map();
238
+ waiting = new Set();
175
239
  answering = Promise.resolve();
176
240
  constructor(house) {
177
241
  this.house = house;
@@ -179,7 +243,7 @@ export class LocalCopies {
179
243
  key: 'owner',
180
244
  submit: async (operation, changes) => {
181
245
  try {
182
- const outcome = await retrying(() => house.post('/kit/door/batch', { operation_id: operation, changes }));
246
+ const outcome = await retrying(() => house.post('/kit/door/files/batch', { operation_id: operation, changes }));
183
247
  return { accepted: outcome.contents };
184
248
  }
185
249
  catch (error) {
@@ -203,70 +267,116 @@ export class LocalCopies {
203
267
  this.copies.set(copy.key, copy);
204
268
  }
205
269
  }
206
- async mapping() {
207
- await this.answering;
208
- return [...this.copies.values()]
209
- .filter((copy) => copy.active)
210
- .map((copy) => `${rootOf(copy.room_handle)}: ${copy.repository}`)
211
- .sort();
270
+ async swept() {
271
+ const rooms = join(localCopies(), 'rooms');
272
+ for (const name of await readdir(rooms).catch(() => [])) {
273
+ if (name.startsWith(stagingPrefix()))
274
+ await rm(join(rooms, name), { recursive: true, force: true });
275
+ }
212
276
  }
213
- async connected() {
214
- const copied = await this.answered(() => this.fetched(false));
215
- if (copied.refused)
216
- logged(copied.text);
277
+ connected() {
278
+ return this.selected();
217
279
  }
218
280
  fetch() {
219
281
  return this.answered(() => this.fetched(true));
220
282
  }
283
+ async selected() {
284
+ const copied = await this.answered(() => this.fetched(false));
285
+ if (copied.refused)
286
+ logged(copied.text);
287
+ }
221
288
  answered(work) {
222
289
  const answered = this.answering.then(work);
223
290
  this.answering = answered.catch(() => undefined);
224
291
  return answered;
225
292
  }
226
293
  async fetched(refreshing) {
227
- const { local_copy: answer } = await this.house.post('/kit/local-copy/selection', {});
228
- const synced = new Map(answer.rooms.map((room) => [`rooms/${room.room_ref}`, room.room_handle]));
294
+ const { local_copy: answer } = await this.house.post('/kit/local-copies/selection/read', {});
295
+ const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
229
296
  return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
230
297
  const said = [];
231
298
  let refused = false;
232
- for (const copy of this.copies.values()) {
233
- if (synced.has(copy.key) || !copy.active)
299
+ const mark = refreshing ? 'said' : 'unsaid';
300
+ for (const copy of [...this.copies.values()]) {
301
+ if (synced.has(copy.key) || copy.unfollowed === 'said' || copy.unfollowed === mark)
234
302
  continue;
235
- await this.save({ ...copy, active: false });
236
- said.push(`${rootOf(copy.room_handle)}: no longer synced to this Environment; its files stay`);
303
+ await this.save({ ...copy, unfollowed: mark });
304
+ if (refreshing)
305
+ said.push(`${rootOf(copy.room_handle)}: no longer followed; kept as it is`);
237
306
  }
238
307
  const current = [];
308
+ const wanted = new Map();
239
309
  for (const [key, handle] of synced) {
240
310
  try {
241
311
  const held = this.copies.get(key);
242
312
  if (held === undefined || !(await exists(held.repository))) {
243
- said.push(await this.materialize(key, handle));
313
+ wanted.set(key, handle);
244
314
  continue;
245
315
  }
246
- const copy = held.room_handle === handle ? held : await this.save({ ...held, room_handle: handle });
247
- if (copy.active)
248
- current.push(copy);
249
- else
250
- said.push(...moved(copy, await this.save(await this.received({ ...copy, active: true }))));
316
+ const { unfollowed, ...followed } = held;
317
+ current.push(unfollowed === undefined && held.room_handle === handle ? held : await this.save({ ...followed, room_handle: handle }));
251
318
  }
252
319
  catch (error) {
253
320
  refused = true;
254
321
  said.push(`${rootOf(handle)}: ${causeOf(error)}`);
255
322
  }
256
323
  }
257
- if (refreshing && current.length > 0) {
258
- try {
259
- const refreshed = await this.refreshed(current);
260
- current.forEach((copy, index) => said.push(...moved(copy, refreshed[index])));
324
+ const unready = [];
325
+ const unsettled = [];
326
+ if (wanted.size > 0) {
327
+ const { authorities } = await this.discovered([...wanted.values()]);
328
+ for (const [key, handle] of wanted) {
329
+ const authority = authorities.find((entry) => `${ROOMS}${entry.room_ref}` === key);
330
+ try {
331
+ if (authority === undefined)
332
+ throw new Error('House serves nothing of it to copy');
333
+ said.push(await this.copied({ key, handle, authority }));
334
+ }
335
+ catch (error) {
336
+ if (error instanceof NotReady) {
337
+ unready.push(...error.rooms);
338
+ said.push(`${rootOf(handle)}: ${NOT_READY}`);
339
+ }
340
+ else {
341
+ refused = true;
342
+ unsettled.push(key.slice(ROOMS.length));
343
+ said.push(`${rootOf(handle)}: ${causeOf(error)}`);
344
+ }
345
+ }
261
346
  }
262
- catch (error) {
347
+ }
348
+ const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
349
+ if (reading.length > 0) {
350
+ const failed = await this.refreshed(reading, unready).then(() => null, houseCause);
351
+ for (const copy of reading) {
352
+ said.push(...(unready.includes(copy.room_ref) ? [`${rootOf(copy.room_handle)}: ${NOT_READY}`] : moved(copy, this.copies.get(copy.key))));
353
+ }
354
+ if (failed !== null) {
263
355
  refused = true;
264
- said.push(error instanceof HouseRefusal ? (refusalOf(error.text)?.text ?? causeOf(error)) : causeOf(error));
356
+ unsettled.push(...reading.map((copy) => copy.room_ref));
357
+ said.push(failed);
265
358
  }
266
359
  }
360
+ for (const room of [...this.waiting]) {
361
+ if (!unready.includes(room) && !unsettled.includes(room))
362
+ this.waiting.delete(room);
363
+ }
364
+ const due = unready.filter((room) => !this.waiting.has(room));
365
+ for (const room of due)
366
+ this.waiting.add(room);
367
+ if (due.length > 0)
368
+ void this.askedAgain(due);
267
369
  return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
268
370
  });
269
371
  }
372
+ async askedAgain(rooms) {
373
+ let asking = rooms;
374
+ for (let step = 0; asking.length > 0; step++) {
375
+ await paused(ASKED_AGAIN_MS[Math.min(step, ASKED_AGAIN_MS.length - 1)]);
376
+ await this.selected().catch(logged);
377
+ asking = asking.filter((room) => this.waiting.has(room));
378
+ }
379
+ }
270
380
  async observed(report) {
271
381
  const key = await this.keyAt(dirname(report.common));
272
382
  if (key === undefined)
@@ -295,11 +405,11 @@ export class LocalCopies {
295
405
  async push(cwd, selector, caller) {
296
406
  const located = await run(cwd, ['rev-parse', '--path-format=absolute', '--git-common-dir', '--show-toplevel']);
297
407
  if (located.status !== 0)
298
- return { refused: true, text: `${cwd} is not a House Local copy; run house push inside one` };
408
+ return outside(cwd);
299
409
  const [common, worktree] = located.stdout.toString('utf8').split('\n');
300
410
  const key = await this.keyAt(dirname(common));
301
411
  if (key === undefined)
302
- return { refused: true, text: `${worktree} is not a House Local copy; run house push inside one` };
412
+ return outside(worktree);
303
413
  return this.exclusive(key, () => this.pushed(key, worktree, selector, caller));
304
414
  }
305
415
  async keyAt(repository) {
@@ -320,7 +430,7 @@ export class LocalCopies {
320
430
  return key === undefined ? work() : this.exclusive(key, () => this.holding(rest, work));
321
431
  }
322
432
  async save(copy) {
323
- const path = join(kitHome(), 'local-copies', `${copy.key}.json`);
433
+ const path = recordOf(copy.key);
324
434
  await mkdir(dirname(path), { recursive: true, mode: 0o700 });
325
435
  const temporary = `${path}.${process.pid}`;
326
436
  await writeFile(temporary, JSON.stringify(copy), { mode: 0o600 });
@@ -328,128 +438,170 @@ export class LocalCopies {
328
438
  this.copies.set(copy.key, copy);
329
439
  return copy;
330
440
  }
331
- async read(handle) {
332
- const prefixes = [rootOf(handle)];
333
- const discovery = await this.house.post('/kit/door/discover', { prefixes });
334
- let bundle = await this.house.post('/kit/door/bootstrap', { prefixes });
335
- if ('transfer' in bundle)
336
- bundle = (await (await fetch(relayed(bundle.transfer.url))).json());
337
- const [view] = bundle.scopes;
338
- if (view === undefined)
339
- return null;
340
- const authority = discovery.authorities.find((entry) => entry.room_ref === view.manifest.anchor);
341
- const contents = new Map(view.contents.map((entry) => [entry.path, entry.content]));
342
- const read = {
343
- anchor: view.manifest.anchor,
344
- position: view.manifest.position,
345
- log_epoch: view.manifest.log_epoch,
346
- write: authority.operations.write,
347
- protected: authority.protected_paths,
348
- writable: new Map(),
349
- revisions: {},
350
- readOnly: new Map(),
351
- };
352
- for (const file of view.manifest.files) {
353
- const content = contents.get(file.path);
354
- if (file.writable) {
355
- read.writable.set(file.path, content);
356
- read.revisions[file.path] = file.revision;
357
- }
358
- else {
359
- read.readOnly.set(file.path, content);
441
+ async discovered(handles) {
442
+ const discover = (group) => this.house.post('/kit/door/rooms/discover', { prefixes: group.map(rootOf) });
443
+ const [first, ...rest] = handles;
444
+ const { bounds, authorities } = await discover([first]);
445
+ for (const group of grouped(rest, bounds.max_prefixes))
446
+ authorities.push(...(await discover(group)).authorities);
447
+ return { bounds, authorities };
448
+ }
449
+ async chunked(root, room, directory) {
450
+ await mkdir(directory, { recursive: true });
451
+ for (let renewed = false;; renewed = true) {
452
+ const { rooms } = await this.house.post('/kit/door/room-chunks/list', { prefixes: [root] });
453
+ const list = rooms.find((entry) => entry.room_ref === room);
454
+ if (list === undefined)
455
+ throw new Error('House serves nothing of it to copy');
456
+ if (!list.ready)
457
+ throw new NotReady([room]);
458
+ const held = new Set(await readdir(directory));
459
+ const failure = await stagedChunks(directory, list.chunks.filter(({ hash }) => !held.has(hash)));
460
+ if (failure === undefined) {
461
+ return { position: list.position, log_epoch: list.log_epoch, texts: chunkTexts(directory, list.chunks, list.models) };
360
462
  }
361
- if (file.provenance !== undefined)
362
- read.readOnly.set(sidecar(file.path), file.provenance);
463
+ if (renewed && (await readdir(directory)).length === held.size)
464
+ throw failure;
363
465
  }
364
- return read;
365
466
  }
366
- async materialize(key, handle) {
367
- const read = await this.read(handle);
368
- if (read === null)
369
- throw new Error('House serves nothing of it to copy');
370
- const repository = join(localCopies(), 'rooms', read.anchor);
467
+ async materialized({ key, handle, authority }, whole) {
468
+ const repository = join(localCopies(), 'rooms', authority.room_ref);
371
469
  if (await exists(repository))
372
- throw new Error(`${repository} already exists and House Kit leaves it as it is`);
373
- const staging = join(dirname(repository), `.${randomUUID()}`);
470
+ throw new Error(`${repository} already exists and Kit leaves it as it is`);
471
+ const staging = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
472
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
374
473
  await mkdir(staging, { recursive: true });
375
- let house;
376
- let readOnly;
474
+ let copy;
377
475
  try {
378
476
  await git(staging, ['init', '-q', '-b', BRANCH]);
379
- house = await importCommits(staging, [
380
- { message: `House ${rootOf(handle)}`, from: null, files: read.writable },
381
- ]);
477
+ const read = await whole(join(staging, '.git', STAGED_CHUNKS));
478
+ const readOnly = await readOnlyFiles(staging, []);
479
+ const { house, revisions } = await importedWhole(staging, scope, read.texts, { message: `House ${rootOf(handle)}`, from: null }, readOnly);
382
480
  await git(staging, ['update-ref', `refs/heads/${BRANCH}`, house]);
383
481
  await git(staging, ['reset', '-q', '--hard']);
384
- readOnly = await readOnlyPlaced(staging, [], read.readOnly, []);
385
- await ignoreRules(staging, [...read.writable.keys()], read);
482
+ copy = {
483
+ key,
484
+ room_ref: authority.room_ref,
485
+ room_handle: handle,
486
+ repository,
487
+ house,
488
+ position: read.position,
489
+ log_epoch: read.log_epoch,
490
+ ...scope,
491
+ read_only: readOnly.held(),
492
+ manifests: { [house]: { parent: null, files: revisions } },
493
+ moves: [],
494
+ };
495
+ await ignoreRules(staging, Object.keys(revisions), copy);
496
+ await rm(join(staging, '.git', STAGED_CHUNKS), { recursive: true, force: true });
386
497
  await rename(staging, repository);
387
498
  }
388
499
  catch (error) {
389
500
  await rm(staging, { recursive: true, force: true });
390
501
  throw error;
391
502
  }
392
- await this.save({
393
- key,
394
- room_ref: read.anchor,
395
- room_handle: handle,
396
- repository,
397
- active: true,
398
- house,
399
- position: read.position,
400
- log_epoch: read.log_epoch,
401
- write: read.write,
402
- protected: read.protected,
403
- read_only: readOnly,
404
- manifests: { [house]: { parent: null, files: read.revisions } },
405
- moves: [],
406
- });
407
- return `${rootOf(handle)}: copied to ${repository}`;
503
+ return this.save(copy);
408
504
  }
409
- async received(copy) {
410
- const read = await this.read(copy.room_handle);
411
- if (read === null)
412
- return copy;
413
- const house = await importCommits(copy.repository, [
414
- { message: `House ${rootOf(copy.room_handle)} at position ${read.position}`, from: copy.house, replaces: true, files: read.writable },
415
- ]);
416
- const removed = copy.read_only.filter((path) => !read.readOnly.has(path));
417
- const readOnly = await readOnlyPlaced(copy.repository, copy.read_only, read.readOnly, removed);
418
- const next = {
419
- ...copy,
420
- house,
421
- position: read.position,
422
- log_epoch: read.log_epoch,
423
- write: read.write,
424
- protected: read.protected,
425
- read_only: readOnly,
426
- manifests: { ...copy.manifests, [house]: { parent: null, files: read.revisions } },
427
- };
428
- await ignoreRules(copy.repository, [...read.writable.keys()], next);
429
- return next;
505
+ async copied(wanted) {
506
+ const { handle, authority } = wanted;
507
+ const root = rootOf(handle);
508
+ let answer = null;
509
+ try {
510
+ if (!this.waiting.has(authority.room_ref)) {
511
+ answer = await this.house.post(ENUMERATE, {
512
+ prefixes: [root],
513
+ positions: [
514
+ { authority: authority.room_ref, position: '0', log_epoch: authority.log_epoch, page: { through: authority.position, read: 0 } },
515
+ ],
516
+ });
517
+ }
518
+ }
519
+ catch (error) {
520
+ if (expiredAuthority(error) !== authority.room_ref)
521
+ throw error;
522
+ }
523
+ const view = answer?.authorities.find((entry) => entry.authority === authority.room_ref);
524
+ if (answer !== null && view === undefined)
525
+ throw new Error('House serves nothing of it to copy');
526
+ if (view !== undefined && view.next === undefined) {
527
+ const texts = view.changes.filter((change) => !('removed' in change));
528
+ const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, texts }));
529
+ return `${root}: copied to ${copy.repository}`;
530
+ }
531
+ const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
532
+ const [caught] = await this.caughtUp([copy], true);
533
+ await succeeded(copy.repository, ['merge', '--ff-only', '-q', caught.house]);
534
+ return `${root}: copied to ${copy.repository}`;
430
535
  }
431
- async refreshed(copies) {
432
- const discovery = await this.house.post('/kit/door/discover', {
433
- prefixes: copies.map((copy) => rootOf(copy.room_handle)),
434
- });
435
- const regranted = new Map();
536
+ async received(copy, authority) {
537
+ const root = rootOf(copy.room_handle);
538
+ const directory = join(copy.repository, '.git', STAGED_CHUNKS);
539
+ const scope = { write: authority.operations.write, protected: authority.protected_paths };
540
+ try {
541
+ const read = await this.chunked(root, copy.room_ref, directory);
542
+ const readOnly = await readOnlyFiles(copy.repository, copy.read_only);
543
+ for (const path of copy.read_only)
544
+ await readOnly.remove(path);
545
+ const commit = { message: `House ${root} at position ${read.position}`, from: copy.house, replaces: true };
546
+ const { house, revisions } = await importedWhole(copy.repository, scope, read.texts, commit, readOnly);
547
+ const next = {
548
+ ...copy,
549
+ ...scope,
550
+ house,
551
+ position: read.position,
552
+ log_epoch: read.log_epoch,
553
+ read_only: readOnly.held(),
554
+ manifests: { ...copy.manifests, [house]: { parent: null, files: revisions } },
555
+ };
556
+ await ignoreRules(copy.repository, Object.keys(revisions), next);
557
+ return next;
558
+ }
559
+ finally {
560
+ await rm(directory, { recursive: true, force: true });
561
+ }
562
+ }
563
+ async reread(copy) {
564
+ const { authorities } = await this.discovered([copy.room_handle]);
565
+ const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
566
+ if (authority === undefined)
567
+ throw new NotReady([copy.room_ref]);
568
+ return this.received(copy, authority);
569
+ }
570
+ async refreshed(copies, unready) {
571
+ const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
572
+ const read = [];
573
+ const unchanged = [];
436
574
  for (const copy of copies) {
437
- const authority = discovery.authorities.find((entry) => entry.room_ref === copy.room_ref);
575
+ const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
438
576
  if (authority !== undefined &&
439
577
  JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
440
- regranted.set(copy.key, await this.save(await this.received(copy)));
578
+ try {
579
+ read.push(await this.save(await this.received(copy, authority)));
580
+ }
581
+ catch (error) {
582
+ if (!(error instanceof NotReady))
583
+ throw error;
584
+ unready.push(...error.rooms);
585
+ }
586
+ }
587
+ else {
588
+ unchanged.push(copy);
441
589
  }
442
590
  }
443
- const caught = await this.caughtUp(copies.filter((copy) => !regranted.has(copy.key)), false);
444
- return copies.map((copy) => regranted.get(copy.key) ?? caught.find((held) => held.key === copy.key));
591
+ for (const group of grouped(unchanged, bounds.max_prefixes))
592
+ await this.caughtUp(group, false, unready);
593
+ for (const group of grouped(read, bounds.max_prefixes))
594
+ await this.caughtUp(group, true, unready);
445
595
  }
446
- async caughtUp(copies, whole) {
596
+ async caughtUp(copies, whole, unready) {
447
597
  const caught = new Map(copies.map((copy) => [copy.key, copy]));
598
+ const read = new Set();
599
+ const waiting = unready ?? [];
448
600
  let asking = [...copies];
449
601
  while (asking.length > 0) {
450
602
  let answer;
451
603
  try {
452
- answer = await this.house.post('/kit/door/enumerate', {
604
+ answer = await this.house.post(ENUMERATE, {
453
605
  prefixes: asking.map((copy) => rootOf(copy.room_handle)),
454
606
  positions: asking.map((copy) => ({ authority: copy.room_ref, position: copy.position, log_epoch: copy.log_epoch })),
455
607
  });
@@ -458,8 +610,22 @@ export class LocalCopies {
458
610
  const expired = asking.find((copy) => copy.room_ref === expiredAuthority(error));
459
611
  if (expired === undefined)
460
612
  throw error;
461
- caught.set(expired.key, await this.save(await this.received(expired)));
462
613
  asking = asking.filter((copy) => copy !== expired);
614
+ if (read.has(expired.key)) {
615
+ waiting.push(expired.room_ref);
616
+ continue;
617
+ }
618
+ read.add(expired.key);
619
+ try {
620
+ const next = await this.save(await this.reread(expired));
621
+ caught.set(expired.key, next);
622
+ asking.push(next);
623
+ }
624
+ catch (refused) {
625
+ if (!(refused instanceof NotReady))
626
+ throw refused;
627
+ waiting.push(expired.room_ref);
628
+ }
463
629
  continue;
464
630
  }
465
631
  const advanced = [];
@@ -471,8 +637,10 @@ export class LocalCopies {
471
637
  caught.set(copy.key, next);
472
638
  advanced.push(next);
473
639
  }
474
- asking = whole ? advanced : [];
640
+ asking = advanced.filter((copy) => whole || read.has(copy.key));
475
641
  }
642
+ if (unready === undefined && waiting.length > 0)
643
+ throw new NotReady(waiting);
476
644
  return copies.map((copy) => caught.get(copy.key));
477
645
  }
478
646
  async incorporated(copy, changes, position, logEpoch) {
@@ -536,9 +704,6 @@ export class LocalCopies {
536
704
  }
537
705
  copy = await this.save(without(copy));
538
706
  }
539
- if (!copy.active) {
540
- return { refused: true, text: `${rootOf(copy.room_handle)} is no longer synced to this Environment, so House takes no push from it` };
541
- }
542
707
  const listed = (await git(copy.repository, ['rev-list', '--topo-order', commit])).split('\n');
543
708
  const base = listed.find((ancestor) => ancestor in copy.manifests);
544
709
  if (base === undefined) {
@@ -549,7 +714,7 @@ export class LocalCopies {
549
714
  changes = await this.derived(copy, base, commit);
550
715
  }
551
716
  catch (error) {
552
- return { refused: true, text: causeOf(error) };
717
+ return { refused: true, text: houseCause(error) };
553
718
  }
554
719
  if (changes.length === 0)
555
720
  return { refused: false, text: `Nothing to submit: ${short(commit)} matches House.` };
@@ -576,13 +741,19 @@ export class LocalCopies {
576
741
  const after = await tree(copy.repository, commit);
577
742
  const moved = movedPaths(copy.moves, before, after);
578
743
  const arrived = new Set(moved.values());
579
- const binary = [];
580
- for (const [path, entry] of after)
581
- if (before.get(path) !== entry && !(await isText(copy.repository, entry)))
582
- binary.push(path);
583
- if (binary.length > 0)
584
- throw new Error(textOnly(binary));
744
+ const arriving = [...after].filter(([path, entry]) => before.get(path) !== entry);
745
+ if (arriving.length > 0) {
746
+ const { bounds } = await this.house.post('/kit/door/rooms/discover', { prefixes: [root] });
747
+ const binary = [];
748
+ for (const [path, entry] of arriving) {
749
+ if (!(await isText(copy.repository, entry, bounds.max_authored_file_bytes)))
750
+ binary.push(path);
751
+ }
752
+ if (binary.length > 0)
753
+ throw new Error(textOnly(binary));
754
+ }
585
755
  const text = (entry) => blobText(copy.repository, entry);
756
+ const model = await modelOf(copy, after);
586
757
  const changes = [];
587
758
  for (const path of [...new Set([...before.keys(), ...after.keys()])].sort()) {
588
759
  if (arrived.has(path))
@@ -606,7 +777,7 @@ export class LocalCopies {
606
777
  else if (prior === undefined)
607
778
  changes.push({ op: 'create', path: `${root}/${path}`, content: await text(next) });
608
779
  else
609
- changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next)));
780
+ changes.push(...changed(`${root}/${path}`, path, known, await text(prior), await text(next), model));
610
781
  }
611
782
  return changes;
612
783
  }
@@ -635,8 +806,10 @@ export class LocalCopies {
635
806
  }
636
807
  if (outcome.refused.code === 'expired')
637
808
  return this.recovered(copy);
638
- const rebased = rebaseFixes(outcome.refused);
639
809
  let settled = await this.save(without(copy));
810
+ if (settled.unfollowed !== undefined)
811
+ return { refused: true, text: outcome.refused.text };
812
+ const rebased = rebaseFixes(outcome.refused);
640
813
  try {
641
814
  settled = (await this.caughtUp([settled], true))[0];
642
815
  }
@@ -660,7 +833,8 @@ export class LocalCopies {
660
833
  const current = (await this.caughtUp([copy], true))[0];
661
834
  const house = await tree(copy.repository, current.house);
662
835
  const root = rootOf(current.room_handle);
663
- if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit)))) {
836
+ const model = await modelOf(current, house);
837
+ if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit), model))) {
664
838
  await this.save(without(current));
665
839
  return {
666
840
  refused: true,
@@ -759,7 +933,7 @@ export class LocalCopies {
759
933
  push: { ...push, acknowledged },
760
934
  };
761
935
  }
762
- async landed(repository, changes, house, selected) {
936
+ async landed(repository, changes, house, selected, model) {
763
937
  const text = async (path) => {
764
938
  const entry = house.get(relative(path));
765
939
  return entry === undefined ? undefined : blobText(repository, entry);
@@ -773,9 +947,8 @@ export class LocalCopies {
773
947
  }
774
948
  if ((change.op === 'create' || change.op === 'replace') && current === change.content)
775
949
  continue;
776
- const parsed = current === undefined ? null : parse(current, change.path);
777
- const document = parsed?.ok === true ? parsed.document : null;
778
- if (document === null)
950
+ const document = current === undefined ? undefined : readDocument(current, relative(change.path), model)?.document;
951
+ if (document === undefined)
779
952
  return false;
780
953
  if (change.op === 'set_field' && JSON.stringify(document.fields[change.field]) === JSON.stringify(change.value))
781
954
  continue;
@@ -788,7 +961,7 @@ export class LocalCopies {
788
961
  return true;
789
962
  }
790
963
  async revisionText(path, wanted) {
791
- const page = await this.house.post('/kit/door/revisions', {
964
+ const page = await this.house.post('/kit/door/files/revisions/list', {
792
965
  path,
793
966
  revision: wanted,
794
967
  });