@agentshouse/kit 0.2.1-ccffb3f637815 → 0.2.1-cd423aab3dbcc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -7
- package/dist/binaries.js +22 -51
- package/dist/bridge.js +77 -23
- package/dist/chunks.js +5 -0
- package/dist/connect.js +20 -7
- package/dist/conversations.js +38 -36
- package/dist/copies.js +991 -830
- package/dist/earlier.js +13 -3
- package/dist/git.js +0 -145
- package/dist/home.js +8 -2
- package/dist/house-cli.js +49 -49
- package/dist/manifest.js +75 -0
- package/dist/mcp.js +6 -5
- package/dist/refusals.js +15 -4
- package/dist/resident.js +18 -13
- package/dist/scripts.js +20 -4
- package/dist/settings.js +82 -23
- package/dist/workspace.js +11 -1
- package/package.json +3 -4
- package/dist/git-main.js +0 -148
- package/dist/old-references.js +0 -27
- package/dist/translate.js +0 -52
package/dist/copies.js
CHANGED
|
@@ -1,26 +1,29 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { isUtf8 } from 'node:buffer';
|
|
2
2
|
import { createHash, randomUUID } from 'node:crypto';
|
|
3
3
|
import { mkdir, readFile, readdir, realpath, rename, rm, rmdir, stat, writeFile } from 'node:fs/promises';
|
|
4
|
-
import { dirname, join } from 'node:path';
|
|
4
|
+
import { basename, dirname, join } from 'node:path';
|
|
5
5
|
import { finalRefusal, HouseRefusal, retrying } from './api.js';
|
|
6
|
-
import {
|
|
6
|
+
import { Changed, eachAtOnce, isDocument, mediaType, receivedBinary, sentBinary } from './binaries.js';
|
|
7
7
|
import { chunkEntries, stagedChunks } from './chunks.js';
|
|
8
|
-
import { blobId, blobText, git, HOUSE_REF, importCommits, localCopies, revision, run, succeeded, tree, } from './git.js';
|
|
9
8
|
import { fileInfo, paused } from './files.js';
|
|
10
|
-
import { kitHome } from './home.js';
|
|
9
|
+
import { kitHome, roomsFolder } from './home.js';
|
|
10
|
+
import { digest, hashed, holding, marked, placed, readManifest, removed, scanned, writeManifest, } from './manifest.js';
|
|
11
11
|
import { ownAgentCall, rpc, toolAnswer, Unsettled } from './mcp.js';
|
|
12
|
-
import { renameOldReferences } from './old-references.js';
|
|
13
12
|
import { operationId } from './operation.js';
|
|
14
13
|
import { firstLine, refusalOf, retryable, stopsPush } from './refusals.js';
|
|
15
|
-
import {
|
|
16
|
-
const BRANCH = 'main';
|
|
17
|
-
const PROVENANCE = '_provenance';
|
|
18
|
-
const IDENTITY = { GIT_COMMITTER_NAME: 'Kit', GIT_COMMITTER_EMAIL: 'house-kit@localhost' };
|
|
14
|
+
import { keptName, moved as carried, present as found, removeFolder, rerooted, within } from './workspace.js';
|
|
19
15
|
const ROOMS = 'rooms/';
|
|
20
16
|
const NOT_READY = 'not ready yet; Kit copies it once House has built it';
|
|
21
|
-
const
|
|
22
|
-
const
|
|
23
|
-
const
|
|
17
|
+
const PROVENANCE = '_provenance';
|
|
18
|
+
const BATCH_FILES = 1000;
|
|
19
|
+
const BATCH_BYTES = 15_000_000;
|
|
20
|
+
const MEGABYTE = 1_000_000;
|
|
21
|
+
const CUT = /\noutput_cut \d+ of \d+ bytes$/;
|
|
22
|
+
const KIT_IGNORES = ['.gitignore', '.ignore'];
|
|
23
|
+
const KIT_GIT = ['.git/refs/house/received', '.git/house-chunks', '.git/house-uploads', '.git/house-downloads'];
|
|
24
|
+
const KIT_NAMES = ['agents.md', 'claude.md'];
|
|
25
|
+
export const DEFAULT_MAX_SIZE = 10;
|
|
26
|
+
export const UPLOAD_NEEDS_OWNER = 'outside an Agent conversation, house upload needs --owner';
|
|
24
27
|
// A Room answered not ready is asked again on this schedule: its chunk build usually catches up within seconds, so the first asks come soon and the last repeats until it does.
|
|
25
28
|
const ASKED_AGAIN_MS = [1000, 2000, 3000, 5000, 8000, 11_000, 15_000];
|
|
26
29
|
class NotReady extends Error {
|
|
@@ -30,6 +33,7 @@ class NotReady extends Error {
|
|
|
30
33
|
this.rooms = rooms;
|
|
31
34
|
}
|
|
32
35
|
}
|
|
36
|
+
const HELD = { attachments: false, maxSize: DEFAULT_MAX_SIZE };
|
|
33
37
|
function causeOf(error) {
|
|
34
38
|
return error instanceof Error ? error.message : String(error);
|
|
35
39
|
}
|
|
@@ -39,22 +43,9 @@ function houseCause(error) {
|
|
|
39
43
|
function logged(error) {
|
|
40
44
|
process.stderr.write(`kit: ${causeOf(error)}\n`);
|
|
41
45
|
}
|
|
42
|
-
const REBASED = new Set([
|
|
43
|
-
'field_base_stale',
|
|
44
|
-
'section_base_stale',
|
|
45
|
-
'preamble_base_stale',
|
|
46
|
-
'base_revision_stale',
|
|
47
|
-
'path_exists',
|
|
48
|
-
'path_absent',
|
|
49
|
-
'base_required',
|
|
50
|
-
'text_file',
|
|
51
|
-
]);
|
|
52
|
-
function rebaseFixes(refusal) {
|
|
53
|
-
return refusal.code === 'edit_conflict' && refusal.conflicts.some(({ code }) => REBASED.has(code));
|
|
54
|
-
}
|
|
55
46
|
function markdownOnly(paths, maxBytes) {
|
|
56
47
|
const [named, them] = paths.length === 1 ? [`${paths[0]} is`, 'it'] : [`${paths.join(', ')} are`, 'them'];
|
|
57
|
-
return `${named} Markdown, which holds text only: UTF-8 without NUL, at most ${maxBytes} bytes;
|
|
48
|
+
return `${named} Markdown, which holds text only: UTF-8 without NUL, at most ${maxBytes} bytes; save ${them} as text and run house upload again`;
|
|
58
49
|
}
|
|
59
50
|
async function stopping(sent) {
|
|
60
51
|
try {
|
|
@@ -67,27 +58,19 @@ async function stopping(sent) {
|
|
|
67
58
|
return { refused: refusal };
|
|
68
59
|
}
|
|
69
60
|
}
|
|
70
|
-
function changesKind(changes) {
|
|
71
|
-
const removed = new Set(changes.flatMap((change) => (change.op === 'remove' ? [change.path] : [])));
|
|
72
|
-
return changes.some((change) => (change.op === 'create' || change.op === 'upload') && removed.has(change.path));
|
|
73
|
-
}
|
|
74
61
|
function unstagedUploads(refusal) {
|
|
75
62
|
const named = refusal.detail.upload;
|
|
76
63
|
return Array.isArray(named) ? named : [];
|
|
77
64
|
}
|
|
78
|
-
function
|
|
79
|
-
|
|
65
|
+
function currentHandle(refusal) {
|
|
66
|
+
const path = typeof refusal.detail.current_path === 'string' ? refusal.detail.current_path : /use (\/rooms\/\S+)/.exec(refusal.text)?.[1];
|
|
67
|
+
return path === undefined ? null : (/^\/rooms\/([^/]+)/.exec(path)?.[1] ?? null);
|
|
80
68
|
}
|
|
81
69
|
function outside(path) {
|
|
82
|
-
return { refused: true, text: `${path} is not a House Local copy; run house
|
|
70
|
+
return { refused: true, text: `${path} is not in a House Local copy; run house upload inside one` };
|
|
83
71
|
}
|
|
84
|
-
function
|
|
85
|
-
return
|
|
86
|
-
}
|
|
87
|
-
function moved(before, after) {
|
|
88
|
-
if (before.house !== after.house)
|
|
89
|
-
return [`${rootOf(after.room_handle)}: ${HOUSE_REF} is at ${short(after.house)}`];
|
|
90
|
-
return before.position === after.position ? [] : [`${rootOf(after.room_handle)}: read-only files refreshed`];
|
|
72
|
+
function counted(count, one, many = `${one}s`) {
|
|
73
|
+
return `${count} ${count === 1 ? one : many}`;
|
|
91
74
|
}
|
|
92
75
|
function expiredAuthority(error) {
|
|
93
76
|
if (!(error instanceof HouseRefusal))
|
|
@@ -117,246 +100,296 @@ function recordOf(key) {
|
|
|
117
100
|
function stagingPrefix() {
|
|
118
101
|
return `.${createHash('sha256').update(kitHome()).digest('hex').slice(0, 16)}-`;
|
|
119
102
|
}
|
|
103
|
+
async function sweptIn(folder) {
|
|
104
|
+
for (const name of await readdir(folder).catch(() => [])) {
|
|
105
|
+
if (name.startsWith(stagingPrefix()))
|
|
106
|
+
await rm(join(folder, name), { recursive: true, force: true });
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
async function staging() {
|
|
110
|
+
const folder = join(roomsFolder(), `${stagingPrefix()}work`);
|
|
111
|
+
await mkdir(folder, { recursive: true });
|
|
112
|
+
return `${folder}/`;
|
|
113
|
+
}
|
|
120
114
|
function rootOf(handle) {
|
|
121
115
|
return `/rooms/${handle}`;
|
|
122
116
|
}
|
|
123
117
|
function relative(path) {
|
|
124
118
|
return path.replace(/^\/rooms\/[^/]+\//, '');
|
|
125
119
|
}
|
|
120
|
+
function kitsOwn(path) {
|
|
121
|
+
return !path.includes('/') && KIT_NAMES.includes(path.toLowerCase());
|
|
122
|
+
}
|
|
126
123
|
function declaredWritable(path, copy) {
|
|
127
124
|
const deepest = (scopes) => Math.max(-1, ...scopes.filter((scope) => covers(path, scope)).map((scope) => scope.length));
|
|
128
125
|
return deepest(copy.write) > deepest(copy.protected);
|
|
129
126
|
}
|
|
130
|
-
function
|
|
131
|
-
|
|
127
|
+
function differing(left, right) {
|
|
128
|
+
const counts = new Map();
|
|
129
|
+
for (const line of left.split('\n'))
|
|
130
|
+
counts.set(line, (counts.get(line) ?? 0) + 1);
|
|
131
|
+
let unmatched = 0;
|
|
132
|
+
for (const line of right.split('\n')) {
|
|
133
|
+
const held = counts.get(line) ?? 0;
|
|
134
|
+
if (held > 0)
|
|
135
|
+
counts.set(line, held - 1);
|
|
136
|
+
else
|
|
137
|
+
unmatched++;
|
|
138
|
+
}
|
|
139
|
+
return unmatched + [...counts.values()].reduce((sum, count) => sum + count, 0);
|
|
140
|
+
}
|
|
141
|
+
function whole(theirs, mine) {
|
|
142
|
+
return `<<<<<<< current\n${ended(theirs)}=======\n${ended(mine)}>>>>>>> yours\n`;
|
|
143
|
+
}
|
|
144
|
+
function ended(text) {
|
|
145
|
+
return text === '' || text.endsWith('\n') ? text : `${text}\n`;
|
|
146
|
+
}
|
|
147
|
+
function sameBytes(file) {
|
|
148
|
+
return `${file.sha256}:${file.bytes}`;
|
|
132
149
|
}
|
|
133
150
|
async function exists(path) {
|
|
134
151
|
return stat(path).then(() => true, () => false);
|
|
135
152
|
}
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
153
|
+
function without(copy) {
|
|
154
|
+
const { pending, ...settled } = copy;
|
|
155
|
+
return settled;
|
|
139
156
|
}
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
for (let folder = dirname(path); folder !== '.'; folder = dirname(folder)) {
|
|
143
|
-
if (!(await rmdir(join(repository, folder)).then(() => true, () => false)))
|
|
144
|
-
return;
|
|
145
|
-
}
|
|
157
|
+
function earlier(copy) {
|
|
158
|
+
return 'manifests' in copy;
|
|
146
159
|
}
|
|
147
|
-
|
|
148
|
-
const
|
|
149
|
-
const
|
|
160
|
+
function rehandled(copy, handle) {
|
|
161
|
+
const from = `${rootOf(copy.room_handle)}/`;
|
|
162
|
+
const to = `${rootOf(handle)}/`;
|
|
163
|
+
const moved = (path) => (path.startsWith(from) ? `${to}${path.slice(from.length)}` : path);
|
|
164
|
+
const { upload, ...pending } = copy.pending;
|
|
150
165
|
return {
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
holding.delete(path);
|
|
166
|
+
...copy,
|
|
167
|
+
room_handle: handle,
|
|
168
|
+
pending: {
|
|
169
|
+
...pending,
|
|
170
|
+
changes: pending.changes.map((change) => (change.op === 'move' ? { ...change, path: moved(change.path), to: moved(change.to) } : { ...change, path: moved(change.path) })),
|
|
157
171
|
},
|
|
158
|
-
place: async (path, content) => {
|
|
159
|
-
if (tracked.has(path))
|
|
160
|
-
return;
|
|
161
|
-
if (!holding.has(path) && (await exists(join(repository, path))))
|
|
162
|
-
return;
|
|
163
|
-
await writeText(join(repository, path), content);
|
|
164
|
-
holding.add(path);
|
|
165
|
-
},
|
|
166
|
-
held: () => [...holding].sort(),
|
|
167
172
|
};
|
|
168
173
|
}
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
for (const path of removed)
|
|
172
|
-
await files.remove(path);
|
|
173
|
-
for (const [path, content] of written)
|
|
174
|
-
await files.place(path, content);
|
|
175
|
-
return files.held();
|
|
174
|
+
export function changedPaths(changes) {
|
|
175
|
+
return [...new Set(changes.flatMap((change) => (change.op === 'move' ? [change.path, change.to] : [change.path])))].sort();
|
|
176
176
|
}
|
|
177
|
-
|
|
178
|
-
const
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
revisions[path] = held;
|
|
191
|
-
yield [path, content];
|
|
192
|
-
}
|
|
193
|
-
else {
|
|
194
|
-
await readOnly.place(path, content);
|
|
195
|
-
}
|
|
196
|
-
if (provenance !== undefined)
|
|
197
|
-
await readOnly.place(sidecar(path), provenance);
|
|
177
|
+
function batched(changes) {
|
|
178
|
+
const batches = [];
|
|
179
|
+
let batch = [];
|
|
180
|
+
let files = 0;
|
|
181
|
+
let bytes = 0;
|
|
182
|
+
for (const change of changes) {
|
|
183
|
+
const named = change.op === 'move' ? 2 : 1;
|
|
184
|
+
const size = Buffer.byteLength(JSON.stringify(change));
|
|
185
|
+
if (batch.length > 0 && (files + named > BATCH_FILES || bytes + size > BATCH_BYTES)) {
|
|
186
|
+
batches.push(batch);
|
|
187
|
+
batch = [];
|
|
188
|
+
files = 0;
|
|
189
|
+
bytes = 0;
|
|
198
190
|
}
|
|
199
|
-
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
191
|
+
batch.push(change);
|
|
192
|
+
files += named;
|
|
193
|
+
bytes += size;
|
|
194
|
+
}
|
|
195
|
+
if (batch.length > 0)
|
|
196
|
+
batches.push(batch);
|
|
197
|
+
return batches;
|
|
198
|
+
}
|
|
199
|
+
function recordedFiles(copy) {
|
|
200
|
+
const found = new Map();
|
|
201
|
+
const seen = new Set();
|
|
202
|
+
for (let manifest = copy.manifests[copy.house]; manifest !== undefined; manifest = manifest.parent === null ? undefined : copy.manifests[manifest.parent]) {
|
|
203
|
+
for (const [path, revision] of Object.entries(manifest.files)) {
|
|
204
|
+
if (seen.has(path))
|
|
203
205
|
continue;
|
|
204
|
-
|
|
205
|
-
binary[
|
|
206
|
-
|
|
206
|
+
seen.add(path);
|
|
207
|
+
const held = manifest.binary?.[path];
|
|
208
|
+
if (revision !== null)
|
|
209
|
+
found.set(path, held === undefined ? { revision } : { revision, held });
|
|
207
210
|
}
|
|
208
211
|
}
|
|
209
|
-
return
|
|
210
|
-
}
|
|
211
|
-
async function ruled(path, rules) {
|
|
212
|
-
if ((await readFile(path, 'utf8').catch(() => null)) !== rules)
|
|
213
|
-
await writeFile(path, rules);
|
|
212
|
+
return found;
|
|
214
213
|
}
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
const protectedPath = (path) => copy.protected.some((prefix) => prefix !== '' && covers(path, prefix));
|
|
218
|
-
const whole = copy.protected.filter((prefix) => prefix !== '' && !writable.some((path) => covers(path, prefix)));
|
|
219
|
-
const house = [
|
|
220
|
-
...whole,
|
|
221
|
-
...readOnly.filter((path) => protectedPath(path) && !whole.some((prefix) => covers(path, prefix))),
|
|
222
|
-
].map(pattern);
|
|
223
|
-
const granted = readOnly.filter((path) => !protectedPath(path)).map(pattern);
|
|
224
|
-
await ruled(join(repository, '.gitignore'), [
|
|
225
|
-
'# Kit writes this file and .ignore. The paths below are read-only House',
|
|
226
|
-
'# content: Git does not track them, and House refuses a change to them even',
|
|
227
|
-
'# when this file is edited or a path is force-added.',
|
|
228
|
-
'/.gitignore',
|
|
229
|
-
'/.ignore',
|
|
230
|
-
PROVENANCE,
|
|
231
|
-
'',
|
|
232
|
-
'# House maintains or protects these paths.',
|
|
233
|
-
...house,
|
|
234
|
-
'',
|
|
235
|
-
'# Your current House grant does not allow writing these paths.',
|
|
236
|
-
...granted,
|
|
237
|
-
'',
|
|
238
|
-
].join('\n'));
|
|
239
|
-
await ruled(join(repository, '.ignore'), [
|
|
240
|
-
'# Kit writes this file so native search still reads the read-only House',
|
|
241
|
-
'# content that .gitignore keeps out of Git.',
|
|
242
|
-
`!${PROVENANCE}`,
|
|
243
|
-
...[...house, ...granted].map((rule) => `!${rule}`),
|
|
244
|
-
'',
|
|
245
|
-
].join('\n'));
|
|
214
|
+
function recordedRevisions(copy, path) {
|
|
215
|
+
return [...new Set(Object.values(copy.manifests).flatMap((manifest) => (typeof manifest.files[path] === 'string' ? [manifest.files[path]] : [])))];
|
|
246
216
|
}
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
217
|
+
class Applying {
|
|
218
|
+
present = new Map();
|
|
219
|
+
binaries = [];
|
|
220
|
+
gone = [];
|
|
221
|
+
repository;
|
|
222
|
+
manifest;
|
|
223
|
+
scope;
|
|
224
|
+
staging;
|
|
225
|
+
report;
|
|
226
|
+
fetching;
|
|
227
|
+
bytesOf;
|
|
228
|
+
constructor(repository, manifest, scope, staging, report, fetching, bytesOf) {
|
|
229
|
+
this.repository = repository;
|
|
230
|
+
this.manifest = manifest;
|
|
231
|
+
this.scope = scope;
|
|
232
|
+
this.staging = staging;
|
|
233
|
+
this.report = report;
|
|
234
|
+
this.fetching = fetching;
|
|
235
|
+
this.bytesOf = bytesOf;
|
|
236
|
+
}
|
|
237
|
+
async start() {
|
|
238
|
+
this.present = await scanned(this.repository);
|
|
239
|
+
}
|
|
240
|
+
nameOf(path) {
|
|
241
|
+
return this.present.get(path)?.name ?? path;
|
|
242
|
+
}
|
|
243
|
+
touch(path) {
|
|
244
|
+
if (!this.report.touched.has(path))
|
|
245
|
+
this.report.touched.set(path, this.present.get(path)?.sha256 ?? null);
|
|
246
|
+
}
|
|
247
|
+
async put(path, content) {
|
|
248
|
+
this.touch(path);
|
|
249
|
+
await placed(this.staging, join(this.repository, this.nameOf(path)), content);
|
|
250
|
+
this.present.set(path, { name: this.nameOf(path), sha256: digest(content), bytes: Buffer.byteLength(content) });
|
|
251
|
+
}
|
|
252
|
+
async drop(path) {
|
|
253
|
+
this.touch(path);
|
|
254
|
+
await removed(this.repository, this.nameOf(path));
|
|
255
|
+
this.present.delete(path);
|
|
256
|
+
}
|
|
257
|
+
async readOnly(path, content, revision) {
|
|
258
|
+
const sha256 = digest(content);
|
|
259
|
+
if (this.present.get(path)?.sha256 !== sha256)
|
|
260
|
+
await this.put(path, content);
|
|
261
|
+
this.manifest.files[path] = { revision, sha256, bytes: Buffer.byteLength(content), read_only: true };
|
|
262
|
+
}
|
|
263
|
+
async text({ path, content, revision, provenance }) {
|
|
264
|
+
if (provenance !== undefined)
|
|
265
|
+
await this.readOnly(sidecar(path), provenance, revision);
|
|
266
|
+
if (!declaredWritable(path, this.scope)) {
|
|
267
|
+
await this.readOnly(path, content, revision);
|
|
268
|
+
return;
|
|
269
|
+
}
|
|
270
|
+
const entry = this.manifest.files[path];
|
|
271
|
+
if (entry !== undefined && !entry.read_only && entry.revision === revision)
|
|
272
|
+
return;
|
|
273
|
+
const sha256 = digest(content);
|
|
274
|
+
const kept = { revision, sha256, bytes: Buffer.byteLength(content) };
|
|
275
|
+
const local = this.present.get(path);
|
|
276
|
+
if (local?.sha256 === sha256) {
|
|
277
|
+
this.manifest.files[path] = kept;
|
|
278
|
+
return;
|
|
279
|
+
}
|
|
280
|
+
const unchanged = entry === undefined ? local === undefined : entry.read_only === true || (entry.conflicted === undefined && local?.sha256 === entry.sha256);
|
|
281
|
+
if (!unchanged || (local !== undefined && !(await holding(this.repository, local.name, local.sha256)))) {
|
|
282
|
+
this.report.kept.add(path);
|
|
283
|
+
return;
|
|
284
|
+
}
|
|
285
|
+
await this.put(path, content);
|
|
286
|
+
this.manifest.files[path] = kept;
|
|
287
|
+
}
|
|
288
|
+
binary(change) {
|
|
289
|
+
this.binaries.push(change);
|
|
290
|
+
}
|
|
291
|
+
removal(path) {
|
|
292
|
+
this.gone.push(path);
|
|
293
|
+
}
|
|
294
|
+
async finish() {
|
|
295
|
+
const leaving = new Map();
|
|
296
|
+
for (const path of this.gone) {
|
|
297
|
+
const entry = this.manifest.files[path];
|
|
298
|
+
if (entry === undefined || entry.skipped || isDocument(path) || this.present.get(path)?.sha256 !== entry.sha256)
|
|
299
|
+
continue;
|
|
300
|
+
leaving.set(sameBytes(entry), leaving.has(sameBytes(entry)) ? null : path);
|
|
301
|
+
}
|
|
302
|
+
const taken = new Set();
|
|
303
|
+
const movedFrom = (change) => {
|
|
304
|
+
const from = leaving.get(sameBytes(change));
|
|
305
|
+
if (from === undefined || from === null || taken.has(from))
|
|
269
306
|
return undefined;
|
|
270
|
-
|
|
271
|
-
return
|
|
307
|
+
taken.add(from);
|
|
308
|
+
return from;
|
|
309
|
+
};
|
|
310
|
+
for (const change of this.binaries)
|
|
311
|
+
await this.attachment(change, movedFrom);
|
|
312
|
+
for (const path of this.gone) {
|
|
313
|
+
if (taken.has(path))
|
|
314
|
+
continue;
|
|
315
|
+
await this.removedHere(path);
|
|
316
|
+
if (this.manifest.files[sidecar(path)]?.read_only)
|
|
317
|
+
await this.removedHere(sidecar(path));
|
|
272
318
|
}
|
|
273
|
-
manifest = manifest.parent === null ? undefined : copy.manifests[manifest.parent];
|
|
274
319
|
}
|
|
275
|
-
|
|
276
|
-
}
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
285
|
-
}
|
|
286
|
-
function binaryParts(changes) {
|
|
287
|
-
return changes.filter((change) => change.op === 'upload');
|
|
288
|
-
}
|
|
289
|
-
function pushedBinary(copy, push) {
|
|
290
|
-
const sent = new Map();
|
|
291
|
-
for (const change of push.changes) {
|
|
292
|
-
if (change.op === 'upload')
|
|
293
|
-
sent.set(relative(change.path), { bytes: change.bytes, sha256: change.sha256 });
|
|
294
|
-
else if (change.op === 'rename') {
|
|
295
|
-
const held = heldAt(copy, push.base, relative(change.path));
|
|
296
|
-
if (held !== undefined)
|
|
297
|
-
sent.set(relative(change.to), held);
|
|
320
|
+
async attachment(change, movedFrom) {
|
|
321
|
+
const { path, revision, bytes, sha256 } = change;
|
|
322
|
+
const entry = this.manifest.files[path];
|
|
323
|
+
if (entry !== undefined && entry.revision === revision)
|
|
324
|
+
return;
|
|
325
|
+
const held = { revision, sha256, bytes, ...(declaredWritable(path, this.scope) ? {} : { read_only: true }) };
|
|
326
|
+
const local = this.present.get(path);
|
|
327
|
+
if (local?.sha256 === sha256) {
|
|
328
|
+
this.manifest.files[path] = held;
|
|
329
|
+
return;
|
|
298
330
|
}
|
|
331
|
+
const holds = entry !== undefined && !entry.skipped;
|
|
332
|
+
if (holds && !entry.read_only && local?.sha256 !== entry.sha256) {
|
|
333
|
+
this.report.kept.add(path);
|
|
334
|
+
return;
|
|
335
|
+
}
|
|
336
|
+
if (!holds && local !== undefined) {
|
|
337
|
+
this.report.kept.add(path);
|
|
338
|
+
return;
|
|
339
|
+
}
|
|
340
|
+
const from = movedFrom(change);
|
|
341
|
+
if (from !== undefined) {
|
|
342
|
+
const target = join(this.repository, this.nameOf(path));
|
|
343
|
+
this.touch(path);
|
|
344
|
+
await mkdir(dirname(target), { recursive: true });
|
|
345
|
+
await rename(join(this.repository, this.nameOf(from)), target);
|
|
346
|
+
await removed(this.repository, this.nameOf(from));
|
|
347
|
+
this.present.set(path, { name: this.nameOf(path), sha256, bytes });
|
|
348
|
+
this.present.delete(from);
|
|
349
|
+
delete this.manifest.files[from];
|
|
350
|
+
this.manifest.files[path] = held;
|
|
351
|
+
return;
|
|
352
|
+
}
|
|
353
|
+
if (!holds && !(this.fetching.attachments && bytes <= this.fetching.maxSize * MEGABYTE)) {
|
|
354
|
+
this.manifest.files[path] = { ...held, skipped: true };
|
|
355
|
+
return;
|
|
356
|
+
}
|
|
357
|
+
const file = await this.bytesOf(change);
|
|
358
|
+
const target = join(this.repository, this.nameOf(path));
|
|
359
|
+
this.touch(path);
|
|
360
|
+
await mkdir(dirname(target), { recursive: true });
|
|
361
|
+
await rename(file, target);
|
|
362
|
+
this.present.set(path, { name: this.nameOf(path), sha256, bytes });
|
|
363
|
+
this.manifest.files[path] = held;
|
|
364
|
+
}
|
|
365
|
+
async removedHere(path) {
|
|
366
|
+
const entry = this.manifest.files[path];
|
|
367
|
+
if (entry === undefined)
|
|
368
|
+
return;
|
|
369
|
+
const local = this.present.get(path);
|
|
370
|
+
if (entry.skipped || local === undefined) {
|
|
371
|
+
delete this.manifest.files[path];
|
|
372
|
+
return;
|
|
373
|
+
}
|
|
374
|
+
if (!entry.read_only && (local.sha256 !== entry.sha256 || !(await holding(this.repository, local.name, entry.sha256)))) {
|
|
375
|
+
this.report.kept.add(path);
|
|
376
|
+
return;
|
|
377
|
+
}
|
|
378
|
+
await this.drop(path);
|
|
379
|
+
delete this.manifest.files[path];
|
|
299
380
|
}
|
|
300
|
-
return sent;
|
|
301
|
-
}
|
|
302
|
-
async function writtenAt(copy, house, path) {
|
|
303
|
-
const named = `${rootOf(copy.room_handle)}/${path}`;
|
|
304
|
-
const entry = house.get(path);
|
|
305
|
-
if (entry === undefined)
|
|
306
|
-
return { path: named, revision: null, content: null };
|
|
307
|
-
const revision = revisionAt(copy, copy.house, path) ?? null;
|
|
308
|
-
return heldAt(copy, copy.house, path) === undefined
|
|
309
|
-
? { path: named, revision, content: await blobText(copy.repository, entry) }
|
|
310
|
-
: { path: named, revision };
|
|
311
|
-
}
|
|
312
|
-
function without(copy) {
|
|
313
|
-
const { push, ...settled } = copy;
|
|
314
|
-
return settled;
|
|
315
|
-
}
|
|
316
|
-
export function changedPaths(changes) {
|
|
317
|
-
return [
|
|
318
|
-
...new Set(changes.flatMap((change) => (change.op === 'rename' ? [change.path, change.to] : [change.path]))),
|
|
319
|
-
].sort();
|
|
320
|
-
}
|
|
321
|
-
function movedPaths(moves, before, after) {
|
|
322
|
-
const pairs = moves.filter(({ from, to }) => before.has(from) && !after.has(from) && after.has(to) && !before.has(to));
|
|
323
|
-
return new Map(pairs
|
|
324
|
-
.filter((pair) => pairs.filter((other) => other.from === pair.from || other.to === pair.to).length === 1)
|
|
325
|
-
.map(({ from, to }) => [from, to]));
|
|
326
|
-
}
|
|
327
|
-
function sameBytesMoved(before, after, held, binary, intended) {
|
|
328
|
-
const arriving = new Set(intended.values());
|
|
329
|
-
const leaving = new Map();
|
|
330
|
-
const added = new Map();
|
|
331
|
-
const listed = (paths, file, path) => {
|
|
332
|
-
paths.set(heldKey(file), [...(paths.get(heldKey(file)) ?? []), path]);
|
|
333
|
-
};
|
|
334
|
-
for (const [path, entry] of before) {
|
|
335
|
-
const file = held(path);
|
|
336
|
-
if (file !== undefined && !intended.has(path) && !binary.has(path) && after.get(path) !== entry)
|
|
337
|
-
listed(leaving, file, path);
|
|
338
|
-
}
|
|
339
|
-
for (const [path, file] of binary) {
|
|
340
|
-
if (!arriving.has(path) && held(path) === undefined)
|
|
341
|
-
listed(added, file, path);
|
|
342
|
-
}
|
|
343
|
-
const moved = new Map();
|
|
344
|
-
for (const [key, [from, ...others]] of leaving) {
|
|
345
|
-
const [to, ...more] = added.get(key) ?? [];
|
|
346
|
-
if (others.length === 0 && more.length === 0 && to !== undefined && !after.has(from) && !before.has(to))
|
|
347
|
-
moved.set(from, to);
|
|
348
|
-
}
|
|
349
|
-
return moved;
|
|
350
381
|
}
|
|
351
382
|
export class LocalCopies {
|
|
352
383
|
owner;
|
|
353
384
|
house;
|
|
385
|
+
launched;
|
|
354
386
|
copies = new Map();
|
|
355
387
|
queues = new Map();
|
|
356
388
|
waiting = new Set();
|
|
357
389
|
answering = Promise.resolve();
|
|
358
|
-
constructor(house) {
|
|
390
|
+
constructor(house, launched) {
|
|
359
391
|
this.house = house;
|
|
392
|
+
this.launched = launched;
|
|
360
393
|
this.owner = {
|
|
361
394
|
key: 'owner',
|
|
362
395
|
submit: async (operation, changes) => {
|
|
@@ -372,7 +405,7 @@ export class LocalCopies {
|
|
|
372
405
|
};
|
|
373
406
|
}
|
|
374
407
|
},
|
|
375
|
-
finish: async (upload,
|
|
408
|
+
finish: async (upload, _path, partSha256) => {
|
|
376
409
|
try {
|
|
377
410
|
return (await house.act('uploads/finish', { upload, part_sha256: partSha256 }));
|
|
378
411
|
}
|
|
@@ -401,29 +434,20 @@ export class LocalCopies {
|
|
|
401
434
|
if (text === null)
|
|
402
435
|
continue;
|
|
403
436
|
const copy = JSON.parse(text);
|
|
404
|
-
|
|
405
|
-
await this.save(copy);
|
|
406
|
-
else
|
|
407
|
-
this.copies.set(copy.key, copy);
|
|
437
|
+
this.copies.set(copy.key, copy);
|
|
408
438
|
}
|
|
409
439
|
}
|
|
410
440
|
async swept() {
|
|
411
|
-
|
|
412
|
-
for (const name of await readdir(rooms).catch(() => [])) {
|
|
413
|
-
if (name.startsWith(stagingPrefix()))
|
|
414
|
-
await rm(join(rooms, name), { recursive: true, force: true });
|
|
415
|
-
}
|
|
416
|
-
for (const copy of this.copies.values())
|
|
417
|
-
await sweptTransfers(copy.repository);
|
|
441
|
+
await sweptIn(roomsFolder());
|
|
418
442
|
}
|
|
419
443
|
connected() {
|
|
420
444
|
return this.selected();
|
|
421
445
|
}
|
|
422
|
-
|
|
423
|
-
return this.answered(() => this.fetched(true, caller));
|
|
446
|
+
download(caller, fetching) {
|
|
447
|
+
return this.answered(() => this.fetched(true, caller, fetching));
|
|
424
448
|
}
|
|
425
449
|
async selected() {
|
|
426
|
-
const copied = await this.answered(() => this.fetched(false, this.owner));
|
|
450
|
+
const copied = await this.answered(() => this.fetched(false, this.owner, HELD));
|
|
427
451
|
if (copied.refused)
|
|
428
452
|
logged(copied.text);
|
|
429
453
|
}
|
|
@@ -432,10 +456,11 @@ export class LocalCopies {
|
|
|
432
456
|
this.answering = answered.catch(() => undefined);
|
|
433
457
|
return answered;
|
|
434
458
|
}
|
|
435
|
-
async fetched(refreshing, caller) {
|
|
459
|
+
async fetched(refreshing, caller, fetching) {
|
|
436
460
|
const { local_copy: answer } = await this.house.act('local-copies/selection/read', {});
|
|
437
461
|
const synced = new Map(answer.rooms.map((room) => [`${ROOMS}${room.room_ref}`, room.room_handle]));
|
|
438
462
|
return this.holding([...new Set([...this.copies.keys(), ...synced.keys()])].sort(), async () => {
|
|
463
|
+
const chosen = await this.launched();
|
|
439
464
|
const said = [];
|
|
440
465
|
let refused = false;
|
|
441
466
|
const mark = refreshing ? 'said' : 'unsaid';
|
|
@@ -456,7 +481,8 @@ export class LocalCopies {
|
|
|
456
481
|
continue;
|
|
457
482
|
}
|
|
458
483
|
const { unfollowed, ...followed } = held;
|
|
459
|
-
|
|
484
|
+
const named = unfollowed === undefined && held.room_handle === handle ? held : await this.save({ ...followed, room_handle: handle });
|
|
485
|
+
current.push(await this.renamed(named, chosen));
|
|
460
486
|
}
|
|
461
487
|
catch (error) {
|
|
462
488
|
refused = true;
|
|
@@ -472,7 +498,7 @@ export class LocalCopies {
|
|
|
472
498
|
try {
|
|
473
499
|
if (authority === undefined)
|
|
474
500
|
throw new Error('House serves nothing of it to copy');
|
|
475
|
-
said.push(await this.copied({ key, handle, authority }));
|
|
501
|
+
said.push(await this.copied({ key, handle, authority }, fetching, chosen));
|
|
476
502
|
}
|
|
477
503
|
catch (error) {
|
|
478
504
|
if (error instanceof NotReady) {
|
|
@@ -489,9 +515,20 @@ export class LocalCopies {
|
|
|
489
515
|
}
|
|
490
516
|
const reading = refreshing ? current : current.filter((copy) => this.waiting.has(copy.room_ref));
|
|
491
517
|
if (reading.length > 0) {
|
|
492
|
-
const
|
|
518
|
+
const reports = new Map(reading.map((copy) => [copy.key, { touched: new Map(), kept: new Set() }]));
|
|
519
|
+
const unconverted = new Map();
|
|
520
|
+
const failed = await this.refreshed(reading, unready, unconverted, caller, fetching, reports).then(() => null, houseCause);
|
|
493
521
|
for (const copy of reading) {
|
|
494
|
-
|
|
522
|
+
const cause = unconverted.get(copy.key);
|
|
523
|
+
if (cause !== undefined) {
|
|
524
|
+
refused = true;
|
|
525
|
+
unsettled.push(copy.room_ref);
|
|
526
|
+
said.push(cause);
|
|
527
|
+
}
|
|
528
|
+
else if (unready.includes(copy.room_ref))
|
|
529
|
+
said.push(`${rootOf(copy.room_handle)}: ${NOT_READY}`);
|
|
530
|
+
else
|
|
531
|
+
said.push(...(await this.reported(this.copies.get(copy.key), reports.get(copy.key))));
|
|
495
532
|
}
|
|
496
533
|
if (failed !== null) {
|
|
497
534
|
refused = true;
|
|
@@ -511,6 +548,17 @@ export class LocalCopies {
|
|
|
511
548
|
return { refused, text: said.length === 0 ? 'Nothing new from House.' : said.join('\n') };
|
|
512
549
|
});
|
|
513
550
|
}
|
|
551
|
+
async reported(copy, report) {
|
|
552
|
+
const root = rootOf(copy.room_handle);
|
|
553
|
+
let updated = 0;
|
|
554
|
+
for (const [path, before] of report.touched)
|
|
555
|
+
if (((await hashed(join(copy.repository, path)))?.sha256 ?? null) !== before)
|
|
556
|
+
updated++;
|
|
557
|
+
return [
|
|
558
|
+
...(updated === 0 ? [] : [`${root}: ${counted(updated, 'file')} updated from House`]),
|
|
559
|
+
...[...report.kept].sort().map((path) => `${root}/${path}: changed here, so House's change was not brought; the next house upload merges it`),
|
|
560
|
+
];
|
|
561
|
+
}
|
|
514
562
|
async askedAgain(rooms) {
|
|
515
563
|
let asking = rooms;
|
|
516
564
|
for (let step = 0; asking.length > 0; step++) {
|
|
@@ -519,48 +567,20 @@ export class LocalCopies {
|
|
|
519
567
|
asking = asking.filter((room) => this.waiting.has(room));
|
|
520
568
|
}
|
|
521
569
|
}
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
if (key === undefined)
|
|
525
|
-
return;
|
|
526
|
-
await this.exclusive(key, async () => {
|
|
527
|
-
const copy = this.copies.get(key);
|
|
528
|
-
let moves = copy.moves;
|
|
529
|
-
for (const { from, to } of report.moves) {
|
|
530
|
-
const held = moves.find((move) => move.worktree === report.worktree && move.to === from);
|
|
531
|
-
moves = moves.filter((move) => move !== held);
|
|
532
|
-
if (held === undefined)
|
|
533
|
-
moves.push({ worktree: report.worktree, from, to });
|
|
534
|
-
else if (held.from !== to)
|
|
535
|
-
moves.push({ ...held, to });
|
|
536
|
-
}
|
|
537
|
-
if (moves.some((move) => move.worktree === report.worktree)) {
|
|
538
|
-
const index = new Set((await git(report.worktree, ['ls-files', '-z'])).split('\0'));
|
|
539
|
-
const head = await tree(report.worktree, 'HEAD').catch(() => new Map());
|
|
540
|
-
const live = ({ from, to }) => (index.has(to) && !index.has(from)) || (head.has(to) && !head.has(from));
|
|
541
|
-
moves = moves.filter((move) => move.worktree !== report.worktree || live(move));
|
|
542
|
-
}
|
|
543
|
-
if (JSON.stringify(moves) !== JSON.stringify(copy.moves))
|
|
544
|
-
await this.save({ ...copy, moves });
|
|
545
|
-
});
|
|
570
|
+
upload(cwd, caller) {
|
|
571
|
+
return this.keyAt(cwd).then((key) => (key === undefined ? outside(cwd) : this.exclusive(key, () => this.uploaded(key, caller))));
|
|
546
572
|
}
|
|
547
|
-
async
|
|
548
|
-
const
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
|
|
557
|
-
async keyAt(repository) {
|
|
558
|
-
const real = await realpath(repository).catch(() => repository);
|
|
559
|
-
for (const copy of this.copies.values()) {
|
|
560
|
-
if ((await realpath(copy.repository).catch(() => copy.repository)) === real)
|
|
561
|
-
return copy.key;
|
|
573
|
+
async keyAt(cwd) {
|
|
574
|
+
const held = new Map();
|
|
575
|
+
for (const copy of this.copies.values())
|
|
576
|
+
held.set(await realpath(copy.repository).catch(() => copy.repository), copy.key);
|
|
577
|
+
for (let folder = await realpath(cwd).catch(() => cwd);; folder = dirname(folder)) {
|
|
578
|
+
const key = held.get(folder);
|
|
579
|
+
if (key !== undefined)
|
|
580
|
+
return key;
|
|
581
|
+
if (dirname(folder) === folder)
|
|
582
|
+
return undefined;
|
|
562
583
|
}
|
|
563
|
-
return undefined;
|
|
564
584
|
}
|
|
565
585
|
exclusive(key, work) {
|
|
566
586
|
const queued = (this.queues.get(key) ?? Promise.resolve()).then(work);
|
|
@@ -606,47 +626,108 @@ export class LocalCopies {
|
|
|
606
626
|
throw failure;
|
|
607
627
|
}
|
|
608
628
|
}
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
629
|
+
chunksFolder() {
|
|
630
|
+
return join(roomsFolder(), `${stagingPrefix()}${randomUUID()}`);
|
|
631
|
+
}
|
|
632
|
+
async setAside(path, chosen) {
|
|
633
|
+
if ((await found(path)) === null)
|
|
634
|
+
return;
|
|
635
|
+
if (chosen.some((folder) => within(folder, path)))
|
|
636
|
+
throw new Error(`${path} holds a folder an Agent works in, so Kit leaves it as it is`);
|
|
637
|
+
const kept = await keptName(path);
|
|
638
|
+
await rename(path, kept);
|
|
639
|
+
for (const copy of [...this.copies.values()])
|
|
640
|
+
if (within(copy.repository, path))
|
|
641
|
+
await this.save(rerooted(copy, path, kept));
|
|
642
|
+
}
|
|
643
|
+
async laidOut(earlier, rooms, kept) {
|
|
644
|
+
const folders = (await readdir(earlier, { withFileTypes: true }).catch(() => []))
|
|
645
|
+
.filter((entry) => entry.isDirectory() && !entry.name.startsWith('.'))
|
|
646
|
+
.map((entry) => join(earlier, entry.name));
|
|
647
|
+
const held = (folder) => [...this.copies.values()].find((copy) => copy.repository === folder);
|
|
648
|
+
const followed = (folder) => held(folder)?.unfollowed === undefined && held(folder) !== undefined;
|
|
649
|
+
const staying = folders.filter((folder) => kept.some((chosen) => within(chosen, folder)));
|
|
650
|
+
const moving = folders.filter((folder) => !staying.includes(folder));
|
|
651
|
+
await mkdir(rooms, { recursive: true, mode: 0o700 });
|
|
652
|
+
for (const source of [...moving.filter(followed), ...moving.filter((folder) => !followed(folder))]) {
|
|
653
|
+
const copy = held(source);
|
|
654
|
+
let target;
|
|
655
|
+
if (copy !== undefined && copy.unfollowed === undefined) {
|
|
656
|
+
target = join(rooms, copy.room_handle);
|
|
657
|
+
await this.setAside(target, kept);
|
|
658
|
+
}
|
|
659
|
+
else {
|
|
660
|
+
target = await keptName(join(rooms, basename(source)));
|
|
661
|
+
}
|
|
662
|
+
await carried(source, target);
|
|
663
|
+
if (copy !== undefined)
|
|
664
|
+
await this.save(rerooted(copy, source, target));
|
|
665
|
+
}
|
|
666
|
+
await sweptIn(earlier);
|
|
667
|
+
await removeFolder(earlier);
|
|
668
|
+
await removeFolder(dirname(earlier));
|
|
669
|
+
return { moved: moving.length > 0, kept: staying };
|
|
670
|
+
}
|
|
671
|
+
async renamed(copy, chosen) {
|
|
672
|
+
const folder = join(roomsFolder(), copy.room_handle);
|
|
673
|
+
if (dirname(copy.repository) !== roomsFolder() || basename(copy.repository) === copy.room_handle)
|
|
674
|
+
return copy;
|
|
675
|
+
if (chosen.some((launch) => within(launch, copy.repository)))
|
|
676
|
+
return copy;
|
|
677
|
+
await this.setAside(folder, chosen);
|
|
678
|
+
await rename(copy.repository, folder);
|
|
679
|
+
return this.save(rerooted(copy, copy.repository, folder));
|
|
680
|
+
}
|
|
681
|
+
async materialized({ key, handle, authority }, whole, chosen) {
|
|
682
|
+
const repository = join(roomsFolder(), handle);
|
|
683
|
+
await this.setAside(repository, chosen);
|
|
684
|
+
const folder = join(dirname(repository), `${stagingPrefix()}${randomUUID()}`);
|
|
685
|
+
const chunks = this.chunksFolder();
|
|
614
686
|
const scope = { write: authority.operations.write, protected: authority.protected_paths };
|
|
615
|
-
|
|
687
|
+
const manifest = { room_ref: authority.room_ref, files: {} };
|
|
688
|
+
await mkdir(folder, { recursive: true });
|
|
616
689
|
let copy;
|
|
617
690
|
try {
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
691
|
+
const read = await whole(chunks);
|
|
692
|
+
const work = await staging();
|
|
693
|
+
for await (const entry of read.entries) {
|
|
694
|
+
if (!('content' in entry)) {
|
|
695
|
+
manifest.files[entry.path] = {
|
|
696
|
+
revision: entry.revision,
|
|
697
|
+
sha256: entry.sha256,
|
|
698
|
+
bytes: entry.bytes,
|
|
699
|
+
skipped: true,
|
|
700
|
+
...(declaredWritable(entry.path, scope) ? {} : { read_only: true }),
|
|
701
|
+
};
|
|
702
|
+
continue;
|
|
703
|
+
}
|
|
704
|
+
const { path, content, revision, provenance } = entry;
|
|
705
|
+
await placed(work, join(folder, path), content);
|
|
706
|
+
manifest.files[path] = {
|
|
707
|
+
revision,
|
|
708
|
+
sha256: digest(content),
|
|
709
|
+
bytes: Buffer.byteLength(content),
|
|
710
|
+
...(declaredWritable(path, scope) ? {} : { read_only: true }),
|
|
711
|
+
};
|
|
712
|
+
if (provenance !== undefined) {
|
|
713
|
+
await placed(work, join(folder, sidecar(path)), provenance);
|
|
714
|
+
manifest.files[sidecar(path)] = { revision, sha256: digest(provenance), bytes: Buffer.byteLength(provenance), read_only: true };
|
|
715
|
+
}
|
|
716
|
+
}
|
|
717
|
+
await writeManifest(folder, work, manifest);
|
|
718
|
+
await rename(folder, repository);
|
|
719
|
+
copy = { key, room_ref: authority.room_ref, room_handle: handle, repository, position: read.position, log_epoch: read.log_epoch, ...scope };
|
|
642
720
|
}
|
|
643
721
|
catch (error) {
|
|
644
|
-
await rm(
|
|
722
|
+
await rm(folder, { recursive: true, force: true });
|
|
645
723
|
throw error;
|
|
646
724
|
}
|
|
725
|
+
finally {
|
|
726
|
+
await rm(chunks, { recursive: true, force: true });
|
|
727
|
+
}
|
|
647
728
|
return this.save(copy);
|
|
648
729
|
}
|
|
649
|
-
async copied(wanted) {
|
|
730
|
+
async copied(wanted, fetching, chosen) {
|
|
650
731
|
const { handle, authority } = wanted;
|
|
651
732
|
const root = rootOf(handle);
|
|
652
733
|
let answer = null;
|
|
@@ -668,66 +749,84 @@ export class LocalCopies {
|
|
|
668
749
|
if (answer !== null && view === undefined)
|
|
669
750
|
throw new Error('House serves nothing of it to copy');
|
|
670
751
|
if (view !== undefined && view.next === undefined) {
|
|
671
|
-
const entries = view.changes.filter((change) => '
|
|
672
|
-
const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, entries }));
|
|
752
|
+
const entries = view.changes.filter((change) => !('removed' in change));
|
|
753
|
+
const copy = await this.materialized(wanted, async () => ({ position: view.position, log_epoch: view.log_epoch, entries }), chosen);
|
|
754
|
+
await this.attached(copy, this.owner, fetching, { touched: new Map(), kept: new Set() });
|
|
673
755
|
return `${root}: copied to ${copy.repository}`;
|
|
674
756
|
}
|
|
675
|
-
const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory));
|
|
676
|
-
const
|
|
677
|
-
await
|
|
757
|
+
const copy = await this.materialized(wanted, (directory) => this.chunked(root, authority.room_ref, directory), chosen);
|
|
758
|
+
const report = { touched: new Map(), kept: new Set() };
|
|
759
|
+
const [caught] = await this.caughtUp([copy], true, this.owner, fetching, new Map([[copy.key, report]]));
|
|
760
|
+
await this.attached(caught, this.owner, fetching, report);
|
|
678
761
|
return `${root}: copied to ${copy.repository}`;
|
|
679
762
|
}
|
|
680
|
-
|
|
763
|
+
applying(copy, manifest, scope, work, report, fetching, caller) {
|
|
681
764
|
const root = rootOf(copy.room_handle);
|
|
682
|
-
|
|
765
|
+
return new Applying(copy.repository, manifest, scope, work, report, fetching, (change) => receivedBinary(work, `${root}/${change.path}`, async () => {
|
|
766
|
+
const file = await caller.info(`${root}/${change.path}`, change.revision);
|
|
767
|
+
if (file === null)
|
|
768
|
+
throw new Error(`House holds no attachment at ${root}/${change.path}`);
|
|
769
|
+
return file;
|
|
770
|
+
}));
|
|
771
|
+
}
|
|
772
|
+
async received(copy, authority, caller, fetching, report) {
|
|
773
|
+
const root = rootOf(copy.room_handle);
|
|
774
|
+
const directory = this.chunksFolder();
|
|
683
775
|
const scope = { write: authority.operations.write, protected: authority.protected_paths };
|
|
684
776
|
try {
|
|
685
777
|
const read = await this.chunked(root, copy.room_ref, directory);
|
|
686
|
-
const
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
if (blob !== null)
|
|
702
|
-
blobs.set(entry.path, blob);
|
|
778
|
+
const manifest = await readManifest(copy.repository);
|
|
779
|
+
const work = await staging();
|
|
780
|
+
const applying = this.applying(copy, manifest, scope, work, report, fetching, caller);
|
|
781
|
+
await applying.start();
|
|
782
|
+
try {
|
|
783
|
+
const whole = new Set();
|
|
784
|
+
for await (const entry of read.entries) {
|
|
785
|
+
whole.add(entry.path);
|
|
786
|
+
if (!('content' in entry)) {
|
|
787
|
+
applying.binary(entry);
|
|
788
|
+
continue;
|
|
789
|
+
}
|
|
790
|
+
if (entry.provenance !== undefined)
|
|
791
|
+
whole.add(sidecar(entry.path));
|
|
792
|
+
await applying.text(entry);
|
|
703
793
|
}
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
read_only: readOnly.held(),
|
|
714
|
-
manifests: { ...copy.manifests, [house]: manifest(null, revisions, binary) },
|
|
715
|
-
};
|
|
716
|
-
await ignoreRules(copy.repository, Object.keys(revisions), next);
|
|
717
|
-
return next;
|
|
794
|
+
for (const path of Object.keys(manifest.files))
|
|
795
|
+
if (!whole.has(path))
|
|
796
|
+
applying.removal(path);
|
|
797
|
+
await applying.finish();
|
|
798
|
+
}
|
|
799
|
+
finally {
|
|
800
|
+
await writeManifest(copy.repository, work, manifest);
|
|
801
|
+
}
|
|
802
|
+
return { ...copy, ...scope, position: read.position, log_epoch: read.log_epoch };
|
|
718
803
|
}
|
|
719
804
|
finally {
|
|
720
805
|
await rm(directory, { recursive: true, force: true });
|
|
721
806
|
}
|
|
722
807
|
}
|
|
723
|
-
async reread(copy, caller) {
|
|
808
|
+
async reread(copy, caller, fetching, report) {
|
|
724
809
|
const { authorities } = await this.discovered([copy.room_handle]);
|
|
725
810
|
const authority = authorities.find((entry) => entry.room_ref === copy.room_ref);
|
|
726
811
|
if (authority === undefined)
|
|
727
812
|
throw new NotReady([copy.room_ref]);
|
|
728
|
-
return this.received(copy, authority, caller);
|
|
813
|
+
return this.received(copy, authority, caller, fetching, report);
|
|
729
814
|
}
|
|
730
|
-
async refreshed(
|
|
815
|
+
async refreshed(held, unready, unconverted, caller, fetching, reports) {
|
|
816
|
+
const copies = [];
|
|
817
|
+
for (const copy of held) {
|
|
818
|
+
try {
|
|
819
|
+
copies.push(await this.converted(copy));
|
|
820
|
+
}
|
|
821
|
+
catch (error) {
|
|
822
|
+
if (error instanceof NotReady)
|
|
823
|
+
unready.push(...error.rooms);
|
|
824
|
+
else
|
|
825
|
+
unconverted.set(copy.key, causeOf(error));
|
|
826
|
+
}
|
|
827
|
+
}
|
|
828
|
+
if (copies.length === 0)
|
|
829
|
+
return;
|
|
731
830
|
const { bounds, authorities } = await this.discovered(copies.map((copy) => copy.room_handle));
|
|
732
831
|
const read = [];
|
|
733
832
|
const unchanged = [];
|
|
@@ -736,7 +835,7 @@ export class LocalCopies {
|
|
|
736
835
|
if (authority !== undefined &&
|
|
737
836
|
JSON.stringify([authority.operations.write, authority.protected_paths]) !== JSON.stringify([copy.write, copy.protected])) {
|
|
738
837
|
try {
|
|
739
|
-
read.push(await this.save(await this.received(copy, authority, caller)));
|
|
838
|
+
read.push(await this.save(await this.received(copy, authority, caller, fetching, reports.get(copy.key))));
|
|
740
839
|
}
|
|
741
840
|
catch (error) {
|
|
742
841
|
if (!(error instanceof NotReady))
|
|
@@ -749,11 +848,42 @@ export class LocalCopies {
|
|
|
749
848
|
}
|
|
750
849
|
}
|
|
751
850
|
for (const group of grouped(unchanged, bounds.max_prefixes))
|
|
752
|
-
await this.caughtUp(group, false, caller, unready);
|
|
851
|
+
await this.caughtUp(group, false, caller, fetching, reports, unready);
|
|
753
852
|
for (const group of grouped(read, bounds.max_prefixes))
|
|
754
|
-
await this.caughtUp(group, true, caller, unready);
|
|
853
|
+
await this.caughtUp(group, true, caller, fetching, reports, unready);
|
|
854
|
+
for (const copy of copies) {
|
|
855
|
+
if (!unready.includes(copy.room_ref))
|
|
856
|
+
await this.attached(this.copies.get(copy.key), caller, fetching, reports.get(copy.key));
|
|
857
|
+
}
|
|
858
|
+
}
|
|
859
|
+
async attached(copy, caller, fetching, report) {
|
|
860
|
+
if (!fetching.attachments)
|
|
861
|
+
return;
|
|
862
|
+
const manifest = await readManifest(copy.repository);
|
|
863
|
+
const wanted = Object.entries(manifest.files).filter(([, entry]) => entry.skipped && entry.bytes <= fetching.maxSize * MEGABYTE);
|
|
864
|
+
if (wanted.length === 0)
|
|
865
|
+
return;
|
|
866
|
+
const root = rootOf(copy.room_handle);
|
|
867
|
+
const work = await staging();
|
|
868
|
+
const present = await scanned(copy.repository);
|
|
869
|
+
for (const [path, { skipped: _skipped, ...entry }] of wanted) {
|
|
870
|
+
if (present.has(path))
|
|
871
|
+
continue;
|
|
872
|
+
const file = await receivedBinary(work, `${root}/${path}`, async () => {
|
|
873
|
+
const held = await caller.info(`${root}/${path}`, entry.revision);
|
|
874
|
+
if (held === null)
|
|
875
|
+
throw new Error(`House holds no attachment at ${root}/${path}`);
|
|
876
|
+
return held;
|
|
877
|
+
});
|
|
878
|
+
if (!report.touched.has(path))
|
|
879
|
+
report.touched.set(path, null);
|
|
880
|
+
await mkdir(dirname(join(copy.repository, path)), { recursive: true });
|
|
881
|
+
await rename(file, join(copy.repository, path));
|
|
882
|
+
manifest.files[path] = entry;
|
|
883
|
+
}
|
|
884
|
+
await writeManifest(copy.repository, work, manifest);
|
|
755
885
|
}
|
|
756
|
-
async caughtUp(copies, whole, caller, unready) {
|
|
886
|
+
async caughtUp(copies, whole, caller, fetching, reports, unready) {
|
|
757
887
|
const caught = new Map(copies.map((copy) => [copy.key, copy]));
|
|
758
888
|
const read = new Set();
|
|
759
889
|
const waiting = unready ?? [];
|
|
@@ -777,7 +907,7 @@ export class LocalCopies {
|
|
|
777
907
|
}
|
|
778
908
|
read.add(expired.key);
|
|
779
909
|
try {
|
|
780
|
-
const next = await this.save(await this.reread(expired, caller));
|
|
910
|
+
const next = await this.save(await this.reread(expired, caller, fetching, reports.get(expired.key)));
|
|
781
911
|
caught.set(expired.key, next);
|
|
782
912
|
asking.push(next);
|
|
783
913
|
}
|
|
@@ -793,7 +923,7 @@ export class LocalCopies {
|
|
|
793
923
|
const view = answer.authorities.find((entry) => entry.authority === copy.room_ref);
|
|
794
924
|
if (view === undefined || (view.position === copy.position && view.log_epoch === copy.log_epoch))
|
|
795
925
|
continue;
|
|
796
|
-
const next = await this.save(await this.incorporated(copy, view
|
|
926
|
+
const next = await this.save(await this.incorporated(copy, view, caller, fetching, reports.get(copy.key)));
|
|
797
927
|
caught.set(copy.key, next);
|
|
798
928
|
advanced.push(next);
|
|
799
929
|
}
|
|
@@ -803,558 +933,589 @@ export class LocalCopies {
|
|
|
803
933
|
throw new NotReady(waiting);
|
|
804
934
|
return copies.map((copy) => caught.get(copy.key));
|
|
805
935
|
}
|
|
806
|
-
async
|
|
807
|
-
const
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
const named = `${rootOf(copy.room_handle)}/${path}`;
|
|
827
|
-
let answered = first;
|
|
828
|
-
return receivedBinary(copy.repository, named, async () => {
|
|
829
|
-
const file = answered ?? (await caller.info(named, revision));
|
|
830
|
-
answered = null;
|
|
831
|
-
if (file === null)
|
|
832
|
-
throw new Error(`House holds no attachment at ${named}`);
|
|
833
|
-
return file;
|
|
834
|
-
});
|
|
936
|
+
async incorporated(copy, view, caller, fetching, report) {
|
|
937
|
+
const manifest = await readManifest(copy.repository);
|
|
938
|
+
const work = await staging();
|
|
939
|
+
const applying = this.applying(copy, manifest, copy, work, report, fetching, caller);
|
|
940
|
+
await applying.start();
|
|
941
|
+
try {
|
|
942
|
+
for (const change of view.changes) {
|
|
943
|
+
if ('removed' in change)
|
|
944
|
+
applying.removal(change.path);
|
|
945
|
+
else if ('content' in change)
|
|
946
|
+
await applying.text(change);
|
|
947
|
+
else
|
|
948
|
+
applying.binary(change);
|
|
949
|
+
}
|
|
950
|
+
await applying.finish();
|
|
951
|
+
}
|
|
952
|
+
finally {
|
|
953
|
+
await writeManifest(copy.repository, work, manifest);
|
|
954
|
+
}
|
|
955
|
+
return { ...copy, position: view.position, log_epoch: view.log_epoch };
|
|
835
956
|
}
|
|
836
|
-
async
|
|
837
|
-
|
|
838
|
-
|
|
839
|
-
const
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
|
|
843
|
-
|
|
844
|
-
|
|
845
|
-
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
|
|
849
|
-
|
|
850
|
-
|
|
851
|
-
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
|
|
857
|
-
|
|
858
|
-
|
|
957
|
+
async converted(held) {
|
|
958
|
+
if (!earlier(held))
|
|
959
|
+
return held;
|
|
960
|
+
const copy = {
|
|
961
|
+
key: held.key,
|
|
962
|
+
room_ref: held.room_ref,
|
|
963
|
+
room_handle: held.room_handle,
|
|
964
|
+
repository: held.repository,
|
|
965
|
+
position: held.position,
|
|
966
|
+
log_epoch: held.log_epoch,
|
|
967
|
+
write: held.write,
|
|
968
|
+
protected: held.protected,
|
|
969
|
+
...(held.unfollowed === undefined ? {} : { unfollowed: held.unfollowed }),
|
|
970
|
+
};
|
|
971
|
+
const root = rootOf(copy.room_handle);
|
|
972
|
+
const directory = this.chunksFolder();
|
|
973
|
+
const recorded = recordedFiles(held);
|
|
974
|
+
const manifest = { room_ref: copy.room_ref, files: {} };
|
|
975
|
+
const work = await staging();
|
|
976
|
+
try {
|
|
977
|
+
const read = await this.chunked(root, copy.room_ref, directory);
|
|
978
|
+
const present = await scanned(copy.repository);
|
|
979
|
+
const current = new Map();
|
|
980
|
+
const unbrought = new Map();
|
|
981
|
+
const binaries = [];
|
|
982
|
+
const readOnly = async (path, content, revision) => {
|
|
983
|
+
if (present.get(path)?.sha256 !== digest(content))
|
|
984
|
+
await placed(work, join(copy.repository, path), content);
|
|
985
|
+
manifest.files[path] = { revision, sha256: digest(content), bytes: Buffer.byteLength(content), read_only: true };
|
|
986
|
+
};
|
|
987
|
+
for await (const entry of read.entries) {
|
|
988
|
+
if (!('content' in entry)) {
|
|
989
|
+
binaries.push(entry);
|
|
859
990
|
continue;
|
|
860
991
|
}
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
continue;
|
|
992
|
+
if (entry.provenance !== undefined)
|
|
993
|
+
await readOnly(sidecar(entry.path), entry.provenance, entry.revision);
|
|
994
|
+
if (declaredWritable(entry.path, copy)) {
|
|
995
|
+
current.set(entry.path, { revision: entry.revision, sha256: digest(entry.content), bytes: Buffer.byteLength(entry.content) });
|
|
996
|
+
if (!present.has(entry.path))
|
|
997
|
+
unbrought.set(entry.path, entry.content);
|
|
998
|
+
}
|
|
999
|
+
else {
|
|
1000
|
+
await readOnly(entry.path, entry.content, entry.revision);
|
|
1001
|
+
}
|
|
872
1002
|
}
|
|
873
|
-
const
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1003
|
+
for (const [path, { revision, held: attached }] of recorded) {
|
|
1004
|
+
if (manifest.files[path]?.read_only)
|
|
1005
|
+
continue;
|
|
1006
|
+
if (attached !== undefined) {
|
|
1007
|
+
manifest.files[path] = present.has(path) ? { revision, ...attached } : { revision, ...attached, skipped: true };
|
|
1008
|
+
continue;
|
|
1009
|
+
}
|
|
1010
|
+
const named = `${root}/${path}`;
|
|
1011
|
+
const now = current.get(path);
|
|
1012
|
+
const read = now?.revision === revision ? null : await this.revisionRead(named, revision);
|
|
1013
|
+
if (now?.revision !== revision && read === null)
|
|
880
1014
|
continue;
|
|
881
|
-
|
|
882
|
-
|
|
1015
|
+
const latest = now?.revision === revision ? now : { revision, sha256: digest(read), bytes: Buffer.byteLength(read) };
|
|
1016
|
+
const local = present.get(path);
|
|
1017
|
+
if (local === undefined) {
|
|
1018
|
+
await placed(work, join(copy.repository, path), read ?? unbrought.get(path));
|
|
1019
|
+
manifest.files[path] = latest;
|
|
1020
|
+
continue;
|
|
1021
|
+
}
|
|
1022
|
+
if (local.sha256 === latest.sha256) {
|
|
1023
|
+
manifest.files[path] = latest;
|
|
1024
|
+
continue;
|
|
1025
|
+
}
|
|
1026
|
+
const edited = await readFile(join(copy.repository, local.name), 'utf8');
|
|
1027
|
+
const latestText = read ?? (await this.revisionText(named, revision));
|
|
1028
|
+
let closest = { revision, text: latestText, apart: differing(latestText, edited) };
|
|
1029
|
+
for (const before of recordedRevisions(held, path).filter((other) => other !== revision)) {
|
|
1030
|
+
const text = await this.revisionRead(named, before);
|
|
1031
|
+
if (text === null)
|
|
1032
|
+
continue;
|
|
1033
|
+
const apart = differing(text, edited);
|
|
1034
|
+
if (apart < closest.apart)
|
|
1035
|
+
closest = { revision: before, text, apart };
|
|
1036
|
+
}
|
|
1037
|
+
if (closest.revision !== revision && closest.text === edited) {
|
|
1038
|
+
await placed(work, join(copy.repository, local.name), latestText);
|
|
1039
|
+
manifest.files[path] = latest;
|
|
1040
|
+
}
|
|
1041
|
+
else {
|
|
1042
|
+
manifest.files[path] = { revision: closest.revision, sha256: digest(closest.text), bytes: Buffer.byteLength(closest.text) };
|
|
1043
|
+
}
|
|
883
1044
|
}
|
|
884
|
-
|
|
885
|
-
|
|
1045
|
+
for (const [path, now] of current) {
|
|
1046
|
+
if (manifest.files[path] === undefined && present.get(path)?.sha256 === now.sha256)
|
|
1047
|
+
manifest.files[path] = now;
|
|
886
1048
|
}
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
1049
|
+
for (const { path, revision, bytes, sha256 } of binaries) {
|
|
1050
|
+
if (manifest.files[path] !== undefined)
|
|
1051
|
+
continue;
|
|
1052
|
+
const writable = declaredWritable(path, copy) ? {} : { read_only: true };
|
|
1053
|
+
manifest.files[path] = present.get(path)?.sha256 === sha256 ? { revision, sha256, bytes, ...writable } : { revision, sha256, bytes, skipped: true, ...writable };
|
|
891
1054
|
}
|
|
1055
|
+
for (const [path, local] of present) {
|
|
1056
|
+
if (manifest.files[path] !== undefined || current.has(path))
|
|
1057
|
+
continue;
|
|
1058
|
+
if (await this.deletedByHouse(held, `${root}/${path}`, path, local.sha256))
|
|
1059
|
+
await removed(copy.repository, local.name);
|
|
1060
|
+
}
|
|
1061
|
+
for (const path of held.read_only)
|
|
1062
|
+
if (manifest.files[path] === undefined)
|
|
1063
|
+
await removed(copy.repository, path);
|
|
1064
|
+
for (const name of [...KIT_IGNORES, ...KIT_GIT])
|
|
1065
|
+
await rm(join(copy.repository, name), { recursive: true, force: true });
|
|
1066
|
+
await rmdir(join(copy.repository, '.git', 'refs', 'house')).catch(() => undefined);
|
|
1067
|
+
await writeManifest(copy.repository, work, manifest);
|
|
892
1068
|
}
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
]);
|
|
898
|
-
manifests = { ...manifests, [house]: manifest(copy.house, revisions, binary) };
|
|
1069
|
+
catch (error) {
|
|
1070
|
+
if (error instanceof NotReady)
|
|
1071
|
+
throw error;
|
|
1072
|
+
throw new Error(`${root}: Kit could not convert the copy an earlier Kit kept yet (${houseCause(error)}); the next house download or house upload converts it`);
|
|
899
1073
|
}
|
|
900
|
-
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
const
|
|
914
|
-
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
if (
|
|
919
|
-
return
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
});
|
|
924
|
-
if (replaced.refused)
|
|
925
|
-
return replaced;
|
|
1074
|
+
finally {
|
|
1075
|
+
await rm(directory, { recursive: true, force: true });
|
|
1076
|
+
}
|
|
1077
|
+
return this.save(copy);
|
|
1078
|
+
}
|
|
1079
|
+
async uploaded(key, caller) {
|
|
1080
|
+
let copy;
|
|
1081
|
+
try {
|
|
1082
|
+
copy = await this.converted(this.copies.get(key));
|
|
1083
|
+
}
|
|
1084
|
+
catch (error) {
|
|
1085
|
+
return { refused: true, text: error instanceof NotReady ? `${rootOf(this.copies.get(key).room_handle)}: ${NOT_READY}` : causeOf(error) };
|
|
1086
|
+
}
|
|
1087
|
+
const lines = [];
|
|
1088
|
+
let files = 0;
|
|
1089
|
+
let landed = 0;
|
|
1090
|
+
if (copy.pending !== undefined) {
|
|
1091
|
+
const settled = await this.settle(copy, caller);
|
|
1092
|
+
if (settled.refused)
|
|
1093
|
+
return settled;
|
|
1094
|
+
files += settled.files;
|
|
1095
|
+
landed++;
|
|
1096
|
+
lines.push(...settled.lines);
|
|
926
1097
|
copy = this.copies.get(key);
|
|
927
1098
|
}
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
1099
|
+
const root = rootOf(copy.room_handle);
|
|
1100
|
+
const manifest = await readManifest(copy.repository);
|
|
1101
|
+
const present = await scanned(copy.repository);
|
|
1102
|
+
const unresolved = [];
|
|
1103
|
+
for (const [path, entry] of Object.entries(manifest.files)) {
|
|
1104
|
+
if (entry.conflicted && (await marked(copy.repository, present.get(path)?.name ?? path)))
|
|
1105
|
+
unresolved.push(`${root}/${path}`);
|
|
934
1106
|
}
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
return { refused: true, text: `${short(commit)} holds no House state; merge or rebase onto ${HOUSE_REF}` };
|
|
1107
|
+
if (unresolved.length > 0) {
|
|
1108
|
+
const [named, they] = unresolved.length === 1 ? [`${unresolved[0]} still holds`, 'it'] : [`${unresolved.join(', ')} still hold`, 'each'];
|
|
1109
|
+
return { refused: true, text: `${named} conflict markers; resolve ${they}, then run house upload again` };
|
|
939
1110
|
}
|
|
940
|
-
let
|
|
1111
|
+
let built;
|
|
941
1112
|
try {
|
|
942
|
-
|
|
943
|
-
if (changesKind(changes)) {
|
|
944
|
-
const read = (await this.caughtUp([copy], true, caller))[0];
|
|
945
|
-
if (read.house !== copy.house)
|
|
946
|
-
changes = await this.derived(read, base, commit);
|
|
947
|
-
copy = read;
|
|
948
|
-
}
|
|
1113
|
+
built = await this.built(copy, manifest, present);
|
|
949
1114
|
}
|
|
950
1115
|
catch (error) {
|
|
951
1116
|
return { refused: true, text: houseCause(error) };
|
|
952
1117
|
}
|
|
953
|
-
if (
|
|
954
|
-
return
|
|
955
|
-
const
|
|
956
|
-
const
|
|
957
|
-
|
|
958
|
-
|
|
959
|
-
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
house: copy.house,
|
|
968
|
-
},
|
|
969
|
-
});
|
|
970
|
-
return this.settle(submitted, caller);
|
|
971
|
-
}
|
|
972
|
-
async attachmentsOf(copy, arriving) {
|
|
973
|
-
const documents = arriving.filter(([path]) => isDocument(path));
|
|
974
|
-
if (documents.length > 0) {
|
|
975
|
-
const { bounds } = await this.house.door('rooms/discover', { prefixes: [rootOf(copy.room_handle)] });
|
|
976
|
-
const refused = await untext(copy.repository, documents, bounds.max_authored_file_bytes);
|
|
977
|
-
if (refused.length > 0)
|
|
978
|
-
throw new Error(markdownOnly(refused, bounds.max_authored_file_bytes));
|
|
1118
|
+
if ('refused' in built)
|
|
1119
|
+
return built;
|
|
1120
|
+
const batches = batched(built.changes);
|
|
1121
|
+
for (const [index, changes] of batches.entries()) {
|
|
1122
|
+
const read = Object.fromEntries(changedPaths(changes).flatMap((path) => (built.read[relative(path)] === undefined ? [] : [[relative(path), built.read[relative(path)]]])));
|
|
1123
|
+
copy = await this.save({ ...this.copies.get(key), pending: { operation_id: operationId(), caller: caller.key, changes, read } });
|
|
1124
|
+
const settled = await this.settle(copy, caller);
|
|
1125
|
+
if (settled.refused) {
|
|
1126
|
+
const done = landed === 0 ? [] : [`House accepted ${counted(landed, 'batch', 'batches')} of ${rootOf(copy.room_handle)} before this one.`];
|
|
1127
|
+
return { refused: true, text: [...done, ...lines, settled.text, ...(index + 1 < batches.length ? [`${counted(batches.length - index - 1, 'batch', 'batches')} after it ${batches.length - index - 1 === 1 ? 'was' : 'were'} not sent`] : [])].join('\n') };
|
|
1128
|
+
}
|
|
1129
|
+
files += settled.files;
|
|
1130
|
+
landed++;
|
|
1131
|
+
lines.push(...settled.lines);
|
|
979
1132
|
}
|
|
980
|
-
|
|
1133
|
+
const now = rootOf(this.copies.get(key).room_handle);
|
|
1134
|
+
if (landed === 0)
|
|
1135
|
+
return { refused: false, text: `Nothing to upload: ${root} matches House.` };
|
|
1136
|
+
return {
|
|
1137
|
+
refused: false,
|
|
1138
|
+
text: [`House accepted ${counted(files, 'file')} of ${now}${landed > 1 ? ` in ${landed} batches` : ''}.`, ...lines].join('\n'),
|
|
1139
|
+
};
|
|
981
1140
|
}
|
|
982
|
-
async
|
|
1141
|
+
async built(copy, manifest, present) {
|
|
983
1142
|
const root = rootOf(copy.room_handle);
|
|
984
|
-
const
|
|
985
|
-
const
|
|
986
|
-
const
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1143
|
+
const added = [];
|
|
1144
|
+
const changed = [];
|
|
1145
|
+
const gone = [];
|
|
1146
|
+
let adopted = false;
|
|
1147
|
+
for (const [path, file] of present) {
|
|
1148
|
+
const entry = manifest.files[path];
|
|
1149
|
+
if (kitsOwn(path))
|
|
1150
|
+
continue;
|
|
1151
|
+
if (entry === undefined)
|
|
1152
|
+
added.push(path);
|
|
1153
|
+
else if (entry.read_only)
|
|
1154
|
+
continue;
|
|
1155
|
+
else if (entry.sha256 !== file.sha256)
|
|
1156
|
+
changed.push(path);
|
|
1157
|
+
else if (entry.skipped) {
|
|
1158
|
+
manifest.files[path] = { revision: entry.revision, sha256: entry.sha256, bytes: entry.bytes };
|
|
1159
|
+
adopted = true;
|
|
1160
|
+
}
|
|
1161
|
+
}
|
|
1162
|
+
for (const [path, entry] of Object.entries(manifest.files)) {
|
|
1163
|
+
if (!kitsOwn(path) && !entry.read_only && !entry.skipped && !present.has(path))
|
|
1164
|
+
gone.push(path);
|
|
1165
|
+
}
|
|
1166
|
+
if (adopted)
|
|
1167
|
+
await writeManifest(copy.repository, await staging(), manifest);
|
|
1168
|
+
const listed = (paths, file) => {
|
|
1169
|
+
const found = new Map();
|
|
1170
|
+
for (const path of paths)
|
|
1171
|
+
found.set(sameBytes(file(path)), [...(found.get(sameBytes(file(path))) ?? []), path]);
|
|
1172
|
+
return found;
|
|
1000
1173
|
};
|
|
1001
|
-
const
|
|
1174
|
+
const leaving = listed(gone, (path) => manifest.files[path]);
|
|
1175
|
+
const arriving = listed(added, (path) => present.get(path));
|
|
1176
|
+
const moves = new Map();
|
|
1177
|
+
for (const [held, [from, ...others]] of leaving) {
|
|
1178
|
+
const [to, ...more] = arriving.get(held) ?? [];
|
|
1179
|
+
if (others.length === 0 && more.length === 0 && to !== undefined && isDocument(from) === isDocument(to))
|
|
1180
|
+
moves.set(from, to);
|
|
1181
|
+
}
|
|
1182
|
+
const arrived = new Set(moves.values());
|
|
1183
|
+
const documents = [...added.filter((path) => !arrived.has(path)), ...changed].filter(isDocument).sort();
|
|
1184
|
+
const texts = new Map();
|
|
1185
|
+
if (documents.length > 0) {
|
|
1186
|
+
const { bounds } = await this.house.door('rooms/discover', { prefixes: [root] });
|
|
1187
|
+
const refused = [];
|
|
1188
|
+
for (const path of documents) {
|
|
1189
|
+
const bytes = await readFile(join(copy.repository, present.get(path).name));
|
|
1190
|
+
if (bytes.length > bounds.max_authored_file_bytes || !isUtf8(bytes) || bytes.includes(0))
|
|
1191
|
+
refused.push(path);
|
|
1192
|
+
else
|
|
1193
|
+
texts.set(path, bytes.toString('utf8'));
|
|
1194
|
+
}
|
|
1195
|
+
if (refused.length > 0)
|
|
1196
|
+
return { refused: true, text: markdownOnly(refused, bounds.max_authored_file_bytes) };
|
|
1197
|
+
}
|
|
1002
1198
|
const changes = [];
|
|
1003
|
-
|
|
1199
|
+
const read = {};
|
|
1200
|
+
for (const path of [...new Set([...added, ...changed, ...gone])].sort()) {
|
|
1004
1201
|
if (arrived.has(path))
|
|
1005
1202
|
continue;
|
|
1006
|
-
const
|
|
1007
|
-
const
|
|
1008
|
-
const
|
|
1009
|
-
const to = moved.get(path);
|
|
1203
|
+
const named = `${root}/${path}`;
|
|
1204
|
+
const entry = manifest.files[path];
|
|
1205
|
+
const to = moves.get(path);
|
|
1010
1206
|
if (to !== undefined) {
|
|
1011
|
-
changes.push({ op: '
|
|
1012
|
-
|
|
1013
|
-
const part = binary.get(to);
|
|
1014
|
-
if (moving !== prior) {
|
|
1015
|
-
changes.push(part === undefined
|
|
1016
|
-
? { op: 'replace', path: `${root}/${to}`, base: known, content: await text(moving) }
|
|
1017
|
-
: { op: 'upload', path: `${root}/${to}`, base: known, ...part });
|
|
1018
|
-
}
|
|
1207
|
+
changes.push({ op: 'move', path: named, to: `${root}/${to}`, base: entry.revision });
|
|
1208
|
+
read[to] = { name: present.get(to).name, sha256: present.get(to).sha256 };
|
|
1019
1209
|
continue;
|
|
1020
1210
|
}
|
|
1021
|
-
|
|
1211
|
+
const file = present.get(path);
|
|
1212
|
+
if (file === undefined) {
|
|
1213
|
+
changes.push({ op: 'delete', path: named, base: entry.revision });
|
|
1022
1214
|
continue;
|
|
1023
|
-
const part = binary.get(path);
|
|
1024
|
-
if (next === undefined)
|
|
1025
|
-
changes.push({ op: 'remove', path: `${root}/${path}`, base: known });
|
|
1026
|
-
else if (prior === undefined)
|
|
1027
|
-
changes.push(await created(path, next));
|
|
1028
|
-
else if (part !== undefined && houseText(path)) {
|
|
1029
|
-
changes.push({ op: 'remove', path: `${root}/${path}`, base: known }, { op: 'upload', path: `${root}/${path}`, ...part });
|
|
1030
1215
|
}
|
|
1031
|
-
|
|
1032
|
-
|
|
1033
|
-
|
|
1034
|
-
changes.push(
|
|
1216
|
+
const base = entry === undefined ? {} : { base: entry.revision };
|
|
1217
|
+
const text = texts.get(path);
|
|
1218
|
+
if (text !== undefined) {
|
|
1219
|
+
changes.push({ op: 'write', path: named, content: text, ...base });
|
|
1220
|
+
read[path] = { name: file.name, sha256: digest(text) };
|
|
1221
|
+
}
|
|
1222
|
+
else {
|
|
1223
|
+
changes.push({ op: 'upload', path: named, ...base, bytes: file.bytes, sha256: file.sha256, media_type: mediaType(path) });
|
|
1224
|
+
read[path] = { name: file.name, sha256: file.sha256 };
|
|
1225
|
+
}
|
|
1035
1226
|
}
|
|
1036
|
-
return changes;
|
|
1227
|
+
return { changes, read };
|
|
1037
1228
|
}
|
|
1038
1229
|
async settle(start, caller, restaged = false) {
|
|
1039
1230
|
let copy = start;
|
|
1040
|
-
if (copy.
|
|
1041
|
-
|
|
1231
|
+
if (copy.pending.caller !== caller.key) {
|
|
1232
|
+
const { upload: _upload, ...pending } = copy.pending;
|
|
1233
|
+
copy = await this.save({ ...copy, pending: { ...pending, caller: caller.key } });
|
|
1042
1234
|
}
|
|
1043
1235
|
const finished = new Set();
|
|
1236
|
+
let rehandledOnce = false;
|
|
1044
1237
|
let outcome;
|
|
1045
1238
|
for (;;) {
|
|
1046
|
-
const
|
|
1239
|
+
const pending = copy.pending;
|
|
1047
1240
|
try {
|
|
1048
|
-
outcome = await caller.submit(
|
|
1049
|
-
copy = await this.save({ ...copy,
|
|
1241
|
+
outcome = await caller.submit(pending.operation_id, pending.changes, pending.upload, async (upload) => {
|
|
1242
|
+
copy = await this.save({ ...copy, pending: { ...copy.pending, upload } });
|
|
1050
1243
|
});
|
|
1051
1244
|
}
|
|
1052
1245
|
catch (error) {
|
|
1053
1246
|
const refusal = error instanceof HouseRefusal ? refusalOf(error.text) : null;
|
|
1054
|
-
return {
|
|
1055
|
-
refused: true,
|
|
1056
|
-
text: refusal?.text ?? `House's answer to ${short(push.commit)} did not arrive; run house push again`,
|
|
1057
|
-
};
|
|
1247
|
+
return { refused: true, text: refusal?.text ?? "House's answer to this upload did not arrive; run house upload again" };
|
|
1058
1248
|
}
|
|
1059
|
-
if (!('refused' in outcome)
|
|
1249
|
+
if (!('refused' in outcome))
|
|
1060
1250
|
break;
|
|
1061
|
-
|
|
1062
|
-
|
|
1251
|
+
if (outcome.refused.code === 'files_unstaged') {
|
|
1252
|
+
const named = unstagedUploads(outcome.refused).filter(({ id }) => !finished.has(id));
|
|
1253
|
+
if (named.length === 0)
|
|
1254
|
+
break;
|
|
1255
|
+
for (const { id } of named)
|
|
1256
|
+
finished.add(id);
|
|
1257
|
+
const failed = await this.sentParts(copy, named, caller);
|
|
1258
|
+
copy = this.copies.get(copy.key);
|
|
1259
|
+
if (failed.size > 0)
|
|
1260
|
+
return this.stopped(copy, failed);
|
|
1261
|
+
continue;
|
|
1262
|
+
}
|
|
1263
|
+
const handle = outcome.refused.code === 'handle_changed' && !rehandledOnce ? currentHandle(outcome.refused) : null;
|
|
1264
|
+
if (handle === null)
|
|
1063
1265
|
break;
|
|
1064
|
-
|
|
1065
|
-
|
|
1066
|
-
const failed = await this.sentUploads(copy, named, caller);
|
|
1067
|
-
copy = this.copies.get(copy.key);
|
|
1068
|
-
if (failed.size > 0)
|
|
1069
|
-
return this.stopped(copy, failed);
|
|
1266
|
+
rehandledOnce = true;
|
|
1267
|
+
copy = await this.save(rehandled(copy, handle));
|
|
1070
1268
|
}
|
|
1071
1269
|
if ('refused' in outcome) {
|
|
1072
1270
|
if (stopsPush(outcome.refused))
|
|
1073
1271
|
return this.stopped(copy, firstLine(outcome.refused));
|
|
1074
|
-
if (outcome.refused.code === 'upload_unavailable' && copy.
|
|
1075
|
-
const { upload, ...unstaged } = copy.
|
|
1076
|
-
return this.settle(await this.save({ ...copy,
|
|
1077
|
-
}
|
|
1078
|
-
if (outcome.refused.code === 'expired') {
|
|
1079
|
-
return this.recovered(copy, caller, async (current) => {
|
|
1080
|
-
await this.save(without(current));
|
|
1081
|
-
return { refused: true, text: `House does not hold ${short(current.push.commit)}; run house push again` };
|
|
1082
|
-
});
|
|
1272
|
+
if (outcome.refused.code === 'upload_unavailable' && copy.pending.upload !== undefined && !restaged) {
|
|
1273
|
+
const { upload: _upload, ...unstaged } = copy.pending;
|
|
1274
|
+
return this.settle(await this.save({ ...copy, pending: unstaged }), caller, true);
|
|
1083
1275
|
}
|
|
1084
|
-
|
|
1085
|
-
if (
|
|
1276
|
+
const settled = await this.save(without(copy));
|
|
1277
|
+
if (outcome.refused.code !== 'edit_conflict')
|
|
1086
1278
|
return { refused: true, text: outcome.refused.text };
|
|
1087
|
-
const
|
|
1088
|
-
|
|
1089
|
-
settled = (await this.caughtUp([settled], true, caller))[0];
|
|
1090
|
-
await this.save(await this.conflicted(settled, outcome.refused, copy.push, caller));
|
|
1091
|
-
}
|
|
1092
|
-
catch (error) {
|
|
1093
|
-
return {
|
|
1094
|
-
refused: true,
|
|
1095
|
-
text: rebased ? `${outcome.refused.text}\nHouse's current state did not arrive: ${houseCause(error)}` : outcome.refused.text,
|
|
1096
|
-
};
|
|
1097
|
-
}
|
|
1098
|
-
return {
|
|
1099
|
-
refused: true,
|
|
1100
|
-
text: rebased
|
|
1101
|
-
? `${outcome.refused.text}\n${HOUSE_REF} holds House's current state; rebase onto it and push again`
|
|
1102
|
-
: outcome.refused.text,
|
|
1103
|
-
};
|
|
1279
|
+
const lines = await this.conflicted(settled, copy.pending, outcome.refused, caller);
|
|
1280
|
+
return { refused: true, text: [outcome.refused.text, ...lines].join('\n') };
|
|
1104
1281
|
}
|
|
1105
|
-
return this.
|
|
1282
|
+
return this.integrated(copy, outcome.accepted);
|
|
1106
1283
|
}
|
|
1107
|
-
async
|
|
1284
|
+
async sentParts(start, named, caller) {
|
|
1108
1285
|
let copy = start;
|
|
1109
|
-
const
|
|
1110
|
-
const
|
|
1111
|
-
const staged = new Set(
|
|
1286
|
+
const pending = copy.pending;
|
|
1287
|
+
const parts = new Map(pending.changes.filter((change) => change.op === 'upload').map((part) => [part.sha256, part]));
|
|
1288
|
+
const staged = new Set(pending.staged ?? []);
|
|
1112
1289
|
const failed = new Map();
|
|
1290
|
+
const work = await staging();
|
|
1113
1291
|
let saving = Promise.resolve();
|
|
1114
1292
|
await eachAtOnce(named, async (upload) => {
|
|
1115
1293
|
const part = parts.get(upload.sha256);
|
|
1116
1294
|
const path = relative(part.path);
|
|
1117
1295
|
try {
|
|
1118
|
-
await sentBinary(copy.repository,
|
|
1296
|
+
await sentBinary(work, join(copy.repository, pending.read[path]?.name ?? path), path, part, upload, (id, partSha256) => caller.finish(id, part.path, partSha256));
|
|
1119
1297
|
}
|
|
1120
1298
|
catch (error) {
|
|
1121
|
-
failed.set(upload.sha256, `its transfer failed: ${causeOf(error)}`);
|
|
1299
|
+
failed.set(upload.sha256, error instanceof Changed ? error : `its transfer failed: ${causeOf(error)}`);
|
|
1122
1300
|
return;
|
|
1123
1301
|
}
|
|
1124
1302
|
staged.add(upload.sha256);
|
|
1125
1303
|
saving = saving.then(async () => {
|
|
1126
|
-
copy = await this.save({ ...copy,
|
|
1304
|
+
copy = await this.save({ ...copy, pending: { ...copy.pending, staged: [...staged] } });
|
|
1127
1305
|
});
|
|
1128
1306
|
await saving;
|
|
1129
1307
|
});
|
|
1130
1308
|
return failed;
|
|
1131
1309
|
}
|
|
1132
|
-
stopped(copy, reasons) {
|
|
1133
|
-
const
|
|
1134
|
-
const
|
|
1135
|
-
const
|
|
1136
|
-
const
|
|
1137
|
-
|
|
1310
|
+
async stopped(copy, reasons) {
|
|
1311
|
+
const pending = copy.pending;
|
|
1312
|
+
const root = rootOf(copy.room_handle);
|
|
1313
|
+
const unchanged = `House holds nothing of this upload yet, so ${root} is unchanged`;
|
|
1314
|
+
const changed = typeof reasons === 'string' ? [] : [...reasons.values()].filter((reason) => reason instanceof Changed);
|
|
1315
|
+
if (changed.length > 0) {
|
|
1316
|
+
await this.save(without(copy));
|
|
1317
|
+
return { refused: true, text: `${unchanged}: ${changed.map((reason) => reason.message).join('; ')}; run house upload again to send the copy as it is now` };
|
|
1318
|
+
}
|
|
1319
|
+
const again = 'house upload again resumes this upload and sends only what it has not sent';
|
|
1320
|
+
const staged = new Set(pending.staged ?? []);
|
|
1321
|
+
const unsent = pending.changes.filter((change) => change.op === 'upload' && !staged.has(change.sha256));
|
|
1138
1322
|
if (typeof reasons === 'string' && unsent.length === 0)
|
|
1139
1323
|
return { refused: true, text: `${unchanged}: ${reasons}\n${again}` };
|
|
1140
|
-
const why = (sha256) => (typeof reasons === 'string' ? reasons : (reasons.get(sha256) ?? 'not sent yet'));
|
|
1324
|
+
const why = (sha256) => (typeof reasons === 'string' ? reasons : String(reasons.get(sha256) ?? 'not sent yet'));
|
|
1141
1325
|
return {
|
|
1142
1326
|
refused: true,
|
|
1143
1327
|
text: [`${unchanged}; these attachments are not sent:`, ...unsent.map(({ path, sha256 }) => `${relative(path)}: ${why(sha256)}`), again].join('\n'),
|
|
1144
1328
|
};
|
|
1145
1329
|
}
|
|
1146
|
-
async conflicted(copy,
|
|
1330
|
+
async conflicted(copy, pending, refusal, caller) {
|
|
1147
1331
|
const root = rootOf(copy.room_handle);
|
|
1148
|
-
const
|
|
1149
|
-
const
|
|
1150
|
-
const
|
|
1151
|
-
|
|
1152
|
-
|
|
1153
|
-
const
|
|
1154
|
-
const
|
|
1155
|
-
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1332
|
+
const manifest = await readManifest(copy.repository);
|
|
1333
|
+
const present = await scanned(copy.repository);
|
|
1334
|
+
const work = await staging();
|
|
1335
|
+
const lines = [];
|
|
1336
|
+
const nameOf = (path) => present.get(path)?.name ?? path;
|
|
1337
|
+
const holds = async (path, sha256) => present.has(path) && (await holding(copy.repository, nameOf(path), sha256));
|
|
1338
|
+
const brought = async (path, revision) => {
|
|
1339
|
+
const named = `${root}/${path}`;
|
|
1340
|
+
const target = join(copy.repository, nameOf(path));
|
|
1341
|
+
if (isDocument(path)) {
|
|
1342
|
+
const text = await this.revisionRead(named, revision);
|
|
1343
|
+
if (text === null)
|
|
1344
|
+
return false;
|
|
1345
|
+
await placed(work, target, text);
|
|
1346
|
+
manifest.files[path] = { revision, sha256: digest(text), bytes: Buffer.byteLength(text) };
|
|
1347
|
+
}
|
|
1348
|
+
else {
|
|
1349
|
+
const held = {};
|
|
1350
|
+
const file = await receivedBinary(work, named, async () => {
|
|
1351
|
+
const info = await caller.info(named, revision);
|
|
1352
|
+
if (info === null)
|
|
1353
|
+
throw new Error(`House holds no attachment at ${named}`);
|
|
1354
|
+
held.file = info;
|
|
1355
|
+
return info;
|
|
1356
|
+
});
|
|
1357
|
+
await mkdir(dirname(target), { recursive: true });
|
|
1358
|
+
await rename(file, target);
|
|
1359
|
+
manifest.files[path] = { revision, sha256: held.file.sha256, bytes: held.file.bytes };
|
|
1360
|
+
}
|
|
1361
|
+
present.set(path, { name: nameOf(path), sha256: manifest.files[path].sha256, bytes: manifest.files[path].bytes });
|
|
1362
|
+
return true;
|
|
1363
|
+
};
|
|
1364
|
+
for (const conflict of refusal.conflicts) {
|
|
1365
|
+
const path = relative(conflict.path);
|
|
1366
|
+
const change = pending.changes.find((sent) => relative(sent.path) === path || (sent.op === 'move' && relative(sent.to) === path));
|
|
1367
|
+
if (change === undefined)
|
|
1368
|
+
continue;
|
|
1369
|
+
const named = `${root}/${path}`;
|
|
1370
|
+
const read = pending.read[path];
|
|
1371
|
+
const busy = `${named}: changed while it uploaded; the next house upload merges it`;
|
|
1372
|
+
if (conflict.revision === undefined) {
|
|
1373
|
+
delete manifest.files[path];
|
|
1374
|
+
lines.push(`${named}: House no longer holds it; the next house upload sends it as a new file`);
|
|
1159
1375
|
continue;
|
|
1160
|
-
files.set(path, { blob: await this.revisionBlob(copy, path, file.revision, caller, file) });
|
|
1161
|
-
revisions[path] = file.revision;
|
|
1162
|
-
binary[path] = { bytes: file.bytes, sha256: file.sha256 };
|
|
1163
|
-
}
|
|
1164
|
-
if (files.size === 0)
|
|
1165
|
-
return copy;
|
|
1166
|
-
const house = await importCommits(copy.repository, [
|
|
1167
|
-
{ message: `House ${root}: the attachments ${short(push.commit)} conflicts with`, from: copy.house, files },
|
|
1168
|
-
]);
|
|
1169
|
-
return { ...copy, house, manifests: { ...copy.manifests, [house]: manifest(copy.house, revisions, binary) } };
|
|
1170
|
-
}
|
|
1171
|
-
async recovered(copy, caller, unheld) {
|
|
1172
|
-
const push = copy.push;
|
|
1173
|
-
const current = (await this.caughtUp([copy], true, caller))[0];
|
|
1174
|
-
const house = await tree(copy.repository, current.house);
|
|
1175
|
-
const model = await modelOf(current, house);
|
|
1176
|
-
if (!(await this.landed(copy.repository, push.changes, house, await tree(copy.repository, push.commit), model))) {
|
|
1177
|
-
return unheld(current);
|
|
1178
|
-
}
|
|
1179
|
-
const written = [];
|
|
1180
|
-
for (const path of changedPaths(push.changes).map(relative))
|
|
1181
|
-
written.push(await writtenAt(current, house, path));
|
|
1182
|
-
return this.finished(await this.save({ ...current, push: { ...push, state: 'accepted', written } }), caller);
|
|
1183
|
-
}
|
|
1184
|
-
async finished(start, caller) {
|
|
1185
|
-
let copy = start;
|
|
1186
|
-
const { commit } = copy.push;
|
|
1187
|
-
try {
|
|
1188
|
-
if (copy.push.acknowledged === undefined)
|
|
1189
|
-
copy = await this.save(await this.acknowledged(copy, caller));
|
|
1190
|
-
const integrated = await this.integrated(copy.push);
|
|
1191
|
-
if (integrated.done)
|
|
1192
|
-
await this.save(without(copy));
|
|
1193
|
-
return { refused: false, text: integrated.text };
|
|
1194
|
-
}
|
|
1195
|
-
catch {
|
|
1196
|
-
return {
|
|
1197
|
-
refused: false,
|
|
1198
|
-
text: `House accepted ${short(commit)}; run house push again to finish integrating it here`,
|
|
1199
|
-
};
|
|
1200
|
-
}
|
|
1201
|
-
}
|
|
1202
|
-
async acknowledged(start, caller) {
|
|
1203
|
-
let copy = start;
|
|
1204
|
-
const push = copy.push;
|
|
1205
|
-
let written = push.written;
|
|
1206
|
-
const read = push.changes.some((change) => change.op === 'rename');
|
|
1207
|
-
if (read) {
|
|
1208
|
-
copy = (await this.caughtUp([copy], true, caller))[0];
|
|
1209
|
-
const house = await tree(copy.repository, copy.house);
|
|
1210
|
-
const sent = await tree(copy.repository, push.house);
|
|
1211
|
-
const unread = push.changes.some((change) => change.op === 'rename' &&
|
|
1212
|
-
house.get(relative(change.path)) === sent.get(relative(change.path)) &&
|
|
1213
|
-
!house.has(relative(change.to)));
|
|
1214
|
-
if (unread)
|
|
1215
|
-
throw new Error(`House's result for ${short(push.commit)} is not readable yet`);
|
|
1216
|
-
const known = new Set(written.map((entry) => relative(entry.path)));
|
|
1217
|
-
for (const path of new Set([...sent.keys(), ...house.keys()])) {
|
|
1218
|
-
if (!known.has(path) && sent.get(path) !== house.get(path))
|
|
1219
|
-
written = [...written, await writtenAt(copy, house, path)];
|
|
1220
1376
|
}
|
|
1221
|
-
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1377
|
+
if (change.op === 'write') {
|
|
1378
|
+
if (read === undefined || !(await holds(path, read.sha256))) {
|
|
1379
|
+
lines.push(busy);
|
|
1380
|
+
continue;
|
|
1381
|
+
}
|
|
1382
|
+
const content = conflict.content ?? whole(await this.revisionText(named, conflict.revision), change.content);
|
|
1383
|
+
await placed(work, join(copy.repository, nameOf(path)), content);
|
|
1384
|
+
manifest.files[path] = { revision: conflict.revision, sha256: digest(content), bytes: Buffer.byteLength(content), conflicted: true };
|
|
1385
|
+
lines.push(`${named}: conflict markers written; resolve them, then run house upload again`);
|
|
1386
|
+
continue;
|
|
1387
|
+
}
|
|
1388
|
+
if (change.op === 'upload') {
|
|
1389
|
+
const current = await caller.info(named, conflict.revision);
|
|
1390
|
+
if (current === null)
|
|
1391
|
+
throw new Error(`House holds no attachment at ${named}`);
|
|
1392
|
+
manifest.files[path] = { revision: conflict.revision, sha256: current.sha256, bytes: current.bytes };
|
|
1393
|
+
const theirs = JSON.stringify({ path: named, file: join(copy.repository, nameOf(path)) });
|
|
1394
|
+
lines.push(`${named}: House holds ${conflict.revision}, a version this copy never held; the next house upload replaces it with this file, and house download '${theirs}' brings House's instead`);
|
|
1395
|
+
continue;
|
|
1396
|
+
}
|
|
1397
|
+
if (change.op === 'delete') {
|
|
1398
|
+
if (present.has(path))
|
|
1399
|
+
lines.push(busy);
|
|
1400
|
+
else if (await brought(path, conflict.revision)) {
|
|
1401
|
+
lines.push(`${named}: House changed it since this copy read it and now holds ${conflict.revision}; that version is back here, so delete it again if you still mean to`);
|
|
1402
|
+
}
|
|
1236
1403
|
continue;
|
|
1237
|
-
revisions[path] = entry.revision;
|
|
1238
|
-
const part = parts.get(path);
|
|
1239
|
-
const kept = heldAt(copy, copy.house, path);
|
|
1240
|
-
if (entry.revision === null)
|
|
1241
|
-
files.set(path, null);
|
|
1242
|
-
else if (part !== undefined) {
|
|
1243
|
-
files.set(path, { blob: selected.get(path) });
|
|
1244
|
-
binary[path] = part;
|
|
1245
1404
|
}
|
|
1246
|
-
|
|
1247
|
-
|
|
1248
|
-
|
|
1405
|
+
const from = relative(change.path);
|
|
1406
|
+
const to = relative(change.to);
|
|
1407
|
+
const moved = pending.read[to];
|
|
1408
|
+
if (!present.has(from) && moved !== undefined && (await holds(to, moved.sha256))) {
|
|
1409
|
+
await mkdir(dirname(join(copy.repository, from)), { recursive: true });
|
|
1410
|
+
await rename(join(copy.repository, nameOf(to)), join(copy.repository, from));
|
|
1411
|
+
await removed(copy.repository, nameOf(to));
|
|
1412
|
+
present.set(from, { ...present.get(to), name: from });
|
|
1413
|
+
present.delete(to);
|
|
1414
|
+
}
|
|
1415
|
+
const entry = manifest.files[from];
|
|
1416
|
+
if (path === from && entry !== undefined && (await holds(from, entry.sha256)) && (await brought(from, conflict.revision))) {
|
|
1417
|
+
lines.push(`${named}: House changed it since this copy read it and now holds ${conflict.revision}; it is back at ${named} with that version, so move it again if you still mean to`);
|
|
1418
|
+
}
|
|
1419
|
+
else if (path === to && !present.has(to) && (await brought(to, conflict.revision))) {
|
|
1420
|
+
lines.push(`${named}: House already holds a file there (${conflict.revision}); ${root}/${from} is back where it was and ${named} holds House's file`);
|
|
1249
1421
|
}
|
|
1250
1422
|
else
|
|
1251
|
-
|
|
1423
|
+
lines.push(busy);
|
|
1252
1424
|
}
|
|
1253
|
-
|
|
1254
|
-
|
|
1255
|
-
|
|
1256
|
-
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1261
|
-
|
|
1262
|
-
from: copy.house,
|
|
1263
|
-
merge: edits.size === 0 ? push.commit : ':1',
|
|
1264
|
-
files: new Map(received.map((path) => [path, files.get(path)])),
|
|
1265
|
-
},
|
|
1266
|
-
]);
|
|
1267
|
-
const acknowledged = edits.size === 0 ? push.commit : await revision(copy.repository, `${house}^2`);
|
|
1268
|
-
const after = await tree(copy.repository, house);
|
|
1269
|
-
return {
|
|
1270
|
-
...copy,
|
|
1271
|
-
house,
|
|
1272
|
-
manifests: {
|
|
1273
|
-
...copy.manifests,
|
|
1274
|
-
[acknowledged]: manifest(push.base, revisions, binary),
|
|
1275
|
-
[house]: manifest(copy.house, Object.fromEntries(received.map((path) => [path, revisions[path] ?? null])), Object.fromEntries(received.flatMap((path) => (binary[path] === undefined ? [] : [[path, binary[path]]])))),
|
|
1276
|
-
},
|
|
1277
|
-
moves: copy.moves.filter(({ from, to }) => !(after.has(to) && !after.has(from))),
|
|
1278
|
-
push: { ...push, acknowledged },
|
|
1279
|
-
};
|
|
1425
|
+
await writeManifest(copy.repository, work, manifest);
|
|
1426
|
+
return lines;
|
|
1427
|
+
}
|
|
1428
|
+
async textOf(written, sent) {
|
|
1429
|
+
if (typeof written.content === 'string' && !CUT.test(written.content))
|
|
1430
|
+
return written.content;
|
|
1431
|
+
if (written.content === undefined && sent !== undefined)
|
|
1432
|
+
return sent;
|
|
1433
|
+
return this.revisionText(written.path, written.revision);
|
|
1280
1434
|
}
|
|
1281
|
-
async
|
|
1282
|
-
const
|
|
1283
|
-
|
|
1435
|
+
async integrated(copy, accepted) {
|
|
1436
|
+
const pending = copy.pending;
|
|
1437
|
+
const root = rootOf(copy.room_handle);
|
|
1438
|
+
const manifest = await readManifest(copy.repository);
|
|
1439
|
+
const work = await staging();
|
|
1440
|
+
const answered = new Map(accepted.filter((written) => written.revision !== null).map((written) => [relative(written.path), written]));
|
|
1441
|
+
const lines = [];
|
|
1442
|
+
const sent = new Set();
|
|
1443
|
+
for (const change of pending.changes) {
|
|
1284
1444
|
const path = relative(change.path);
|
|
1445
|
+
sent.add(path);
|
|
1446
|
+
if (change.op === 'delete') {
|
|
1447
|
+
delete manifest.files[path];
|
|
1448
|
+
continue;
|
|
1449
|
+
}
|
|
1450
|
+
if (change.op === 'move') {
|
|
1451
|
+
const to = relative(change.to);
|
|
1452
|
+
sent.add(to);
|
|
1453
|
+
const entry = manifest.files[path];
|
|
1454
|
+
delete manifest.files[path];
|
|
1455
|
+
manifest.files[to] = { revision: answered.get(to)?.revision ?? entry.revision, sha256: entry.sha256, bytes: entry.bytes };
|
|
1456
|
+
continue;
|
|
1457
|
+
}
|
|
1458
|
+
const written = answered.get(path);
|
|
1285
1459
|
if (change.op === 'upload') {
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
return false;
|
|
1460
|
+
manifest.files[path] = { revision: written.revision, sha256: change.sha256, bytes: change.bytes };
|
|
1461
|
+
continue;
|
|
1289
1462
|
}
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1463
|
+
const text = await this.textOf(written, change.content);
|
|
1464
|
+
const read = pending.read[path];
|
|
1465
|
+
if (text === change.content) {
|
|
1466
|
+
manifest.files[path] = { revision: written.revision, sha256: read.sha256, bytes: Buffer.byteLength(text) };
|
|
1294
1467
|
}
|
|
1295
|
-
if (
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
return false;
|
|
1468
|
+
else if (await holding(copy.repository, read.name, read.sha256)) {
|
|
1469
|
+
await placed(work, join(copy.repository, read.name), text);
|
|
1470
|
+
manifest.files[path] = { revision: written.revision, sha256: digest(text), bytes: Buffer.byteLength(text) };
|
|
1299
1471
|
}
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
if (change.op === 'set_field' && JSON.stringify(document.fields[change.field]) === JSON.stringify(change.value))
|
|
1308
|
-
continue;
|
|
1309
|
-
if (change.op === 'replace_section' && document.sections[change.section] === change.content)
|
|
1472
|
+
else {
|
|
1473
|
+
lines.push(`${root}/${path}: changed while it uploaded; House accepted it and local integration is pending, so the next house upload merges it`);
|
|
1474
|
+
}
|
|
1475
|
+
}
|
|
1476
|
+
for (const [path, written] of answered) {
|
|
1477
|
+
const entry = manifest.files[path];
|
|
1478
|
+
if (sent.has(path) || entry === undefined || entry.revision === written.revision)
|
|
1310
1479
|
continue;
|
|
1311
|
-
|
|
1480
|
+
const text = await this.textOf(written, undefined);
|
|
1481
|
+
if (!(await holding(copy.repository, path, entry.sha256))) {
|
|
1482
|
+
lines.push(`${root}/${path}: changed here, so House's change was not brought; the next house upload merges it`);
|
|
1312
1483
|
continue;
|
|
1313
|
-
|
|
1484
|
+
}
|
|
1485
|
+
await placed(work, join(copy.repository, path), text);
|
|
1486
|
+
manifest.files[path] = { revision: written.revision, sha256: digest(text), bytes: Buffer.byteLength(text) };
|
|
1314
1487
|
}
|
|
1315
|
-
|
|
1488
|
+
await writeManifest(copy.repository, work, manifest);
|
|
1489
|
+
await this.save(without(copy));
|
|
1490
|
+
return { refused: false, files: pending.changes.length, lines };
|
|
1316
1491
|
}
|
|
1317
|
-
async
|
|
1492
|
+
async deletedByHouse(held, named, path, sha256) {
|
|
1493
|
+
const attachments = Object.values(held.manifests).flatMap((manifest) => (manifest.binary?.[path] === undefined ? [] : [manifest.binary[path].sha256]));
|
|
1494
|
+
if (attachments.length > 0)
|
|
1495
|
+
return attachments.includes(sha256);
|
|
1496
|
+
for (const revision of recordedRevisions(held, path)) {
|
|
1497
|
+
const text = await this.revisionRead(named, revision);
|
|
1498
|
+
if (text !== null && digest(text) === sha256)
|
|
1499
|
+
return true;
|
|
1500
|
+
}
|
|
1501
|
+
return false;
|
|
1502
|
+
}
|
|
1503
|
+
async revisionRead(path, wanted) {
|
|
1318
1504
|
const page = await this.house.door('files/revisions/list', {
|
|
1319
1505
|
path,
|
|
1320
1506
|
revision: wanted,
|
|
1321
1507
|
});
|
|
1322
|
-
const
|
|
1323
|
-
if (
|
|
1324
|
-
|
|
1325
|
-
|
|
1326
|
-
|
|
1327
|
-
|
|
1328
|
-
|
|
1329
|
-
|
|
1330
|
-
const
|
|
1331
|
-
if (
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
done: false,
|
|
1335
|
-
text: `${accepted}; local integration is pending (${reason}): ${acknowledged} holds House's result.`,
|
|
1336
|
-
});
|
|
1337
|
-
const head = await git(worktree, ['symbolic-ref', '-q', 'HEAD']).then((name) => name.trim(), () => null);
|
|
1338
|
-
if (branch === null || head !== branch)
|
|
1339
|
-
return pending('the branch changed');
|
|
1340
|
-
const tip = await revision(worktree, 'HEAD');
|
|
1341
|
-
const name = branch.replace(/^refs\/heads\//, '');
|
|
1342
|
-
if (await succeeded(worktree, ['merge-base', '--is-ancestor', acknowledged, tip])) {
|
|
1343
|
-
return { done: true, text: `${accepted}; ${name} holds it.` };
|
|
1344
|
-
}
|
|
1345
|
-
if (!(await succeeded(worktree, ['merge-base', '--is-ancestor', commit, tip])))
|
|
1346
|
-
return pending('the branch changed');
|
|
1347
|
-
if ((await git(worktree, ['status', '--porcelain', '--untracked-files=no'])).trim() !== '') {
|
|
1348
|
-
return pending('uncommitted changes');
|
|
1349
|
-
}
|
|
1350
|
-
if (tip === commit) {
|
|
1351
|
-
if (!(await succeeded(worktree, ['merge', '--ff-only', '-q', acknowledged])))
|
|
1352
|
-
return pending('a conflict');
|
|
1353
|
-
}
|
|
1354
|
-
else if (!(await succeeded(worktree, ['rebase', '-q', '--onto', acknowledged, commit], IDENTITY))) {
|
|
1355
|
-
await succeeded(worktree, ['rebase', '--abort']);
|
|
1356
|
-
return pending('a conflict');
|
|
1357
|
-
}
|
|
1358
|
-
return { done: true, text: `${accepted}; ${name} is at ${short(await revision(worktree, 'HEAD'))}.` };
|
|
1508
|
+
const found = page.revisions.find((entry) => entry.revision === wanted);
|
|
1509
|
+
if (found === undefined)
|
|
1510
|
+
return null;
|
|
1511
|
+
if (typeof found.content !== 'string')
|
|
1512
|
+
throw new Error(`House's text of ${path} at ${wanted} is not readable yet`);
|
|
1513
|
+
return found.content;
|
|
1514
|
+
}
|
|
1515
|
+
async revisionText(path, wanted) {
|
|
1516
|
+
const text = await this.revisionRead(path, wanted);
|
|
1517
|
+
if (text === null)
|
|
1518
|
+
throw new Error(`House holds no revision ${wanted} of ${path}`);
|
|
1519
|
+
return text;
|
|
1359
1520
|
}
|
|
1360
1521
|
}
|