mainflow 0.32.1 → 0.33.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +45 -9
- package/dist/hooks.js +2 -2
- package/dist/{host-C7GKIjSc.js → host-CObkJQmr.js} +86 -56
- package/dist/host.js +1 -1
- package/dist/{launch-6839iAsc.js → launch-BemxyV9P.js} +2 -2
- package/dist/launch.js +1 -1
- package/dist/native/mainflow-swap-darwin-arm64 +0 -0
- package/dist/native/mainflow-swap-darwin-x64 +0 -0
- package/dist/{operation-C0dUHUHg.js → operation-B2pq2Q9z.js} +1646 -237
- package/dist/{ui-Bx2wUBnA.js → ui-CFLjhYJ9.js} +63 -6
- package/dist/web/assets/index-CC7l6bNN.css +1 -0
- package/dist/web/assets/index-CDNCPMjD.js +82 -0
- package/dist/web/index.html +2 -2
- package/package.json +1 -1
- package/dist/web/assets/index-CyQML0b0.css +0 -1
- package/dist/web/assets/index-D2iMe1-S.js +0 -82
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { $ as
|
|
1
|
+
import { $ as MAX_FILE_BYTES, F as renumberFiles, I as renumberPath, L as renumberText, N as applyOffline, P as renumberRequest, Q as MAX_BLOB_BYTES, R as scopeOfPath, X as ACCESS_HEADERS, Y as nodeStorage, Z as BLOB_HASH, _t as splitFrontMatter, at as requestTime, et as ROUTES, gt as contentHash, ht as WriteError, it as readsAsText, mt as DATA_DIR, nt as comparePaths, ot as textBytes, pt as ConflictError, q as nowMinute, rt as exceedsFileLimit, tt as SOCKET_MAX_AGE_CLOSE, z as applyOperation } from "./host-CObkJQmr.js";
|
|
2
2
|
import { createHash, randomBytes } from "node:crypto";
|
|
3
|
-
import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, watch, writeFileSync } from "node:fs";
|
|
3
|
+
import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, createReadStream, createWriteStream, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, watch, writeFileSync } from "node:fs";
|
|
4
4
|
import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
6
|
import { execFileSync, spawn, spawnSync } from "node:child_process";
|
|
@@ -9,8 +9,11 @@ import { homedir } from "node:os";
|
|
|
9
9
|
import { createBrotliDecompress, createGunzip, createInflate, createZstdDecompress } from "node:zlib";
|
|
10
10
|
import { Agent, createServer, request } from "node:http";
|
|
11
11
|
import { Agent as Agent$1, request as request$1 } from "node:https";
|
|
12
|
+
import { Readable, Transform } from "node:stream";
|
|
13
|
+
import { pipeline } from "node:stream/promises";
|
|
12
14
|
import { EventEmitter } from "node:events";
|
|
13
|
-
import {
|
|
15
|
+
import { COPYFILE_EXCL } from "node:constants";
|
|
16
|
+
import { appendFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
|
|
14
17
|
import { diff3Merge } from "node-diff3";
|
|
15
18
|
import ignore from "ignore";
|
|
16
19
|
//#region ../sync/dist/paths.js
|
|
@@ -27,10 +30,17 @@ const syncPaths = (root) => {
|
|
|
27
30
|
tmp: join(own, "tmp"),
|
|
28
31
|
conflicts: join(own, "conflicts"),
|
|
29
32
|
known: join(own, "known.json"),
|
|
30
|
-
/** The files only on the server, the artifacts of closed mainlines not downloaded here, by their content hash */
|
|
33
|
+
/** The files only on the server, the artifacts of closed mainlines not downloaded here, by their content hash: a binary file's sha256 */
|
|
31
34
|
remote: join(own, "remote.json"),
|
|
32
35
|
/** The text of each version known.json names, in a file named by its content hash */
|
|
33
36
|
knownTexts: join(own, "known"),
|
|
37
|
+
/** The content of each binary file a write in the outbox uploads, in a file named by its sha256 */
|
|
38
|
+
blobs: join(own, "blobs"),
|
|
39
|
+
/**
|
|
40
|
+
* Present while a hub taken without known.json has had no snapshot that named binary files, as when the first probe
|
|
41
|
+
* failed: a binary file such a snapshot names removed was removed on the server since the copy
|
|
42
|
+
*/
|
|
43
|
+
copyBlobs: join(own, "copy-blobs"),
|
|
34
44
|
queue: join(own, "queue.json"),
|
|
35
45
|
notices: join(own, "notices.jsonl"),
|
|
36
46
|
daemon: join(own, "daemon.json"),
|
|
@@ -140,6 +150,13 @@ function earlierToken(root, env = process.env) {
|
|
|
140
150
|
}
|
|
141
151
|
//#endregion
|
|
142
152
|
//#region ../sync/dist/api.js
|
|
153
|
+
/** The sha256 of `bytes` in lowercase hex, as a binary file is named (BLOB_HASH). */
|
|
154
|
+
const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
|
|
155
|
+
/**
|
|
156
|
+
* The sha256 the probe of the binary files' route names (blobRoute): not the sha256 of the empty content it sends, so a
|
|
157
|
+
* server with the route refuses it and keeps nothing.
|
|
158
|
+
*/
|
|
159
|
+
const PROBE_HASH = "0".repeat(64);
|
|
143
160
|
/**
|
|
144
161
|
* The server could not be reached, or the connection broke. `sent` says whether the request may have reached the
|
|
145
162
|
* server: a refused connection did not, a reset or timeout after sending may have, so its result is unknown.
|
|
@@ -165,6 +182,19 @@ var ServerError = class extends Error {
|
|
|
165
182
|
};
|
|
166
183
|
/** Cloudflare Access refused the request: no service token, a wrong or revoked one, or a policy that does not admit it. */
|
|
167
184
|
var AccessRefused = class extends ServerError {};
|
|
185
|
+
/** Content downloaded as one sha256 turned out to be of another (downloadBlob): damaged on its way, or by the server. */
|
|
186
|
+
var BlobMismatch = class extends Error {};
|
|
187
|
+
/**
|
|
188
|
+
* The downloaded content could not be written here (downloadBlob), as on a full disk (ENOSPC): an error of this machine,
|
|
189
|
+
* not of the server or the network. `code` is the file system's error code.
|
|
190
|
+
*/
|
|
191
|
+
var LocalWriteError = class extends Error {
|
|
192
|
+
code;
|
|
193
|
+
constructor(message, code) {
|
|
194
|
+
super(message);
|
|
195
|
+
this.code = code;
|
|
196
|
+
}
|
|
197
|
+
};
|
|
168
198
|
/** Error codes of a connection that never carried the request. */
|
|
169
199
|
const NOT_SENT = /* @__PURE__ */ new Set([
|
|
170
200
|
"ECONNREFUSED",
|
|
@@ -198,6 +228,16 @@ const ACCEPT_ENCODING = Object.keys(DECODERS).join(", ");
|
|
|
198
228
|
const causeCode = (error) => {
|
|
199
229
|
for (let cause = error; cause; cause = cause.cause) if (typeof cause.code === "string") return cause.code;
|
|
200
230
|
};
|
|
231
|
+
/** What an answer's body is once decompressed, as Content-Encoding says; throws for an encoding this client cannot read. */
|
|
232
|
+
function decoded(res) {
|
|
233
|
+
const encoding = String(res.headers["content-encoding"] ?? "identity").toLowerCase();
|
|
234
|
+
const decode = DECODERS[encoding];
|
|
235
|
+
if (!decode && encoding !== "identity") {
|
|
236
|
+
res.resume();
|
|
237
|
+
throw new Error(`the answer came in content-encoding ${encoding}, which this client cannot read`);
|
|
238
|
+
}
|
|
239
|
+
return decode ? res.pipe(decode()) : res;
|
|
240
|
+
}
|
|
201
241
|
var ServerApi = class {
|
|
202
242
|
settings;
|
|
203
243
|
timeoutMs;
|
|
@@ -208,6 +248,11 @@ var ServerApi = class {
|
|
|
208
248
|
agent;
|
|
209
249
|
/** The connections snapshots go on, apart from the writes', so a snapshot never waits behind a write or holds one up */
|
|
210
250
|
reads;
|
|
251
|
+
/**
|
|
252
|
+
* The connections binary files' contents go up and down on, apart from the writes' and the snapshots', so a large file
|
|
253
|
+
* holds neither up; a few at a time, as the files of a snapshot are downloaded side by side.
|
|
254
|
+
*/
|
|
255
|
+
transfers;
|
|
211
256
|
/** Requests on `agent` not answered yet */
|
|
212
257
|
busy = 0;
|
|
213
258
|
transfer = {
|
|
@@ -232,6 +277,11 @@ var ServerApi = class {
|
|
|
232
277
|
maxSockets: 2,
|
|
233
278
|
maxFreeSockets: 1
|
|
234
279
|
});
|
|
280
|
+
this.transfers = new Agent$2({
|
|
281
|
+
keepAlive: true,
|
|
282
|
+
maxSockets: 4,
|
|
283
|
+
maxFreeSockets: 1
|
|
284
|
+
});
|
|
235
285
|
}
|
|
236
286
|
/** The service token's headers, which every request and the WebSocket's upgrade carry. Never logged. */
|
|
237
287
|
get access() {
|
|
@@ -240,6 +290,18 @@ var ServerApi = class {
|
|
|
240
290
|
[ACCESS_HEADERS.clientSecret]: this.settings.clientSecret
|
|
241
291
|
};
|
|
242
292
|
}
|
|
293
|
+
/** A failure to reach the server or of the connection, as the Unreachable it is. */
|
|
294
|
+
unreachable(error) {
|
|
295
|
+
const code = causeCode(error);
|
|
296
|
+
const detail = [code, error.cause?.message ?? error.message].filter(Boolean).join(": ");
|
|
297
|
+
const sent = !(code && NOT_SENT.has(code));
|
|
298
|
+
return new Unreachable(`cannot reach the sync server at ${this.settings.url}${detail ? ` (${detail})` : ""}`, sent, sent && STALE.has(code ?? ""));
|
|
299
|
+
}
|
|
300
|
+
/** Access answers for the server, with a page rather than JSON: 401 or 403, or a redirect to its login page. */
|
|
301
|
+
refusedByAccess(status, location, url) {
|
|
302
|
+
if (status !== 401 && status !== 403 && !(status >= 300 && status < 400)) return null;
|
|
303
|
+
return new AccessRefused(`Cloudflare Access in front of the sync server refused the request (HTTP ${status}${location ? ` to ${new URL(location, url).origin}` : ""}); check the service token in ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}`, status);
|
|
304
|
+
}
|
|
243
305
|
async call(method, route, body, connection) {
|
|
244
306
|
const url = `${this.settings.url}${route}`;
|
|
245
307
|
const text = JSON.stringify(body);
|
|
@@ -247,15 +309,10 @@ var ServerApi = class {
|
|
|
247
309
|
try {
|
|
248
310
|
answer = connection === "read" ? await this.read(method, url, text) : await this.write(method, url, text, connection === "fresh");
|
|
249
311
|
} catch (error) {
|
|
250
|
-
|
|
251
|
-
const detail = [code, error.cause?.message ?? error.message].filter(Boolean).join(": ");
|
|
252
|
-
const sent = !(code && NOT_SENT.has(code));
|
|
253
|
-
throw new Unreachable(`cannot reach the sync server at ${this.settings.url}${detail ? ` (${detail})` : ""}`, sent, sent && STALE.has(code ?? ""));
|
|
254
|
-
}
|
|
255
|
-
if (answer.status === 401 || answer.status === 403 || answer.status >= 300 && answer.status < 400) {
|
|
256
|
-
const to = answer.location ? ` to ${new URL(answer.location, url).origin}` : "";
|
|
257
|
-
throw new AccessRefused(`Cloudflare Access in front of the sync server refused the request (HTTP ${answer.status}${to}); check the service token in ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}`, answer.status);
|
|
312
|
+
throw this.unreachable(error);
|
|
258
313
|
}
|
|
314
|
+
const access = this.refusedByAccess(answer.status, answer.location, url);
|
|
315
|
+
if (access) throw access;
|
|
259
316
|
let parsed;
|
|
260
317
|
try {
|
|
261
318
|
parsed = JSON.parse(answer.text);
|
|
@@ -293,50 +350,171 @@ var ServerApi = class {
|
|
|
293
350
|
write(method, url, body, fresh) {
|
|
294
351
|
return this.exchange(method, url, body, fresh ? false : this.agent, this.timeoutMs);
|
|
295
352
|
}
|
|
296
|
-
|
|
353
|
+
/**
|
|
354
|
+
* Send one request with the service token and resolve with the answer as it starts to come. `body` is a JSON text,
|
|
355
|
+
* the raw bytes of a stream, or nothing; `headers` add to the token's and say what the body is.
|
|
356
|
+
*/
|
|
357
|
+
open(method, url, body, headers, agent, timeoutMs) {
|
|
297
358
|
const send = url.startsWith("https:") ? request$1 : request;
|
|
298
|
-
const headers = {
|
|
299
|
-
...this.access,
|
|
300
|
-
"accept-encoding": ACCEPT_ENCODING
|
|
301
|
-
};
|
|
302
|
-
if (body !== void 0) Object.assign(headers, {
|
|
303
|
-
"content-type": "application/json",
|
|
304
|
-
"content-length": String(Buffer.byteLength(body))
|
|
305
|
-
});
|
|
306
|
-
if (agent === this.agent) this.busy++;
|
|
307
359
|
return new Promise((resolve, reject) => {
|
|
308
360
|
const req = send(url, {
|
|
309
361
|
method,
|
|
310
362
|
agent,
|
|
311
|
-
headers
|
|
363
|
+
headers: {
|
|
364
|
+
...this.access,
|
|
365
|
+
"accept-encoding": ACCEPT_ENCODING,
|
|
366
|
+
...headers
|
|
367
|
+
},
|
|
312
368
|
timeout: timeoutMs
|
|
313
|
-
},
|
|
369
|
+
}, resolve);
|
|
370
|
+
req.on("timeout", () => req.destroy(Object.assign(/* @__PURE__ */ new Error("no answer in time"), { code: req.socket?.connecting === false ? "ETIMEDOUT" : "ECONNECT_TIMEOUT" })));
|
|
371
|
+
req.on("error", reject);
|
|
372
|
+
if (body instanceof Readable) {
|
|
373
|
+
body.on("error", (error) => req.destroy(error));
|
|
374
|
+
body.pipe(req);
|
|
375
|
+
} else req.end(body);
|
|
376
|
+
});
|
|
377
|
+
}
|
|
378
|
+
/** One request and its whole answer as text. */
|
|
379
|
+
async exchange(method, url, body, agent, timeoutMs, headers) {
|
|
380
|
+
const kind = typeof body === "string" ? {
|
|
381
|
+
"content-type": "application/json",
|
|
382
|
+
"content-length": String(Buffer.byteLength(body))
|
|
383
|
+
} : {};
|
|
384
|
+
if (agent === this.agent) this.busy++;
|
|
385
|
+
try {
|
|
386
|
+
const res = await this.open(method, url, body, {
|
|
387
|
+
...kind,
|
|
388
|
+
...headers
|
|
389
|
+
}, agent, timeoutMs);
|
|
390
|
+
return await new Promise((resolve, reject) => {
|
|
314
391
|
let wire = 0;
|
|
315
392
|
res.on("data", (chunk) => wire += chunk.length);
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
res
|
|
320
|
-
|
|
393
|
+
res.on("error", reject);
|
|
394
|
+
let stream;
|
|
395
|
+
try {
|
|
396
|
+
stream = decoded(res);
|
|
397
|
+
} catch (error) {
|
|
398
|
+
return reject(error);
|
|
321
399
|
}
|
|
322
|
-
const stream = decode ? res.pipe(decode()) : res;
|
|
323
400
|
const chunks = [];
|
|
324
401
|
stream.on("data", (chunk) => chunks.push(chunk));
|
|
325
402
|
stream.on("end", () => resolve({
|
|
326
403
|
status: res.statusCode ?? 0,
|
|
327
404
|
text: Buffer.concat(chunks).toString("utf8"),
|
|
328
405
|
location: res.headers.location,
|
|
406
|
+
retryAfter: res.headers["retry-after"],
|
|
329
407
|
wire
|
|
330
408
|
}));
|
|
331
409
|
stream.on("error", reject);
|
|
332
|
-
res.on("error", reject);
|
|
333
410
|
});
|
|
334
|
-
|
|
335
|
-
req.on("error", reject);
|
|
336
|
-
req.end(body);
|
|
337
|
-
}).finally(() => {
|
|
411
|
+
} finally {
|
|
338
412
|
if (agent === this.agent) this.busy--;
|
|
339
|
-
}
|
|
413
|
+
}
|
|
414
|
+
}
|
|
415
|
+
/**
|
|
416
|
+
* Whether the server has the route of binary files' contents (ROUTES.blob), as a server from before them does not:
|
|
417
|
+
* the server's version (GET /version) tells only its protocol, which binary files left as it was, so the route is
|
|
418
|
+
* asked. The probe uploads empty content as PROBE_HASH, which is not its sha256: a server with the route refuses it
|
|
419
|
+
* and keeps nothing, with 400 `hash-mismatch`, or 4xx for any other reason it refuses an upload; one without the route
|
|
420
|
+
* answers 404 `not-found`, from the Worker's router itself. A 5xx comes from a server that got as far as the route, as
|
|
421
|
+
* one whose R2 is failing: it keeps binary files, and the uploads wait until it answers them. Asked once per connection,
|
|
422
|
+
* before its snapshot, since a snapshot names the binary files only to a server that keeps them. It goes on a
|
|
423
|
+
* connection of the reads, so one the server closed meanwhile is asked again.
|
|
424
|
+
*/
|
|
425
|
+
async blobRoute() {
|
|
426
|
+
const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, PROBE_HASH)}`;
|
|
427
|
+
let answer;
|
|
428
|
+
try {
|
|
429
|
+
answer = await this.read("PUT", url, "");
|
|
430
|
+
} catch (error) {
|
|
431
|
+
throw this.unreachable(error);
|
|
432
|
+
}
|
|
433
|
+
const access = this.refusedByAccess(answer.status, answer.location, url);
|
|
434
|
+
if (access) throw access;
|
|
435
|
+
return answer.status !== 404;
|
|
436
|
+
}
|
|
437
|
+
/**
|
|
438
|
+
* Upload the content of a binary file, the `size` bytes of `file`, as sha256 `hash` (ROUTES.blob), on a connection of
|
|
439
|
+
* the transfers. The server out of reach, or failing, throws Unreachable; Access refusing it throws AccessRefused.
|
|
440
|
+
*/
|
|
441
|
+
async putBlob(hash, file, size) {
|
|
442
|
+
const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, hash)}`;
|
|
443
|
+
let answer;
|
|
444
|
+
try {
|
|
445
|
+
answer = await this.exchange("PUT", url, createReadStream(file), this.transfers, this.timeoutMs, {
|
|
446
|
+
"content-type": "application/octet-stream",
|
|
447
|
+
"content-length": String(size)
|
|
448
|
+
});
|
|
449
|
+
} catch (error) {
|
|
450
|
+
throw this.unreachable(error);
|
|
451
|
+
}
|
|
452
|
+
const access = this.refusedByAccess(answer.status, answer.location, url);
|
|
453
|
+
if (access) throw access;
|
|
454
|
+
let body;
|
|
455
|
+
try {
|
|
456
|
+
body = JSON.parse(answer.text);
|
|
457
|
+
} catch {
|
|
458
|
+
throw new Unreachable(`the sync server answered ${answer.status} without a readable body`, true);
|
|
459
|
+
}
|
|
460
|
+
if (answer.status === 503 && body.reason === "blob-deleting") return {
|
|
461
|
+
ok: false,
|
|
462
|
+
retryAfter: Math.max(1, Number(answer.retryAfter) || 1)
|
|
463
|
+
};
|
|
464
|
+
if (answer.status >= 500) throw new Unreachable(`the sync server failed (${answer.status})`, true);
|
|
465
|
+
if (body.ok === true) return { ok: true };
|
|
466
|
+
const message = body.message ?? `HTTP ${answer.status}`;
|
|
467
|
+
if (body.reason === "hash-mismatch" || body.reason === "too-large") return {
|
|
468
|
+
ok: false,
|
|
469
|
+
refused: body.reason,
|
|
470
|
+
message
|
|
471
|
+
};
|
|
472
|
+
return {
|
|
473
|
+
ok: false,
|
|
474
|
+
refused: answer.status === 404 ? "unsupported" : "other",
|
|
475
|
+
message
|
|
476
|
+
};
|
|
477
|
+
}
|
|
478
|
+
/**
|
|
479
|
+
* Download the content of sha256 `hash` into `file`, checking it is of that hash as it comes: true once it is there,
|
|
480
|
+
* false when the server holds no such content. Content of another hash throws BlobMismatch, and `file` that cannot be
|
|
481
|
+
* written here throws LocalWriteError; both leave `file` to the caller to remove, and so do the server out of reach
|
|
482
|
+
* (Unreachable) and Access refusing it (AccessRefused).
|
|
483
|
+
*/
|
|
484
|
+
async downloadBlob(hash, file) {
|
|
485
|
+
const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, hash)}`;
|
|
486
|
+
let res;
|
|
487
|
+
try {
|
|
488
|
+
res = await this.open("GET", url, void 0, {}, this.transfers, this.timeoutMs);
|
|
489
|
+
} catch (error) {
|
|
490
|
+
throw this.unreachable(error);
|
|
491
|
+
}
|
|
492
|
+
const status = res.statusCode ?? 0;
|
|
493
|
+
const access = this.refusedByAccess(status, res.headers.location, url);
|
|
494
|
+
if (status !== 200) {
|
|
495
|
+
res.resume();
|
|
496
|
+
if (access) throw access;
|
|
497
|
+
if (status === 404) return false;
|
|
498
|
+
if (status >= 500) throw new Unreachable(`the sync server failed (${status})`, true);
|
|
499
|
+
throw new ServerError(`the sync server refused the download of ${hash} (HTTP ${status})`, status);
|
|
500
|
+
}
|
|
501
|
+
const digest = createHash("sha256");
|
|
502
|
+
const tap = new Transform({ transform(chunk, _encoding, done) {
|
|
503
|
+
digest.update(chunk);
|
|
504
|
+
done(null, chunk);
|
|
505
|
+
} });
|
|
506
|
+
const out = createWriteStream(file);
|
|
507
|
+
let local;
|
|
508
|
+
out.once("error", (error) => local = error);
|
|
509
|
+
try {
|
|
510
|
+
await pipeline(decoded(res), tap, out);
|
|
511
|
+
} catch (error) {
|
|
512
|
+
if (local !== void 0) throw new LocalWriteError(`could not write the download here: ${local.message}`, local.code);
|
|
513
|
+
throw this.unreachable(error);
|
|
514
|
+
}
|
|
515
|
+
const got = digest.digest("hex");
|
|
516
|
+
if (got !== hash) throw new BlobMismatch(`the content downloaded as sha256 ${hash} is of ${got}`);
|
|
517
|
+
return true;
|
|
340
518
|
}
|
|
341
519
|
/**
|
|
342
520
|
* Keep the write connection open: a request that needs no hub, on it, unless a request is on it already. Opens it
|
|
@@ -352,6 +530,7 @@ var ServerApi = class {
|
|
|
352
530
|
close() {
|
|
353
531
|
this.agent.destroy();
|
|
354
532
|
this.reads.destroy();
|
|
533
|
+
this.transfers.destroy();
|
|
355
534
|
}
|
|
356
535
|
/** Send one request; the response is the hub's answer, applied or refused. */
|
|
357
536
|
async send(request, options = {}) {
|
|
@@ -377,7 +556,8 @@ var ServerApi = class {
|
|
|
377
556
|
* is below the batch before, as when the server's storage was replaced meanwhile, starts the snapshot over. A path of
|
|
378
557
|
* `have` the server never held a file at comes as `unknown` rather than `removed`, from a server that tells them apart.
|
|
379
558
|
* `listOnly` asks for the files under those prefixes as `listed`, without their text. A part of the hub alone is taken
|
|
380
|
-
* from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of.
|
|
559
|
+
* from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of. `blobs` takes
|
|
560
|
+
* the binary files too, each that differs from `have` as `blob`; `have` names a binary file by its sha256 then.
|
|
381
561
|
*/
|
|
382
562
|
async snapshot(have = /* @__PURE__ */ new Map(), options = {}) {
|
|
383
563
|
const paths = [...have.keys()].sort(comparePaths);
|
|
@@ -394,7 +574,8 @@ var ServerApi = class {
|
|
|
394
574
|
have: range,
|
|
395
575
|
unknown: true,
|
|
396
576
|
...after === void 0 ? {} : { after },
|
|
397
|
-
...options.listOnly?.length ? { listOnly: options.listOnly } : {}
|
|
577
|
+
...options.listOnly?.length ? { listOnly: options.listOnly } : {},
|
|
578
|
+
...options.blobs ? { blobs: true } : {}
|
|
398
579
|
});
|
|
399
580
|
if (parts.length > 0 && seq < parts.at(-1).seq) break;
|
|
400
581
|
parts.push({
|
|
@@ -431,13 +612,32 @@ var ServerApi = class {
|
|
|
431
612
|
}, "fresh");
|
|
432
613
|
return body;
|
|
433
614
|
}
|
|
434
|
-
/**
|
|
615
|
+
/**
|
|
616
|
+
* Open the hub's broadcast connection, taking the binary files too (ROUTES.socket). It opens before the client knows
|
|
617
|
+
* whether the server keeps binary files (blobRoute), and a server from before them reads nothing of `blobs`.
|
|
618
|
+
*/
|
|
435
619
|
socket() {
|
|
436
|
-
const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=3`;
|
|
620
|
+
const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=3&blobs=true`;
|
|
437
621
|
return new WebSocket(url, { headers: this.access });
|
|
438
622
|
}
|
|
439
623
|
};
|
|
440
624
|
//#endregion
|
|
625
|
+
//#region ../sync/dist/edit.js
|
|
626
|
+
/**
|
|
627
|
+
* The text an Edit of `edit` replaces in `text`: old_string, with the line break after it when new_string is empty and
|
|
628
|
+
* old_string ends in none (Claude Code 2.1.285). Null when old_string is empty or not in `text` exactly once, as the
|
|
629
|
+
* Edit then fails.
|
|
630
|
+
*/
|
|
631
|
+
function editTarget(text, edit) {
|
|
632
|
+
if (edit.old === "" || text.split(edit.old).length !== 2) return null;
|
|
633
|
+
return edit.new === "" && !edit.old.endsWith("\n") && text.includes(`${edit.old}\n`) ? `${edit.old}\n` : edit.old;
|
|
634
|
+
}
|
|
635
|
+
/** `text` with the Edit of `edit` made as Claude Code makes it, or null when the Edit fails on it. */
|
|
636
|
+
function applyEdit(text, edit) {
|
|
637
|
+
const target = editTarget(text, edit);
|
|
638
|
+
return target === null ? null : text.replace(target, () => edit.new);
|
|
639
|
+
}
|
|
640
|
+
//#endregion
|
|
441
641
|
//#region ../sync/dist/lines.js
|
|
442
642
|
const split = (text) => text.split("\n");
|
|
443
643
|
/**
|
|
@@ -694,6 +894,8 @@ function oversizedFiles(root) {
|
|
|
694
894
|
}
|
|
695
895
|
//#endregion
|
|
696
896
|
//#region ../sync/dist/state.js
|
|
897
|
+
/** The requestId an outbox entry goes under: its id, with the number of the try for a binary file's write sent again. */
|
|
898
|
+
const requestIdOf = (entry) => entry.kind === "blob" && entry.retry ? `${entry.id}-${entry.retry}` : entry.id;
|
|
697
899
|
/** Whether `entry` waits for the server: not one it refused. */
|
|
698
900
|
const waiting = (entry) => entry.refused === void 0;
|
|
699
901
|
/** A new entry's id. */
|
|
@@ -752,6 +954,8 @@ function writeJson(file, value) {
|
|
|
752
954
|
renameSync(temp, file);
|
|
753
955
|
}
|
|
754
956
|
const readKnown = (root) => readJson(syncPaths(root).known, {}).files ?? {};
|
|
957
|
+
/** What known.json writes before a binary file's sha256, apart from a text's content hash. */
|
|
958
|
+
const BLOB_PREFIX = "blob:";
|
|
755
959
|
/** The text of the version known.json names by `hash`; undefined when it was not kept, as by a sync client before known/. */
|
|
756
960
|
function readKnownText(root, hash) {
|
|
757
961
|
try {
|
|
@@ -765,7 +969,7 @@ function readKnownText(root, hash) {
|
|
|
765
969
|
* Write known.json, and into known/ the text of each version it names that is not
|
|
766
970
|
* there yet; the texts of versions it no longer names are removed. A version whose text the client does not have keeps
|
|
767
971
|
* the text known/ holds, if any. The new texts go first, then known.json, then the removals, so a client killed at any
|
|
768
|
-
* step leaves a known.json whose every text is in known/.
|
|
972
|
+
* step leaves a known.json whose every text is in known/. A binary file is given as its sha256 with BLOB_PREFIX, and no text.
|
|
769
973
|
*/
|
|
770
974
|
function writeKnown(root, files) {
|
|
771
975
|
const dir = syncPaths(root).knownTexts;
|
|
@@ -1108,6 +1312,13 @@ function exists(file) {
|
|
|
1108
1312
|
return false;
|
|
1109
1313
|
}
|
|
1110
1314
|
}
|
|
1315
|
+
const isBlob = (file) => typeof file === "object" && file !== null && file.kind === "blob";
|
|
1316
|
+
/** The hash the server names a file of its by: a text's content hash, a binary file's sha256. */
|
|
1317
|
+
const serverHash = (file) => typeof file === "string" ? contentHash(file) : file.hash;
|
|
1318
|
+
/** Whether two tool calls are of one session, the same subagent in it or both of the session itself. */
|
|
1319
|
+
const sameCaller = (a, b) => a !== void 0 && b !== void 0 && a.session === b.session && a.agent === b.agent;
|
|
1320
|
+
/** Whether `edit` makes `to` of `from`, as the Edit tool makes it (applyEdit). */
|
|
1321
|
+
const replaces = (edit, from, to) => applyEdit(from, edit) === to;
|
|
1111
1322
|
const HISTORY = 20;
|
|
1112
1323
|
/** The statuses of a closed mainline, whose artifacts stay on the server until they are used */
|
|
1113
1324
|
const CLOSED = /* @__PURE__ */ new Set(["done", "dropped"]);
|
|
@@ -1125,6 +1336,26 @@ const SETTLE_LIMIT_MS = 1e3;
|
|
|
1125
1336
|
* ones before the last are each followed by a pause before the next
|
|
1126
1337
|
*/
|
|
1127
1338
|
const STALLED_SNAPSHOTS = 4;
|
|
1339
|
+
/** How long after a failed probe of binary files (probeBlobs) the snapshot is taken again, at first and at most */
|
|
1340
|
+
const PROBE_RETRY_MS = 3e4;
|
|
1341
|
+
const PROBE_RETRY_MAX_MS = 6e5;
|
|
1342
|
+
/**
|
|
1343
|
+
* How long an upload of a binary file's content waits after a failure (uploadEntry): the first two failures are tried
|
|
1344
|
+
* again after BLOB_RETRY_MS, with up to as much again at random; from the third on, the wait doubles from
|
|
1345
|
+
* BLOB_BACKOFF_MS up to BLOB_BACKOFF_MAX_MS
|
|
1346
|
+
*/
|
|
1347
|
+
const BLOB_RETRY_MS = 1e3;
|
|
1348
|
+
const BLOB_BACKOFF_MS = 2e3;
|
|
1349
|
+
const BLOB_BACKOFF_MAX_MS = 6e5;
|
|
1350
|
+
/** Failures of an upload of the same write in a row before it waits longer each time and the writer is told */
|
|
1351
|
+
const BLOB_FAILURES = 3;
|
|
1352
|
+
/**
|
|
1353
|
+
* How long a download of a binary file's content waits after it failed (download), doubling from DOWNLOAD_BACKOFF_MS up to
|
|
1354
|
+
* BLOB_BACKOFF_MAX_MS. Content that does not match or cannot be written here is tried DOWNLOAD_TRIES times; then the file
|
|
1355
|
+
* is left as it is until the next connection.
|
|
1356
|
+
*/
|
|
1357
|
+
const DOWNLOAD_BACKOFF_MS = 2e3;
|
|
1358
|
+
const DOWNLOAD_TRIES = 5;
|
|
1128
1359
|
/** How long after a read that is not UTF-8 the file is read again before it counts as binary */
|
|
1129
1360
|
const BINARY_RECHECK_MS = 150;
|
|
1130
1361
|
/**
|
|
@@ -1147,6 +1378,91 @@ const OUTBOX = `${SYNC_DIR}/queue.json`;
|
|
|
1147
1378
|
const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
1148
1379
|
const decoder = new TextDecoder("utf-8", { fatal: true });
|
|
1149
1380
|
const sameLocal = (a, b) => a.kind === b.kind && (a.kind !== "text" || a.text === b.text) && (a.kind !== "binary" || a.hash === b.hash);
|
|
1381
|
+
/** What `bytes` read from a file are: a text when they are UTF-8, a binary file otherwise. */
|
|
1382
|
+
function contentOf(bytes) {
|
|
1383
|
+
try {
|
|
1384
|
+
return {
|
|
1385
|
+
kind: "text",
|
|
1386
|
+
text: decoder.decode(bytes)
|
|
1387
|
+
};
|
|
1388
|
+
} catch {
|
|
1389
|
+
return {
|
|
1390
|
+
kind: "binary",
|
|
1391
|
+
hash: sha256(bytes),
|
|
1392
|
+
size: bytes.length,
|
|
1393
|
+
bytes
|
|
1394
|
+
};
|
|
1395
|
+
}
|
|
1396
|
+
}
|
|
1397
|
+
/** The hash of `content`, as the server names it: a text's content hash, a binary file's sha256. */
|
|
1398
|
+
const hashOfContent = (content) => content.kind === "text" ? contentHash(content.text) : content.hash;
|
|
1399
|
+
/** Whether `content` is what the server holds as `file`. */
|
|
1400
|
+
const holdsServer = (content, file) => typeof file === "string" ? content.kind === "text" && content.text === file : isBlob(file) ? content.kind === "binary" && content.hash === file.hash : false;
|
|
1401
|
+
/**
|
|
1402
|
+
* The sha256 and size of the file at `file`, and whether it is UTF-8, read in chunks of a MiB: a file of up to
|
|
1403
|
+
* MAX_BLOB_BYTES is never held in memory whole.
|
|
1404
|
+
*/
|
|
1405
|
+
async function scanFile(file) {
|
|
1406
|
+
const handle = await open(file, "r");
|
|
1407
|
+
try {
|
|
1408
|
+
const digest = createHash("sha256");
|
|
1409
|
+
const utf8 = new TextDecoder("utf-8", { fatal: true });
|
|
1410
|
+
const chunk = Buffer.allocUnsafe(1 << 20);
|
|
1411
|
+
let size = 0;
|
|
1412
|
+
let text = true;
|
|
1413
|
+
for (;;) {
|
|
1414
|
+
const { bytesRead } = await handle.read(chunk, 0, chunk.length, null);
|
|
1415
|
+
if (bytesRead === 0) break;
|
|
1416
|
+
const part = chunk.subarray(0, bytesRead);
|
|
1417
|
+
digest.update(part);
|
|
1418
|
+
size += bytesRead;
|
|
1419
|
+
if (text) try {
|
|
1420
|
+
utf8.decode(part, { stream: true });
|
|
1421
|
+
} catch {
|
|
1422
|
+
text = false;
|
|
1423
|
+
}
|
|
1424
|
+
}
|
|
1425
|
+
if (text) try {
|
|
1426
|
+
utf8.decode();
|
|
1427
|
+
} catch {
|
|
1428
|
+
text = false;
|
|
1429
|
+
}
|
|
1430
|
+
return {
|
|
1431
|
+
hash: digest.digest("hex"),
|
|
1432
|
+
size,
|
|
1433
|
+
text
|
|
1434
|
+
};
|
|
1435
|
+
} finally {
|
|
1436
|
+
await handle.close();
|
|
1437
|
+
}
|
|
1438
|
+
}
|
|
1439
|
+
/** The sha256 of the file at `file`, read in chunks; null when it cannot be read. */
|
|
1440
|
+
function sha256File(file) {
|
|
1441
|
+
let fd;
|
|
1442
|
+
try {
|
|
1443
|
+
fd = openSync(file, "r");
|
|
1444
|
+
} catch {
|
|
1445
|
+
return null;
|
|
1446
|
+
}
|
|
1447
|
+
try {
|
|
1448
|
+
const digest = createHash("sha256");
|
|
1449
|
+
const chunk = Buffer.allocUnsafe(1 << 20);
|
|
1450
|
+
for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) digest.update(chunk.subarray(0, read));
|
|
1451
|
+
return digest.digest("hex");
|
|
1452
|
+
} catch {
|
|
1453
|
+
return null;
|
|
1454
|
+
} finally {
|
|
1455
|
+
closeSync(fd);
|
|
1456
|
+
}
|
|
1457
|
+
}
|
|
1458
|
+
/** What the client leaves on disk when a file holds `content`: a text with its hash, or a binary file's sha256. */
|
|
1459
|
+
const diskOf = (content) => content.kind === "text" ? {
|
|
1460
|
+
hash: contentHash(content.text),
|
|
1461
|
+
text: content.text
|
|
1462
|
+
} : {
|
|
1463
|
+
hash: content.hash,
|
|
1464
|
+
blob: true
|
|
1465
|
+
};
|
|
1150
1466
|
/**
|
|
1151
1467
|
* The queued edit of `path` to the writer's `text` that started from `start` (src/state.ts): on the server's version
|
|
1152
1468
|
* that holds a confirmed start, waiting for an unconfirmed one, or, without a start, on `base`, the hash of the version
|
|
@@ -1194,7 +1510,7 @@ function heldBase(entry) {
|
|
|
1194
1510
|
*/
|
|
1195
1511
|
function textAt(entry, path) {
|
|
1196
1512
|
if (entry.kind === "op") return entry.written[path];
|
|
1197
|
-
if (entry.path !== path) return void 0;
|
|
1513
|
+
if (entry.path !== path || entry.kind === "blob") return void 0;
|
|
1198
1514
|
if (entry.kind === "remove") return null;
|
|
1199
1515
|
return entry.text;
|
|
1200
1516
|
}
|
|
@@ -1233,7 +1549,7 @@ function patchRequest(entry) {
|
|
|
1233
1549
|
function holdsFiles(parts, held) {
|
|
1234
1550
|
const missing = /* @__PURE__ */ new Set();
|
|
1235
1551
|
for (const part of parts) for (const event of part.files) {
|
|
1236
|
-
if (event.kind === "file" || event.kind === "listed") return true;
|
|
1552
|
+
if (event.kind === "file" || event.kind === "listed" || event.kind === "blob") return true;
|
|
1237
1553
|
if (event.kind === "removed" || event.kind === "unknown") missing.add(event.path);
|
|
1238
1554
|
}
|
|
1239
1555
|
return [...held.keys()].some((path) => !missing.has(path));
|
|
@@ -1245,6 +1561,18 @@ var SyncClient = class extends EventEmitter {
|
|
|
1245
1561
|
api;
|
|
1246
1562
|
problems = [];
|
|
1247
1563
|
reported = 0;
|
|
1564
|
+
/** The local change of each path waiting for an Edit's report (awaitEdit) */
|
|
1565
|
+
awaitingEdit = /* @__PURE__ */ new Map();
|
|
1566
|
+
/** The Edits announced before they ran (announceEdit) and not reported yet, by path, oldest first */
|
|
1567
|
+
announced = /* @__PURE__ */ new Map();
|
|
1568
|
+
/** Edits reported before the change they made waited, by path, until the report's own replay on the path's chain ran */
|
|
1569
|
+
reportedEdits = /* @__PURE__ */ new Map();
|
|
1570
|
+
/** Flushes under way that send every change at once (flush): no change waits for an Edit's report meanwhile */
|
|
1571
|
+
sendingAll = 0;
|
|
1572
|
+
/** The files the tools reported by flushes under way wrote, by count: a change of them does not wait for an Edit's report */
|
|
1573
|
+
sendingPaths = /* @__PURE__ */ new Map();
|
|
1574
|
+
/** The path the reconcile or step on a path's chain that `work` tracks is about (schedule, withChain) */
|
|
1575
|
+
workPaths = /* @__PURE__ */ new Map();
|
|
1248
1576
|
/** What each path holds on disk as the client last left it, confirmed or not */
|
|
1249
1577
|
expected = /* @__PURE__ */ new Map();
|
|
1250
1578
|
/**
|
|
@@ -1263,17 +1591,65 @@ var SyncClient = class extends EventEmitter {
|
|
|
1263
1591
|
/** The content hash last worked out for a text the client named in a snapshot request, by path */
|
|
1264
1592
|
hashes = /* @__PURE__ */ new Map();
|
|
1265
1593
|
/**
|
|
1266
|
-
* The
|
|
1267
|
-
* it holds, or named removed. A file this client never synced, as one in a hub taken with git, that
|
|
1268
|
-
*
|
|
1594
|
+
* The content the last snapshot request named for each path, by its hash, with what the server made of it: left out,
|
|
1595
|
+
* as the same content it holds, or named removed. A file this client never synced, as one in a hub taken with git, that
|
|
1596
|
+
* still holds that content holds the server's version, or one the server removed since; not one written here (reconcile).
|
|
1269
1597
|
*/
|
|
1270
1598
|
snapshotAnswers = /* @__PURE__ */ new Map();
|
|
1271
1599
|
/**
|
|
1600
|
+
* Whether the server keeps binary files, as the probe of its route told (ServerApi.blobRoute); null until a connection
|
|
1601
|
+
* asked. Only then does a snapshot name binary files, and the outbox send one: a server from before binary files names
|
|
1602
|
+
* a path of `have` it holds no file at removed, which would remove the binary file here. While it is null, a binary file
|
|
1603
|
+
* waits as it is here, neither sent nor written over: the snapshot after the probe reconciles every file again.
|
|
1604
|
+
*/
|
|
1605
|
+
blobSupport = null;
|
|
1606
|
+
/** Whether the current connection asked the server whether it keeps binary files; cleared when a connection opens */
|
|
1607
|
+
probed = false;
|
|
1608
|
+
/** The contents this run uploaded, by sha256: a write of one is sent without uploading it again */
|
|
1609
|
+
uploadedBlobs = /* @__PURE__ */ new Set();
|
|
1610
|
+
/** The binary files the server refused as too large, as `path\0hash`: left as they are here, as larger than MAX_BLOB_BYTES */
|
|
1611
|
+
oversized = /* @__PURE__ */ new Set();
|
|
1612
|
+
/** The contents a download found missing on the server on this connection, by sha256, so a snapshot is reloaded once for each */
|
|
1613
|
+
missingDownloads = /* @__PURE__ */ new Set();
|
|
1614
|
+
/** The failed uploads of each binary file's write in a row this run, by outbox id (uploadEntry) */
|
|
1615
|
+
uploadFailures = /* @__PURE__ */ new Map();
|
|
1616
|
+
/**
|
|
1617
|
+
* Until when each binary file's write waits after its upload failed, by outbox id: meanwhile the writes of other paths
|
|
1618
|
+
* behind it go ahead (sendOutbox)
|
|
1619
|
+
*/
|
|
1620
|
+
uploadHolds = /* @__PURE__ */ new Map();
|
|
1621
|
+
/**
|
|
1622
|
+
* When the content of each sha256 the server is deleting may be uploaded again, as its Retry-After said: the outbox is
|
|
1623
|
+
* sent again on any change to queue.json, which would otherwise upload it again at once
|
|
1624
|
+
*/
|
|
1625
|
+
uploadAfter = /* @__PURE__ */ new Map();
|
|
1626
|
+
/** The download to try again shortly, after one failed while online: at most one timer for each path */
|
|
1627
|
+
downloadRetries = /* @__PURE__ */ new Map();
|
|
1628
|
+
/**
|
|
1629
|
+
* The failed downloads of each content in a row on this connection, by sha256, for the ones that came of another sha256
|
|
1630
|
+
* or could not be written here (download)
|
|
1631
|
+
*/
|
|
1632
|
+
downloadFailures = /* @__PURE__ */ new Map();
|
|
1633
|
+
/** Why the last download of each path failed, until one succeeds: what fetch tells its caller */
|
|
1634
|
+
downloadErrors = /* @__PURE__ */ new Map();
|
|
1635
|
+
/** The paths whose download failed DOWNLOAD_TRIES times in a row, reconciled again after the next snapshot */
|
|
1636
|
+
stalledDownloads = /* @__PURE__ */ new Set();
|
|
1637
|
+
/** The snapshot taken again after a failed probe (probeBlobs), and how long after the next failure it is */
|
|
1638
|
+
probeTimer;
|
|
1639
|
+
probeRetry = PROBE_RETRY_MS;
|
|
1640
|
+
/**
|
|
1272
1641
|
* Whether this run started without known.json, as in a hub taken with git that never synced here. Only then does the
|
|
1273
1642
|
* first snapshot's `removed` for a file the client named tell of a file the server removed since the copy: the server
|
|
1274
1643
|
* names removed every path it does not have, so later, or with known.json, such a file is one created here.
|
|
1275
1644
|
*/
|
|
1276
1645
|
copied = false;
|
|
1646
|
+
/**
|
|
1647
|
+
* Whether the binary files of a copy (copied) were compared with the server's yet: false until a snapshot that named
|
|
1648
|
+
* binary files ends, which the first snapshot does not when the client does not know yet that the server keeps them.
|
|
1649
|
+
* Until then a binary file that snapshot names removed was removed on the server since the copy, as `copied` tells of
|
|
1650
|
+
* a text in the first snapshot. Kept across runs by copyBlobs, since the first snapshot saves known.json.
|
|
1651
|
+
*/
|
|
1652
|
+
blobsCompared = true;
|
|
1277
1653
|
/** The server has no copy of the hub, which this environment, downloading on demand, does not upload (snapshotNow) */
|
|
1278
1654
|
uploadWanted = false;
|
|
1279
1655
|
/** Why the server refused the upload of this environment's copy, as last reported (snapshotNow); null once it holds the hub */
|
|
@@ -1425,15 +1801,17 @@ var SyncClient = class extends EventEmitter {
|
|
|
1425
1801
|
/**
|
|
1426
1802
|
* The file at `path` as the server last had it to this client's knowledge: its text, null when the server has no such
|
|
1427
1803
|
* file, or undefined when the client does not know it, such as before the first snapshot, or for a file only on the
|
|
1428
|
-
* server, whose text is not here.
|
|
1804
|
+
* server, whose text is not here, and for a binary file, which has no text.
|
|
1429
1805
|
*/
|
|
1430
1806
|
serverText(path) {
|
|
1431
1807
|
if (this.remote.has(path) || this.awaiting.has(path)) return void 0;
|
|
1432
|
-
|
|
1808
|
+
if (!this.cloud.has(path)) return this.synced ? null : void 0;
|
|
1809
|
+
const held = this.cloud.get(path);
|
|
1810
|
+
return isBlob(held) ? void 0 : held;
|
|
1433
1811
|
}
|
|
1434
1812
|
/** How many files are only on the server, not downloaded here (onDemand). */
|
|
1435
1813
|
get remoteCount() {
|
|
1436
|
-
return this.
|
|
1814
|
+
return this.listedRemote().size;
|
|
1437
1815
|
}
|
|
1438
1816
|
/**
|
|
1439
1817
|
* Whether the sync server has no copy of the hub and this environment, downloading on demand, waits for one that keeps
|
|
@@ -1446,6 +1824,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
1446
1824
|
async start() {
|
|
1447
1825
|
await mkdir(this.paths.tmp, { recursive: true });
|
|
1448
1826
|
await mkdir(this.paths.conflicts, { recursive: true });
|
|
1827
|
+
await mkdir(this.paths.blobs, { recursive: true });
|
|
1449
1828
|
if (this.opts.swap !== false) {
|
|
1450
1829
|
const swapper = startSwapper(this.paths.tmp);
|
|
1451
1830
|
this.swapper = swapper.helper;
|
|
@@ -1459,6 +1838,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
1459
1838
|
await this.restoreKnown();
|
|
1460
1839
|
this.restoreRemote();
|
|
1461
1840
|
await this.whenFree(() => this.syncQueue());
|
|
1841
|
+
await this.whenFree(() => this.locked(() => this.pruneStaged(this.queue)));
|
|
1462
1842
|
await this.resumeRefusals();
|
|
1463
1843
|
if (this.opts.watch !== false) this.startWatcher();
|
|
1464
1844
|
await this.scan(true, true);
|
|
@@ -1474,10 +1854,21 @@ var SyncClient = class extends EventEmitter {
|
|
|
1474
1854
|
* its write based on that change and undo it, and a change made while no client ran would have no base text to tell
|
|
1475
1855
|
* the writer's lines from another environment's (layer 2). Without its text, as known.json of an older client, only
|
|
1476
1856
|
* a file that still holds it gives it. What the outbox wrote, refused writes included, is read after it (restoreEntry).
|
|
1857
|
+
* A binary file's version is its sha256 alone (BLOB_PREFIX): it is no base for a write, which is never merged.
|
|
1477
1858
|
*/
|
|
1478
1859
|
async restoreKnown() {
|
|
1479
1860
|
this.copied = !existsSync(this.paths.known);
|
|
1861
|
+
this.blobsCompared = !this.copied && !existsSync(this.paths.copyBlobs);
|
|
1480
1862
|
for (const [path, hash] of Object.entries(readKnown(this.root))) {
|
|
1863
|
+
if (hash.startsWith("blob:")) {
|
|
1864
|
+
const disk = {
|
|
1865
|
+
hash: hash.slice(5),
|
|
1866
|
+
blob: true
|
|
1867
|
+
};
|
|
1868
|
+
this.expected.set(path, disk);
|
|
1869
|
+
this.confirmed.set(path, disk);
|
|
1870
|
+
continue;
|
|
1871
|
+
}
|
|
1481
1872
|
const local = await this.readLocal(path);
|
|
1482
1873
|
const text = local.kind === "text" && contentHash(local.text) === hash ? local.text : readKnownText(this.root, hash);
|
|
1483
1874
|
const disk = text === void 0 ? { hash } : {
|
|
@@ -1535,6 +1926,12 @@ var SyncClient = class extends EventEmitter {
|
|
|
1535
1926
|
clearTimeout(this.queueTimer);
|
|
1536
1927
|
for (const timer of this.watchTimers.values()) clearTimeout(timer);
|
|
1537
1928
|
this.watchTimers.clear();
|
|
1929
|
+
for (const timer of this.downloadRetries.values()) clearTimeout(timer);
|
|
1930
|
+
this.downloadRetries.clear();
|
|
1931
|
+
clearTimeout(this.probeTimer);
|
|
1932
|
+
for (const wait of this.awaitingEdit.values()) wait.end(null);
|
|
1933
|
+
this.reportedEdits.clear();
|
|
1934
|
+
this.announced.clear();
|
|
1538
1935
|
this.watcher?.close();
|
|
1539
1936
|
this.ws?.close();
|
|
1540
1937
|
const settled = Promise.allSettled([
|
|
@@ -1566,14 +1963,19 @@ var SyncClient = class extends EventEmitter {
|
|
|
1566
1963
|
this.connect();
|
|
1567
1964
|
});
|
|
1568
1965
|
}
|
|
1569
|
-
/**
|
|
1570
|
-
|
|
1966
|
+
/**
|
|
1967
|
+
* Resolve once nothing is being sent, applied or waited for, and the queue is empty or cannot be sent now. With
|
|
1968
|
+
* `besidesAwaiting`, the work on a path waiting for an Edit's report (awaitEdit) is left out, as is the work behind it
|
|
1969
|
+
* on that path's chain: a tool's flush does not wait for an Edit of another session to be reported.
|
|
1970
|
+
*/
|
|
1971
|
+
async idle(timeoutMs = 3e4, besidesAwaiting = false) {
|
|
1571
1972
|
const deadline = Date.now() + timeoutMs;
|
|
1572
1973
|
this.outboxChanged();
|
|
1573
1974
|
for (;;) {
|
|
1574
|
-
|
|
1975
|
+
const work = besidesAwaiting ? [...this.work].filter((promise) => !this.awaitingEdit.has(this.workPaths.get(promise) ?? "")) : [...this.work];
|
|
1976
|
+
if (!(work.length > 0 || [...this.inflight.values()].some((n) => n > 0) || this.flushing || this.resyncing !== null || this.online && this.nextEntry(this.queue).entry !== void 0)) return true;
|
|
1575
1977
|
if (Date.now() > deadline) return false;
|
|
1576
|
-
await (
|
|
1978
|
+
await (work.length > 0 ? Promise.race([Promise.allSettled(work), sleep$1(50)]) : sleep$1(50));
|
|
1577
1979
|
}
|
|
1578
1980
|
}
|
|
1579
1981
|
connect() {
|
|
@@ -1591,6 +1993,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
1591
1993
|
this.connected = true;
|
|
1592
1994
|
this.rejoining = false;
|
|
1593
1995
|
this.current = false;
|
|
1996
|
+
this.probed = false;
|
|
1997
|
+
this.missingDownloads.clear();
|
|
1998
|
+
this.downloadFailures.clear();
|
|
1594
1999
|
if (!this.uploadWanted && this.uploadRefusal === null) this.backoff = 500;
|
|
1595
2000
|
this.lastPong = Date.now();
|
|
1596
2001
|
clearInterval(this.pingTimer);
|
|
@@ -1725,7 +2130,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
1725
2130
|
* One file as the server has it as of the hub's `seq`, from a broadcast or a snapshot. A version no newer than the
|
|
1726
2131
|
* one the client holds, such as a broadcast that arrives after a later snapshot, is left out. A file only on the
|
|
1727
2132
|
* server whose mainline is still closed gets its new hash, or leaves `remote` when removed; nothing is written here.
|
|
1728
|
-
* A mainline's file that opens it again brings its artifacts here.
|
|
2133
|
+
* A mainline's file that opens it again brings its artifacts here. A binary file comes as its hash (`blob`), and one
|
|
2134
|
+
* that may stay on the server, as a new screenshot of a closed mainline, goes into `remote` instead of being downloaded,
|
|
2135
|
+
* as a listed text does in a snapshot.
|
|
1729
2136
|
*/
|
|
1730
2137
|
receive(event, seq) {
|
|
1731
2138
|
if (event.kind === "dir") {
|
|
@@ -1733,26 +2140,42 @@ var SyncClient = class extends EventEmitter {
|
|
|
1733
2140
|
return;
|
|
1734
2141
|
}
|
|
1735
2142
|
if (!this.syncs(event.path)) return;
|
|
1736
|
-
const
|
|
2143
|
+
const file = event.kind === "file" ? event.text : event.kind === "blob" ? {
|
|
2144
|
+
kind: "blob",
|
|
2145
|
+
hash: event.hash,
|
|
2146
|
+
size: event.size
|
|
2147
|
+
} : null;
|
|
2148
|
+
const label = isBlob(file) ? `blob ${file.hash.slice(0, 7)}` : short(file);
|
|
1737
2149
|
const held = this.fileSeq.get(event.path) ?? -1;
|
|
1738
2150
|
if (seq <= held) {
|
|
1739
|
-
this.trace(`older ${event.path} ${
|
|
2151
|
+
this.trace(`older ${event.path} ${label} seq ${seq}, holding seq ${held}`);
|
|
1740
2152
|
return;
|
|
1741
2153
|
}
|
|
1742
2154
|
this.fileSeq.set(event.path, seq);
|
|
2155
|
+
if (this.downloadErrors.has(event.path)) {
|
|
2156
|
+
const before = this.cloud.get(event.path);
|
|
2157
|
+
if (!(isBlob(file) && isBlob(before) && before.hash === file.hash) && this.downloadErrors.delete(event.path)) this.saveRemote();
|
|
2158
|
+
}
|
|
1743
2159
|
if (event.kind === "unknown") this.unheld.add(event.path);
|
|
1744
2160
|
else this.unheld.delete(event.path);
|
|
1745
|
-
if (this.remote.has(event.path) && !this.awaiting.has(event.path) && (
|
|
1746
|
-
this.trace(`remote ${event.path} ${
|
|
1747
|
-
if (
|
|
1748
|
-
else this.remote.set(event.path,
|
|
2161
|
+
if (this.remote.has(event.path) && !this.awaiting.has(event.path) && (file === null || this.staysRemote(event.path))) {
|
|
2162
|
+
this.trace(`remote ${event.path} ${label} seq ${seq}`);
|
|
2163
|
+
if (file === null) this.remote.delete(event.path);
|
|
2164
|
+
else this.remote.set(event.path, serverHash(file));
|
|
2165
|
+
this.saveRemote();
|
|
2166
|
+
return;
|
|
2167
|
+
}
|
|
2168
|
+
if (isBlob(file) && !this.awaiting.has(event.path) && this.mayStay(event.path) && !existsSync(join(this.paths.data, event.path))) {
|
|
2169
|
+
this.trace(`remote ${event.path} ${label} seq ${seq}`);
|
|
2170
|
+
this.remote.set(event.path, file.hash);
|
|
2171
|
+
this.cloud.delete(event.path);
|
|
1749
2172
|
this.saveRemote();
|
|
1750
2173
|
return;
|
|
1751
2174
|
}
|
|
1752
2175
|
if (this.remote.delete(event.path)) this.saveRemote();
|
|
1753
|
-
this.trace(`receive ${event.path} ${
|
|
2176
|
+
this.trace(`receive ${event.path} ${label} seq ${seq}`);
|
|
1754
2177
|
this.awaiting.delete(event.path);
|
|
1755
|
-
this.cloud.set(event.path,
|
|
2178
|
+
this.cloud.set(event.path, file);
|
|
1756
2179
|
this.schedule(event.path);
|
|
1757
2180
|
const mainline = MAINLINE_FILE.exec(event.path)?.[1];
|
|
1758
2181
|
if (mainline !== void 0 && this.remote.size > 0) this.fetchOpened(mainline);
|
|
@@ -1865,6 +2288,14 @@ var SyncClient = class extends EventEmitter {
|
|
|
1865
2288
|
* downloaded before the snapshot ends, with the files only on the server whose mainline is open now. The first
|
|
1866
2289
|
* snapshot of a hub without known.json lists all of mainlines/, since the client may not know a mainline or its
|
|
1867
2290
|
* status yet: the mainlines' own files are downloaded after, then the artifacts of the open ones.
|
|
2291
|
+
*
|
|
2292
|
+
* Each connection first asks whether the server keeps binary files (blobSupport). One that does is sent the binary
|
|
2293
|
+
* files the client holds by their sha256, and tells of its own as `blob`; one that does not is told of none, and tells
|
|
2294
|
+
* of none, nor any path it holds one at: such a path is neither named nor taken for removed. A server the client
|
|
2295
|
+
* uploaded the hub to holds none of them, since an upload holds texts alone: known.json stops naming them before the
|
|
2296
|
+
* upload (forgetBinaries), so they are files never synced here, sent as new even when this snapshot fails or the
|
|
2297
|
+
* client is killed; and in the snapshot after the upload, a binary file of the client's the server names removed, as
|
|
2298
|
+
* when it once held one there, is taken as one it never held (unheld), also in a run that started without known.json.
|
|
1868
2299
|
*/
|
|
1869
2300
|
takeSnapshot() {
|
|
1870
2301
|
return this.exclusive(() => this.snapshotNow());
|
|
@@ -1872,6 +2303,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
1872
2303
|
async snapshotNow() {
|
|
1873
2304
|
let parts;
|
|
1874
2305
|
let held;
|
|
2306
|
+
let heldBlobs;
|
|
1875
2307
|
let have;
|
|
1876
2308
|
let before;
|
|
1877
2309
|
/** Whether the hub was uploaded, and why the server refused the upload; asked again, the snapshot may hold the hub */
|
|
@@ -1879,14 +2311,26 @@ var SyncClient = class extends EventEmitter {
|
|
|
1879
2311
|
let refusal = null;
|
|
1880
2312
|
/** The rules' generation the snapshot is asked for under: what it holds is taken in as they leave it in */
|
|
1881
2313
|
let asked;
|
|
2314
|
+
/** Whether the snapshot is asked for with the binary files */
|
|
2315
|
+
let withBlobs = false;
|
|
2316
|
+
if (!this.probed) await this.probeBlobs();
|
|
1882
2317
|
for (;;) {
|
|
1883
2318
|
before = this.hubSeq;
|
|
1884
|
-
held = await this.holdings();
|
|
1885
|
-
for (const path of this.remote.keys()) if (held.has(path) || this.expected.get(path) !== void 0) this.remote.delete(path);
|
|
2319
|
+
({texts: held, blobs: heldBlobs} = await this.holdings());
|
|
2320
|
+
for (const path of this.remote.keys()) if (held.has(path) || heldBlobs.has(path) || this.expected.get(path) !== void 0) this.remote.delete(path);
|
|
1886
2321
|
have = new Map([...held].map(([path, text]) => [path, this.hashOf(path, text)]));
|
|
2322
|
+
for (const [path, blob] of heldBlobs) have.set(path, blob.hash);
|
|
1887
2323
|
for (const [path, hash] of this.remote) have.set(path, hash);
|
|
1888
2324
|
asked = this.scope.generation;
|
|
1889
|
-
|
|
2325
|
+
withBlobs = this.blobSupport === true;
|
|
2326
|
+
if (!withBlobs && !this.blobsCompared && !existsSync(this.paths.copyBlobs)) {
|
|
2327
|
+
mkdirSync(this.paths.own, { recursive: true });
|
|
2328
|
+
writeFileSync(this.paths.copyBlobs, "");
|
|
2329
|
+
}
|
|
2330
|
+
parts = await this.api.snapshot(have, {
|
|
2331
|
+
listOnly: this.listOnly(this.copied && !this.synced),
|
|
2332
|
+
blobs: this.blobSupport === true
|
|
2333
|
+
});
|
|
1890
2334
|
if (uploaded || holdsFiles(parts, have)) break;
|
|
1891
2335
|
if (this.opts.settings.onDemand === true) {
|
|
1892
2336
|
if (!this.uploadWanted) {
|
|
@@ -1927,6 +2371,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
1927
2371
|
let sent = 0;
|
|
1928
2372
|
this.snapshotAnswers.clear();
|
|
1929
2373
|
const fromCopy = this.copied && !this.synced;
|
|
2374
|
+
const blobsFromCopy = withBlobs && !this.blobsCompared;
|
|
1930
2375
|
for (const part of parts) for (const event of part.files) {
|
|
1931
2376
|
if (event.kind !== "dir" && !this.syncs(event.path)) continue;
|
|
1932
2377
|
if (event.kind === "listed") {
|
|
@@ -1940,21 +2385,31 @@ var SyncClient = class extends EventEmitter {
|
|
|
1940
2385
|
continue;
|
|
1941
2386
|
}
|
|
1942
2387
|
named.add(event.path);
|
|
1943
|
-
if (event.kind === "file") {
|
|
2388
|
+
if (event.kind === "file" || event.kind === "blob") {
|
|
1944
2389
|
present.add(event.path);
|
|
1945
|
-
sent++;
|
|
2390
|
+
if (event.kind === "file") sent++;
|
|
2391
|
+
}
|
|
2392
|
+
if (uploaded && event.kind === "removed" && heldBlobs.has(event.path)) {
|
|
2393
|
+
this.receive({
|
|
2394
|
+
kind: "unknown",
|
|
2395
|
+
path: event.path
|
|
2396
|
+
}, part.seq);
|
|
2397
|
+
continue;
|
|
2398
|
+
}
|
|
2399
|
+
if (event.kind === "removed" && (held.has(event.path) ? fromCopy : blobsFromCopy && heldBlobs.has(event.path))) {
|
|
2400
|
+
const text = held.get(event.path);
|
|
2401
|
+
this.snapshotAnswers.set(event.path, {
|
|
2402
|
+
hash: text === void 0 ? heldBlobs.get(event.path).hash : this.hashOf(event.path, text),
|
|
2403
|
+
removed: true
|
|
2404
|
+
});
|
|
1946
2405
|
}
|
|
1947
|
-
if (event.kind === "removed" && fromCopy && held.has(event.path)) this.snapshotAnswers.set(event.path, {
|
|
1948
|
-
text: held.get(event.path),
|
|
1949
|
-
removed: true
|
|
1950
|
-
});
|
|
1951
2406
|
this.receive(event, part.seq);
|
|
1952
2407
|
}
|
|
1953
2408
|
for (const [path, text] of held) {
|
|
1954
2409
|
if (named.has(path)) continue;
|
|
1955
2410
|
present.add(path);
|
|
1956
2411
|
this.snapshotAnswers.set(path, {
|
|
1957
|
-
text,
|
|
2412
|
+
hash: this.hashOf(path, text),
|
|
1958
2413
|
removed: false
|
|
1959
2414
|
});
|
|
1960
2415
|
this.receive({
|
|
@@ -1963,37 +2418,123 @@ var SyncClient = class extends EventEmitter {
|
|
|
1963
2418
|
text
|
|
1964
2419
|
}, seq);
|
|
1965
2420
|
}
|
|
1966
|
-
for (const path
|
|
1967
|
-
|
|
1968
|
-
path
|
|
1969
|
-
|
|
2421
|
+
for (const [path, blob] of heldBlobs) {
|
|
2422
|
+
if (named.has(path)) continue;
|
|
2423
|
+
present.add(path);
|
|
2424
|
+
this.snapshotAnswers.set(path, {
|
|
2425
|
+
hash: blob.hash,
|
|
2426
|
+
removed: false
|
|
2427
|
+
});
|
|
2428
|
+
this.receive({
|
|
2429
|
+
kind: "blob",
|
|
2430
|
+
path,
|
|
2431
|
+
hash: blob.hash,
|
|
2432
|
+
size: blob.size ?? 0
|
|
2433
|
+
}, seq);
|
|
2434
|
+
}
|
|
2435
|
+
const blind = this.blobSupport !== true;
|
|
2436
|
+
for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
|
|
2437
|
+
if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
|
|
2438
|
+
if (blind && this.knownBinary(path)) continue;
|
|
2439
|
+
this.receive({
|
|
2440
|
+
kind: "removed",
|
|
2441
|
+
path
|
|
2442
|
+
}, seq);
|
|
2443
|
+
}
|
|
1970
2444
|
this.scopeTaken = asked;
|
|
1971
2445
|
this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
|
|
1972
2446
|
this.takenUpTo(seq);
|
|
1973
2447
|
this.synced = true;
|
|
2448
|
+
if (withBlobs && !this.blobsCompared) {
|
|
2449
|
+
this.blobsCompared = true;
|
|
2450
|
+
rmSync(this.paths.copyBlobs, { force: true });
|
|
2451
|
+
}
|
|
1974
2452
|
this.saveRemote();
|
|
1975
2453
|
await this.fetchWanted(wanted);
|
|
1976
2454
|
await this.settleUnsynced();
|
|
1977
2455
|
this.wake();
|
|
1978
2456
|
for (const path of await this.walk()) this.schedule(path);
|
|
2457
|
+
for (const path of this.stalledDownloads) this.schedule(path);
|
|
2458
|
+
this.stalledDownloads.clear();
|
|
2459
|
+
}
|
|
2460
|
+
/**
|
|
2461
|
+
* Ask the server whether it keeps binary files (blobSupport), once per connection, before its snapshot. A probe that
|
|
2462
|
+
* fails leaves what the last one found, null on a first connection, so the texts sync all the same and binary files
|
|
2463
|
+
* wait as they are here: the snapshot is taken again after PROBE_RETRY_MS, doubling up to PROBE_RETRY_MAX_MS while it
|
|
2464
|
+
* keeps failing, and asks again.
|
|
2465
|
+
*/
|
|
2466
|
+
async probeBlobs() {
|
|
2467
|
+
const had = this.blobSupport;
|
|
2468
|
+
try {
|
|
2469
|
+
await this.askBlobs();
|
|
2470
|
+
} finally {
|
|
2471
|
+
if (had !== true && this.blobSupport === true) this.fileSeq.clear();
|
|
2472
|
+
}
|
|
2473
|
+
}
|
|
2474
|
+
async askBlobs() {
|
|
2475
|
+
const had = this.blobSupport;
|
|
2476
|
+
try {
|
|
2477
|
+
this.blobSupport = await this.api.blobRoute();
|
|
2478
|
+
} catch (error) {
|
|
2479
|
+
if (!(error instanceof Unreachable)) throw error;
|
|
2480
|
+
this.log(`could not ask the sync server whether it keeps binary files (${error.message}); binary files wait for a later snapshot`);
|
|
2481
|
+
clearTimeout(this.probeTimer);
|
|
2482
|
+
this.probeTimer = setTimeout(() => {
|
|
2483
|
+
if (this.connected && !this.probed) this.track(this.resync().catch((failed) => this.log(`could not reload the snapshot: ${failed.message}`)));
|
|
2484
|
+
}, this.probeRetry);
|
|
2485
|
+
this.probeRetry = Math.min(this.probeRetry * 2, PROBE_RETRY_MAX_MS);
|
|
2486
|
+
return;
|
|
2487
|
+
}
|
|
2488
|
+
this.probed = true;
|
|
2489
|
+
this.probeRetry = PROBE_RETRY_MS;
|
|
2490
|
+
if (had !== this.blobSupport) this.log(this.blobSupport ? "the sync server keeps binary files" : "the sync server keeps no binary files, as one from before them: binary files are not synced");
|
|
1979
2491
|
}
|
|
1980
2492
|
/**
|
|
1981
2493
|
* The text the client holds of each synced file, which a snapshot request names by hash so the server leaves out what
|
|
1982
2494
|
* is the same (SnapshotRequest.have): the server's version this run last took in, else the version known.json names,
|
|
1983
2495
|
* else the file on disk, as in a hub cloned with git that never synced here. Only a text the client has is named,
|
|
1984
2496
|
* since a file the server leaves out holds the text named for it; a file larger than the server keeps is not synced,
|
|
1985
|
-
* so it is not named either.
|
|
2497
|
+
* so it is not named either. The binary files (`blobs`) come in the same order, by their sha256, only for a server that
|
|
2498
|
+
* keeps them (blobSupport), and only ones the client sends (binarySkip).
|
|
1986
2499
|
*/
|
|
1987
2500
|
async holdings() {
|
|
1988
|
-
const
|
|
1989
|
-
|
|
1990
|
-
|
|
2501
|
+
const texts = /* @__PURE__ */ new Map();
|
|
2502
|
+
const blobs = /* @__PURE__ */ new Map();
|
|
2503
|
+
const support = this.blobSupport === true;
|
|
2504
|
+
const unnamed = /* @__PURE__ */ new Set();
|
|
2505
|
+
for (const [path, file] of this.cloud) {
|
|
2506
|
+
if (!this.syncs(path)) continue;
|
|
2507
|
+
if (typeof file === "string") texts.set(path, file);
|
|
2508
|
+
else if (isBlob(file) && support) blobs.set(path, file);
|
|
2509
|
+
else if (isBlob(file)) unnamed.add(path);
|
|
2510
|
+
}
|
|
2511
|
+
for (const [path, disk] of this.confirmed) {
|
|
2512
|
+
if (texts.has(path) || blobs.has(path) || unnamed.has(path) || !this.syncs(path)) continue;
|
|
2513
|
+
if (disk?.text !== void 0) texts.set(path, disk.text);
|
|
2514
|
+
else if (disk?.blob && support) blobs.set(path, {
|
|
2515
|
+
kind: "blob",
|
|
2516
|
+
hash: disk.hash
|
|
2517
|
+
});
|
|
2518
|
+
else if (disk?.blob) unnamed.add(path);
|
|
2519
|
+
}
|
|
1991
2520
|
for (const path of await this.walk()) {
|
|
1992
|
-
if (
|
|
2521
|
+
if (texts.has(path) || blobs.has(path) || unnamed.has(path)) continue;
|
|
1993
2522
|
const local = await this.readLocal(path);
|
|
1994
|
-
if (local.kind === "text" && !exceedsFileLimit(local.text))
|
|
2523
|
+
if (local.kind === "text" && !exceedsFileLimit(local.text)) texts.set(path, local.text);
|
|
2524
|
+
else if (local.kind === "binary" && support && !this.binarySkip(path, local, false)) blobs.set(path, {
|
|
2525
|
+
kind: "blob",
|
|
2526
|
+
hash: local.hash,
|
|
2527
|
+
size: local.size
|
|
2528
|
+
});
|
|
1995
2529
|
}
|
|
1996
|
-
return
|
|
2530
|
+
return {
|
|
2531
|
+
texts,
|
|
2532
|
+
blobs
|
|
2533
|
+
};
|
|
2534
|
+
}
|
|
2535
|
+
/** Whether the client knows `path` as a binary file: the server's version it took in, or what it left on disk. */
|
|
2536
|
+
knownBinary(path) {
|
|
2537
|
+
return isBlob(this.cloud.get(path)) || this.expected.get(path)?.blob === true;
|
|
1997
2538
|
}
|
|
1998
2539
|
/** The content hash of `text`, the one `path` was named with, worked out once per text. */
|
|
1999
2540
|
hashOf(path, text) {
|
|
@@ -2032,6 +2573,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2032
2573
|
}
|
|
2033
2574
|
if (Object.keys(files).length === 0) return null;
|
|
2034
2575
|
const dirs = await this.walkDirs();
|
|
2576
|
+
const forgotten = await this.forgetBinaries();
|
|
2035
2577
|
let result;
|
|
2036
2578
|
try {
|
|
2037
2579
|
result = await this.api.upload(files, dirs);
|
|
@@ -2044,14 +2586,49 @@ var SyncClient = class extends EventEmitter {
|
|
|
2044
2586
|
}
|
|
2045
2587
|
if (!result.ok) {
|
|
2046
2588
|
this.log(`upload refused: ${result.message}`);
|
|
2589
|
+
this.recallBinaries(forgotten);
|
|
2047
2590
|
return {
|
|
2048
2591
|
ok: false,
|
|
2049
2592
|
message: result.message
|
|
2050
2593
|
};
|
|
2051
2594
|
}
|
|
2052
2595
|
this.log(`uploaded the hub: ${result.files} files`);
|
|
2596
|
+
await this.whenFree(() => this.editQueue((entries) => {
|
|
2597
|
+
const stale = entries.filter((entry) => entry.kind === "blob" && entry.sent === void 0 && waiting(entry) && entry.base !== null);
|
|
2598
|
+
for (const entry of stale) if (entry.kind === "blob") entry.base = null;
|
|
2599
|
+
return stale.length > 0;
|
|
2600
|
+
}));
|
|
2053
2601
|
return { ok: true };
|
|
2054
2602
|
}
|
|
2603
|
+
/**
|
|
2604
|
+
* Before an upload of the hub, which holds texts alone: known.json stops naming the binary files the server confirmed,
|
|
2605
|
+
* and the client stops expecting them, saved before the upload is sent. Whatever happens after it, a failed snapshot
|
|
2606
|
+
* or a client killed, the client then takes each for a file never synced here, and sends it as new (reconcile), rather
|
|
2607
|
+
* than for one the server removed, which would remove it here. A run that started without known.json confirmed none,
|
|
2608
|
+
* and writes no known.json here: a refused upload would leave it taking a text the server removed since the copy for
|
|
2609
|
+
* one created here. Returns what it forgot, for recallBinaries.
|
|
2610
|
+
*/
|
|
2611
|
+
async forgetBinaries() {
|
|
2612
|
+
const forgotten = /* @__PURE__ */ new Map();
|
|
2613
|
+
for (const [path, disk] of this.confirmed) if (disk?.blob) forgotten.set(path, disk);
|
|
2614
|
+
for (const [path, disk] of forgotten) {
|
|
2615
|
+
this.confirmed.delete(path);
|
|
2616
|
+
this.snapshotAnswers.delete(path);
|
|
2617
|
+
const expected = this.expected.get(path);
|
|
2618
|
+
if (expected?.blob && expected.hash === disk.hash) this.expected.delete(path);
|
|
2619
|
+
if (isBlob(this.cloud.get(path))) this.cloud.delete(path);
|
|
2620
|
+
}
|
|
2621
|
+
if (forgotten.size > 0) await this.whenFree(() => underLock(this.root, () => writeKnown(this.root, this.knownFiles())));
|
|
2622
|
+
return forgotten;
|
|
2623
|
+
}
|
|
2624
|
+
/** The server refused the upload, so it holds what it held: the binary files forgetBinaries forgot are known again. */
|
|
2625
|
+
recallBinaries(forgotten) {
|
|
2626
|
+
for (const [path, disk] of forgotten) {
|
|
2627
|
+
if (!this.confirmed.has(path)) this.confirmed.set(path, disk);
|
|
2628
|
+
if (!this.expected.has(path)) this.expected.set(path, disk);
|
|
2629
|
+
}
|
|
2630
|
+
if (forgotten.size > 0) this.saveKnown();
|
|
2631
|
+
}
|
|
2055
2632
|
/** Tell the writer, once for each reason, that the server has no copy of the hub and refused this one (snapshotNow). */
|
|
2056
2633
|
refusedUpload(message) {
|
|
2057
2634
|
if (this.uploadRefusal === message) return;
|
|
@@ -2067,13 +2644,18 @@ var SyncClient = class extends EventEmitter {
|
|
|
2067
2644
|
if (this.closing) return;
|
|
2068
2645
|
const next = (this.chains.get(path) ?? Promise.resolve()).then(() => this.reconcile(path)).catch((error) => this.log(`${path}: ${error.stack ?? String(error)}`));
|
|
2069
2646
|
this.chains.set(path, next);
|
|
2070
|
-
this.track(next).then(() => {
|
|
2647
|
+
this.track(next, path).then(() => {
|
|
2071
2648
|
if (this.chains.get(path) === next) this.chains.delete(path);
|
|
2072
2649
|
});
|
|
2073
2650
|
}
|
|
2074
|
-
|
|
2651
|
+
/** Count `promise` as work under way until it settles; `path` for one on that path's chain. */
|
|
2652
|
+
track(promise, path) {
|
|
2075
2653
|
this.work.add(promise);
|
|
2076
|
-
|
|
2654
|
+
if (path !== void 0) this.workPaths.set(promise, path);
|
|
2655
|
+
promise.finally(() => {
|
|
2656
|
+
this.work.delete(promise);
|
|
2657
|
+
this.workPaths.delete(promise);
|
|
2658
|
+
}).catch(() => {});
|
|
2077
2659
|
return promise;
|
|
2078
2660
|
}
|
|
2079
2661
|
/**
|
|
@@ -2084,26 +2666,34 @@ var SyncClient = class extends EventEmitter {
|
|
|
2084
2666
|
pending(path) {
|
|
2085
2667
|
return (this.inflight.get(path) ?? 0) > 0 || this.queue.some((entry) => waiting(entry) && entryPaths(entry).includes(path));
|
|
2086
2668
|
}
|
|
2669
|
+
/**
|
|
2670
|
+
* What the file at `path` holds now. A file larger than any text the server keeps (MAX_FILE_BYTES) is read in chunks
|
|
2671
|
+
* first (scanFile): a binary file comes as its sha256 and size alone, so every reconcile after a snapshot, which reads
|
|
2672
|
+
* every file, holds at most a chunk of it, and its bytes are read again only to be saved (bytesOf). Such a file that is
|
|
2673
|
+
* UTF-8 is read whole, as a smaller file is, since the text is what reconcile compares; it is not sent either way
|
|
2674
|
+
* (exceedsFileLimit).
|
|
2675
|
+
*/
|
|
2087
2676
|
async readLocal(path) {
|
|
2088
|
-
|
|
2677
|
+
const file = join(this.paths.data, path);
|
|
2089
2678
|
try {
|
|
2090
|
-
|
|
2679
|
+
if ((await stat(file)).size > 19e5) {
|
|
2680
|
+
const scanned = await scanFile(file);
|
|
2681
|
+
if (!scanned.text) return {
|
|
2682
|
+
kind: "binary",
|
|
2683
|
+
hash: scanned.hash,
|
|
2684
|
+
size: scanned.size
|
|
2685
|
+
};
|
|
2686
|
+
}
|
|
2687
|
+
return contentOf(await readFile(file));
|
|
2091
2688
|
} catch (error) {
|
|
2092
2689
|
const code = error.code;
|
|
2093
2690
|
if (code === "ENOENT" || code === "ENOTDIR" || code === "EISDIR") return { kind: "absent" };
|
|
2094
2691
|
throw error;
|
|
2095
2692
|
}
|
|
2096
|
-
|
|
2097
|
-
|
|
2098
|
-
|
|
2099
|
-
|
|
2100
|
-
};
|
|
2101
|
-
} catch {
|
|
2102
|
-
return {
|
|
2103
|
-
kind: "binary",
|
|
2104
|
-
hash: contentHash(bytes.toString("latin1"))
|
|
2105
|
-
};
|
|
2106
|
-
}
|
|
2693
|
+
}
|
|
2694
|
+
/** The bytes of binary file `content` at `path`: read again from the file when readLocal read it in chunks. */
|
|
2695
|
+
async bytesOf(path, content) {
|
|
2696
|
+
return content.bytes ?? await readFile(join(this.paths.data, path));
|
|
2107
2697
|
}
|
|
2108
2698
|
/**
|
|
2109
2699
|
* Compare the local file with what the client last left there and with the server's version, and act: queue a
|
|
@@ -2119,9 +2709,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
2119
2709
|
await sleep$1(BINARY_RECHECK_MS);
|
|
2120
2710
|
const again = await this.readLocal(path);
|
|
2121
2711
|
if (again.kind !== "binary" || again.hash !== local.hash) return this.schedule(path);
|
|
2122
|
-
|
|
2712
|
+
if (this.binarySkip(path, local, true)) return;
|
|
2123
2713
|
}
|
|
2124
|
-
if (local.kind === "text" && exceedsFileLimit(local.text)) return this.skipLarge(path, local.text);
|
|
2714
|
+
if (local.kind === "text" && exceedsFileLimit(local.text)) return this.skipLarge(path, contentHash(local.text), false);
|
|
2125
2715
|
const seen = this.expected.get(path);
|
|
2126
2716
|
if (seen === void 0 ? local.kind !== "absent" : !this.matches(seen, local)) {
|
|
2127
2717
|
await this.whenFree(() => this.syncQueue());
|
|
@@ -2130,48 +2720,49 @@ var SyncClient = class extends EventEmitter {
|
|
|
2130
2720
|
const after = this.expected.get(path);
|
|
2131
2721
|
if (after !== seen && after !== void 0 && this.matches(after, local)) return this.schedule(path);
|
|
2132
2722
|
}
|
|
2133
|
-
const target = this.cloud.has(path) ? this.cloud.get(path) : this.synced && !this.scopeStale ? null : void 0;
|
|
2723
|
+
const target = this.cloud.has(path) ? this.cloud.get(path) : this.synced && !this.scopeStale && (this.blobSupport === true || !this.knownBinary(path)) ? null : void 0;
|
|
2134
2724
|
const disk = this.expected.get(path);
|
|
2135
2725
|
const unheld = target === null && this.unheld.has(path);
|
|
2136
2726
|
if (disk === void 0) {
|
|
2137
2727
|
if (local.kind === "absent") {
|
|
2138
|
-
if (
|
|
2728
|
+
if (target !== null && target !== void 0) await this.replace(path, target);
|
|
2139
2729
|
return;
|
|
2140
2730
|
}
|
|
2141
|
-
if (target
|
|
2731
|
+
if (holdsServer(local, target)) return this.adoptContent(path, local);
|
|
2142
2732
|
if (!this.synced || target === void 0) return;
|
|
2143
2733
|
const answer = this.snapshotAnswers.get(path);
|
|
2144
|
-
if (answer?.
|
|
2734
|
+
if (answer?.hash === hashOfContent(local) && !this.pending(path)) {
|
|
2145
2735
|
this.snapshotAnswers.delete(path);
|
|
2146
2736
|
if (!answer.removed) {
|
|
2147
|
-
this.
|
|
2737
|
+
this.adoptContent(path, local);
|
|
2148
2738
|
return this.schedule(path);
|
|
2149
2739
|
}
|
|
2150
|
-
const saved = await this.saveConflict(path, local.text);
|
|
2151
|
-
this.expected.set(path,
|
|
2152
|
-
hash: contentHash(local.text),
|
|
2153
|
-
text: local.text
|
|
2154
|
-
});
|
|
2740
|
+
const saved = await this.saveConflict(path, local.kind === "text" ? local.text : await this.bytesOf(path, local));
|
|
2741
|
+
this.expected.set(path, diskOf(local));
|
|
2155
2742
|
return this.replace(path, null, saved);
|
|
2156
2743
|
}
|
|
2744
|
+
if (local.kind === "binary") return this.queueBlob(path, local, { base: null });
|
|
2157
2745
|
return this.queueLocal(path, local.text, { start: target === null ? null : {
|
|
2158
2746
|
text: local.text,
|
|
2159
2747
|
server: local.text
|
|
2160
2748
|
} });
|
|
2161
2749
|
}
|
|
2162
2750
|
if (!this.matches(disk, local)) {
|
|
2163
|
-
if (local
|
|
2164
|
-
this.
|
|
2751
|
+
if (holdsServer(local, target) && !this.pending(path)) {
|
|
2752
|
+
this.adoptContent(path, local);
|
|
2165
2753
|
return this.release(path);
|
|
2166
2754
|
}
|
|
2167
2755
|
if (local.kind === "text") {
|
|
2168
|
-
if (local.text === "" && disk?.text) {
|
|
2756
|
+
if (local.text === "" && (disk?.text || disk?.blob)) {
|
|
2169
2757
|
await sleep$1(EMPTY_RECHECK_MS);
|
|
2170
2758
|
const again = await this.readLocal(path);
|
|
2171
2759
|
if (again.kind !== "text" || again.text !== "") return this.schedule(path);
|
|
2172
2760
|
}
|
|
2761
|
+
const reported = !unheld && disk !== null ? await this.awaitEdit(path, local.text, disk) : null;
|
|
2762
|
+
if (reported) return this.replayEdit(reported);
|
|
2173
2763
|
return this.queueLocal(path, local.text, unheld ? { start: null } : {});
|
|
2174
2764
|
}
|
|
2765
|
+
if (local.kind === "binary") return this.queueBlob(path, local, unheld ? { base: null } : {});
|
|
2175
2766
|
if (target === null || disk === null) {
|
|
2176
2767
|
this.forget(path);
|
|
2177
2768
|
this.confirm(path, null);
|
|
@@ -2179,7 +2770,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2179
2770
|
}
|
|
2180
2771
|
return this.queueRemove(path, disk);
|
|
2181
2772
|
}
|
|
2182
|
-
if (target === void 0 || (disk === null ? target === null : target !== null && disk.hash ===
|
|
2773
|
+
if (target === void 0 || (disk === null ? target === null : target !== null && disk.hash === serverHash(target))) {
|
|
2183
2774
|
if (disk?.text !== void 0) this.remember(path, {
|
|
2184
2775
|
text: disk.text,
|
|
2185
2776
|
server: disk.text
|
|
@@ -2190,6 +2781,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2190
2781
|
}
|
|
2191
2782
|
if (this.pending(path)) return;
|
|
2192
2783
|
if (unheld && local.kind === "text") return this.queueUnheld(path, local.text);
|
|
2784
|
+
if (unheld && local.kind === "binary") return this.queueBlob(path, local, { unheld: true });
|
|
2193
2785
|
await this.replace(path, target);
|
|
2194
2786
|
}
|
|
2195
2787
|
/**
|
|
@@ -2214,6 +2806,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2214
2806
|
}
|
|
2215
2807
|
matches(disk, local) {
|
|
2216
2808
|
if (disk === null) return local.kind === "absent";
|
|
2809
|
+
if (disk.blob) return local.kind === "binary" && disk.hash === local.hash;
|
|
2217
2810
|
return local.kind === "text" && disk.hash === contentHash(local.text);
|
|
2218
2811
|
}
|
|
2219
2812
|
adopt(path, text) {
|
|
@@ -2230,6 +2823,17 @@ var SyncClient = class extends EventEmitter {
|
|
|
2230
2823
|
text
|
|
2231
2824
|
});
|
|
2232
2825
|
}
|
|
2826
|
+
/** The file at `path` holds `content`, the server's version: adopted, a text as adopt does, a binary file by its sha256. */
|
|
2827
|
+
adoptContent(path, content) {
|
|
2828
|
+
if (content.kind === "text") return this.adopt(path, content.text);
|
|
2829
|
+
const disk = {
|
|
2830
|
+
hash: content.hash,
|
|
2831
|
+
blob: true
|
|
2832
|
+
};
|
|
2833
|
+
this.expected.set(path, disk);
|
|
2834
|
+
this.history.delete(path);
|
|
2835
|
+
this.confirm(path, disk);
|
|
2836
|
+
}
|
|
2233
2837
|
/** A removed file starts over: its earlier versions are gone from the server and are no base for a new one. */
|
|
2234
2838
|
forget(path) {
|
|
2235
2839
|
this.expected.set(path, null);
|
|
@@ -2260,24 +2864,51 @@ var SyncClient = class extends EventEmitter {
|
|
|
2260
2864
|
if (list.length > HISTORY) list.shift();
|
|
2261
2865
|
this.history.set(path, list);
|
|
2262
2866
|
}
|
|
2263
|
-
|
|
2264
|
-
|
|
2867
|
+
/**
|
|
2868
|
+
* Whether the binary file at `path`, which holds `local`, stays as it is here: neither sent nor written over nor
|
|
2869
|
+
* removed. So it does at a path the hub reads as text (readsAsText), larger than MAX_BLOB_BYTES or than the server took,
|
|
2870
|
+
* on a server that keeps no binary files, and while the client does not know yet whether it does (blobSupport), which
|
|
2871
|
+
* the snapshot after the probe reconciles again. With `tell`, the writer is told once for each content.
|
|
2872
|
+
*/
|
|
2873
|
+
binarySkip(path, local, tell) {
|
|
2874
|
+
if (readsAsText(path)) {
|
|
2875
|
+
if (tell) this.tellOnce(`text-only:${path}:${local.hash}`, {
|
|
2876
|
+
kind: "binary-text-file",
|
|
2877
|
+
path,
|
|
2878
|
+
message: `${path} is not UTF-8 text, and the hub reads it as text (config.yaml, .md, .yaml, .yml, ops/*.log, .gitignore), so it is not synced and not overwritten; write it as UTF-8 text, or rename it if it is meant to be a binary file`
|
|
2879
|
+
});
|
|
2880
|
+
return true;
|
|
2881
|
+
}
|
|
2882
|
+
if (local.size > 52428800 || this.oversized.has(`${path}\0${local.hash}`)) {
|
|
2883
|
+
if (tell) this.skipLarge(path, local.hash, true);
|
|
2884
|
+
return true;
|
|
2885
|
+
}
|
|
2886
|
+
if (this.blobSupport === false) {
|
|
2887
|
+
if (tell) this.skipBinary(path, local.hash);
|
|
2888
|
+
return true;
|
|
2889
|
+
}
|
|
2890
|
+
return this.blobSupport === null;
|
|
2891
|
+
}
|
|
2892
|
+
/** Tell the writer of `problem` once for `key`, whatever runs into it again. */
|
|
2893
|
+
tellOnce(key, problem) {
|
|
2265
2894
|
if (this.skipped.has(key)) return;
|
|
2266
2895
|
this.skipped.add(key);
|
|
2267
|
-
this.problem(
|
|
2896
|
+
this.problem(problem);
|
|
2897
|
+
}
|
|
2898
|
+
skipBinary(path, hash) {
|
|
2899
|
+
this.tellOnce(`binary:${path}:${hash}`, {
|
|
2268
2900
|
kind: "skipped-binary",
|
|
2269
2901
|
path,
|
|
2270
|
-
message: `${path} is not UTF-8 text,
|
|
2902
|
+
message: `${path} is not UTF-8 text, and the sync server keeps no binary files, as one from before them: it stays as it is here, not overwritten, and is sent once the sync server keeps binary files; other environments do not see it until then`
|
|
2271
2903
|
});
|
|
2272
2904
|
}
|
|
2273
|
-
|
|
2274
|
-
|
|
2275
|
-
|
|
2276
|
-
this.
|
|
2277
|
-
this.problem({
|
|
2905
|
+
/** A text of `path` larger than MAX_FILE_BYTES, or a binary file larger than the server keeps, of `hash`. */
|
|
2906
|
+
skipLarge(path, hash, binary) {
|
|
2907
|
+
const limit = binary ? `the ${MAX_BLOB_BYTES} bytes the sync server keeps of a binary file` : `the ${MAX_FILE_BYTES} bytes the sync server keeps of a file`;
|
|
2908
|
+
this.tellOnce(`large:${path}:${hash}`, {
|
|
2278
2909
|
kind: "skipped-large",
|
|
2279
2910
|
path,
|
|
2280
|
-
message: `${path} is larger than
|
|
2911
|
+
message: `${path} is larger than ${limit}, so it is not synced and not overwritten; other environments do not see it. ${binary ? "Make it smaller" : "Split it"}, or add it to .mainflow/.gitignore`
|
|
2281
2912
|
});
|
|
2282
2913
|
}
|
|
2283
2914
|
/**
|
|
@@ -2324,7 +2955,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2324
2955
|
*/
|
|
2325
2956
|
mayStay(path, held) {
|
|
2326
2957
|
if (!this.staysRemote(path)) return false;
|
|
2327
|
-
return !(held?.has(path) ?? false) &&
|
|
2958
|
+
return !(held?.has(path) ?? false) && (this.cloud.get(path) ?? null) === null && this.expected.get(path) === void 0 && !this.pending(path);
|
|
2328
2959
|
}
|
|
2329
2960
|
/** Note `path` as only on the server, with the hash of the server's text as of `seq`, unless a newer version came. */
|
|
2330
2961
|
keepRemote(path, hash, seq) {
|
|
@@ -2386,22 +3017,34 @@ var SyncClient = class extends EventEmitter {
|
|
|
2386
3017
|
* file the client knows by its hash, so the server sends the targets and only what changed of the rest; the artifacts
|
|
2387
3018
|
* of closed mainlines other than the targets' stay listed. What comes is taken in as a broadcast of its batch's seq
|
|
2388
3019
|
* is: a target's text is written here, and a target named nowhere is not on the server, since the range covers it
|
|
2389
|
-
* and `have` does not name it. Returns, besides, the files of other paths listed that may not stay on the server.
|
|
3020
|
+
* and `have` does not name it. Returns, besides, the files of other paths listed that may not stay on the server. A
|
|
3021
|
+
* binary file only on the server is not asked for while the client does not know that the server keeps binary files
|
|
3022
|
+
* (blobSupport): such a snapshot would name it nowhere, as if it were gone. Nor is a target such a snapshot names
|
|
3023
|
+
* nowhere taken for gone, since the server may hold a binary file there now. Both stay only on the server,
|
|
3024
|
+
* `unavailable`.
|
|
2390
3025
|
*/
|
|
2391
3026
|
async fetchNow(targets) {
|
|
2392
|
-
const
|
|
3027
|
+
const asked = [...new Set(targets)].filter((path) => this.syncs(path));
|
|
3028
|
+
const unavailable = this.blobSupport === true ? [] : asked.filter((path) => BLOB_HASH.test(this.remote.get(path) ?? ""));
|
|
3029
|
+
const sorted = asked.filter((path) => !unavailable.includes(path)).sort(comparePaths);
|
|
2393
3030
|
const again = /* @__PURE__ */ new Set();
|
|
2394
3031
|
if (sorted.length === 0 || this.closing) return {
|
|
2395
3032
|
fetched: [],
|
|
2396
3033
|
gone: [],
|
|
2397
|
-
again
|
|
3034
|
+
again,
|
|
3035
|
+
unavailable
|
|
2398
3036
|
};
|
|
2399
3037
|
const want = new Set(sorted);
|
|
2400
3038
|
for (const path of sorted) this.awaiting.add(path);
|
|
2401
3039
|
let done = false;
|
|
2402
3040
|
try {
|
|
2403
3041
|
const have = /* @__PURE__ */ new Map();
|
|
2404
|
-
|
|
3042
|
+
const support = this.blobSupport === true;
|
|
3043
|
+
for (const [path, file] of this.cloud) {
|
|
3044
|
+
if (want.has(path) || !this.syncs(path)) continue;
|
|
3045
|
+
if (typeof file === "string") have.set(path, this.hashOf(path, file));
|
|
3046
|
+
else if (isBlob(file) && support) have.set(path, file.hash);
|
|
3047
|
+
}
|
|
2405
3048
|
for (const [path, hash] of this.remote) if (!want.has(path)) have.set(path, hash);
|
|
2406
3049
|
const listOnly = this.listOnly(false)?.filter((prefix) => !sorted.some((path) => path.startsWith(prefix)));
|
|
2407
3050
|
const first = sorted[0];
|
|
@@ -2409,12 +3052,14 @@ var SyncClient = class extends EventEmitter {
|
|
|
2409
3052
|
const parts = await this.api.snapshot(have, {
|
|
2410
3053
|
listOnly,
|
|
2411
3054
|
after,
|
|
2412
|
-
through: sorted.at(-1)
|
|
3055
|
+
through: sorted.at(-1),
|
|
3056
|
+
blobs: support
|
|
2413
3057
|
});
|
|
2414
3058
|
if (this.closing) return {
|
|
2415
3059
|
fetched: [],
|
|
2416
3060
|
gone: [],
|
|
2417
|
-
again
|
|
3061
|
+
again,
|
|
3062
|
+
unavailable
|
|
2418
3063
|
};
|
|
2419
3064
|
const mentioned = /* @__PURE__ */ new Set();
|
|
2420
3065
|
const fetched = [];
|
|
@@ -2434,13 +3079,17 @@ var SyncClient = class extends EventEmitter {
|
|
|
2434
3079
|
if (want.has(event.path)) {
|
|
2435
3080
|
this.remote.delete(event.path);
|
|
2436
3081
|
if ((this.fileSeq.get(event.path) ?? -1) <= part.seq) this.fileSeq.delete(event.path);
|
|
2437
|
-
if (event.kind === "file") fetched.push(event.path);
|
|
3082
|
+
if (event.kind === "file" || event.kind === "blob") fetched.push(event.path);
|
|
2438
3083
|
}
|
|
2439
3084
|
this.receive(event, part.seq);
|
|
2440
3085
|
}
|
|
2441
3086
|
const gone = [];
|
|
2442
3087
|
for (const path of sorted) {
|
|
2443
3088
|
if (mentioned.has(path)) continue;
|
|
3089
|
+
if (!support) {
|
|
3090
|
+
unavailable.push(path);
|
|
3091
|
+
continue;
|
|
3092
|
+
}
|
|
2444
3093
|
if (this.remote.delete(path)) gone.push(path);
|
|
2445
3094
|
else this.receive({
|
|
2446
3095
|
kind: "removed",
|
|
@@ -2453,7 +3102,8 @@ var SyncClient = class extends EventEmitter {
|
|
|
2453
3102
|
return {
|
|
2454
3103
|
fetched,
|
|
2455
3104
|
gone,
|
|
2456
|
-
again
|
|
3105
|
+
again,
|
|
3106
|
+
unavailable
|
|
2457
3107
|
};
|
|
2458
3108
|
} finally {
|
|
2459
3109
|
for (const path of sorted) {
|
|
@@ -2474,28 +3124,27 @@ var SyncClient = class extends EventEmitter {
|
|
|
2474
3124
|
if (targets.length === 0) return {
|
|
2475
3125
|
fetched: [],
|
|
2476
3126
|
gone: [],
|
|
2477
|
-
none
|
|
3127
|
+
none,
|
|
3128
|
+
unavailable: []
|
|
2478
3129
|
};
|
|
2479
|
-
const { fetched, gone } = await this.fetchNow(targets);
|
|
3130
|
+
const { fetched, gone, unavailable } = await this.fetchNow(targets);
|
|
2480
3131
|
return {
|
|
2481
3132
|
fetched,
|
|
2482
3133
|
gone,
|
|
2483
|
-
none
|
|
3134
|
+
none,
|
|
3135
|
+
unavailable
|
|
2484
3136
|
};
|
|
2485
3137
|
});
|
|
2486
3138
|
}
|
|
2487
3139
|
/**
|
|
2488
3140
|
* Download the files only on the server that `specs` name (fetchPaths), and resolve once each is written here: for
|
|
2489
3141
|
* `mainflow sync fetch` and the PreToolUse hook. Offline, or when the server cannot be reached, nothing is
|
|
2490
|
-
* downloaded and `error` says why.
|
|
3142
|
+
* downloaded and `error` says why. So it says why for each file that was not downloaded: a binary file whose download
|
|
3143
|
+
* failed, or one asked for while the client does not know that the server keeps binary files.
|
|
2491
3144
|
*/
|
|
2492
3145
|
async fetch(specs) {
|
|
2493
3146
|
const covers = (spec) => [...this.remote.keys(), ...this.awaiting].some((path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`));
|
|
2494
|
-
if (!specs.some(covers)) return
|
|
2495
|
-
fetched: [],
|
|
2496
|
-
gone: [],
|
|
2497
|
-
none: specs
|
|
2498
|
-
};
|
|
3147
|
+
if (!specs.some(covers)) return this.retryStuck(specs);
|
|
2499
3148
|
if (!this.online) return {
|
|
2500
3149
|
fetched: [],
|
|
2501
3150
|
gone: [],
|
|
@@ -2514,10 +3163,44 @@ var SyncClient = class extends EventEmitter {
|
|
|
2514
3163
|
};
|
|
2515
3164
|
}
|
|
2516
3165
|
await Promise.allSettled(result.fetched.map((path) => this.chains.get(path) ?? Promise.resolve()));
|
|
3166
|
+
const fetched = result.fetched.filter((path) => existsSync(join(this.paths.data, path)));
|
|
3167
|
+
const failed = [...result.fetched.flatMap((path) => fetched.includes(path) || !this.downloadErrors.has(path) ? [] : [`${path}: ${this.downloadErrors.get(path)}`]), ...result.unavailable.map((path) => `${path}: it may be a binary file, and ${this.blobSupport === false ? "the sync server keeps no binary files now" : "it is not known yet whether the sync server keeps binary files"}; it stays on the server, try again once mainflow sync is online`)];
|
|
2517
3168
|
return {
|
|
2518
|
-
fetched
|
|
3169
|
+
fetched,
|
|
2519
3170
|
gone: result.gone,
|
|
2520
|
-
none: result.none
|
|
3171
|
+
none: result.none,
|
|
3172
|
+
...failed.length > 0 ? { error: `could not download ${failed.join("; ")}` } : {}
|
|
3173
|
+
};
|
|
3174
|
+
}
|
|
3175
|
+
/**
|
|
3176
|
+
* `fetch` for specs that name no file only on the server: a download that failed is no longer one, and is tried again
|
|
3177
|
+
* on its own. Try it now, and say why for each that still is not here.
|
|
3178
|
+
*/
|
|
3179
|
+
async retryStuck(specs) {
|
|
3180
|
+
const covered = (spec, path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`);
|
|
3181
|
+
const missing = (path) => !existsSync(join(this.paths.data, path));
|
|
3182
|
+
const stuck = [...this.stuckDownloads().keys()].filter((path) => specs.some((spec) => covered(spec, path)));
|
|
3183
|
+
if (stuck.length === 0) return {
|
|
3184
|
+
fetched: [],
|
|
3185
|
+
gone: [],
|
|
3186
|
+
none: specs
|
|
3187
|
+
};
|
|
3188
|
+
const none = specs.filter((spec) => !stuck.some((path) => covered(spec, path)));
|
|
3189
|
+
if (!this.online) return {
|
|
3190
|
+
fetched: [],
|
|
3191
|
+
gone: [],
|
|
3192
|
+
none,
|
|
3193
|
+
error: "mainflow sync is offline: the sync server cannot be reached now"
|
|
3194
|
+
};
|
|
3195
|
+
for (const path of stuck) this.schedule(path);
|
|
3196
|
+
await Promise.allSettled(stuck.map((path) => this.chains.get(path) ?? Promise.resolve()));
|
|
3197
|
+
const failed = stuck.filter(missing).map((path) => `${path}: ${this.downloadErrors.get(path) ?? "it did not come"}`);
|
|
3198
|
+
const again = stuck.filter(missing).every((path) => this.stalledDownloads.has(path)) ? "at the next connection" : "later on its own";
|
|
3199
|
+
return {
|
|
3200
|
+
fetched: stuck.filter((path) => !missing(path)),
|
|
3201
|
+
gone: [],
|
|
3202
|
+
none,
|
|
3203
|
+
...failed.length > 0 ? { error: `could not download ${failed.join("; ")}; mainflow sync tries again ${again}` } : {}
|
|
2521
3204
|
};
|
|
2522
3205
|
}
|
|
2523
3206
|
/** Mainline `dir`'s file came: when it opens the mainline, its artifacts only on the server come here. */
|
|
@@ -2542,6 +3225,28 @@ var SyncClient = class extends EventEmitter {
|
|
|
2542
3225
|
restoreRemote() {
|
|
2543
3226
|
for (const [path, hash] of Object.entries(readRemote(this.root))) if (this.syncs(path) && artifactOf(path) !== null && this.expected.get(path) === void 0) this.remote.set(path, hash);
|
|
2544
3227
|
}
|
|
3228
|
+
/**
|
|
3229
|
+
* What remote.json lists: the files only on the server, and the artifacts whose download failed and waits for a retry.
|
|
3230
|
+
* The interface lists both as files not here yet, so a failed download stays in sight; fetch tries it again and says
|
|
3231
|
+
* why it failed. A file leaves the list when its download succeeds.
|
|
3232
|
+
*/
|
|
3233
|
+
listedRemote() {
|
|
3234
|
+
const listed = new Map(this.remote);
|
|
3235
|
+
for (const [path, hash] of this.stuckDownloads()) if (!listed.has(path)) listed.set(path, hash);
|
|
3236
|
+
return listed;
|
|
3237
|
+
}
|
|
3238
|
+
/**
|
|
3239
|
+
* The artifacts whose download failed and that are still not here, by the sha256 the server holds: the server still
|
|
3240
|
+
* has that binary version, and the client keeps the error for it. A file the server removed or changed since is not one.
|
|
3241
|
+
*/
|
|
3242
|
+
stuckDownloads() {
|
|
3243
|
+
const stuck = /* @__PURE__ */ new Map();
|
|
3244
|
+
for (const path of this.downloadErrors.keys()) {
|
|
3245
|
+
const file = this.cloud.get(path);
|
|
3246
|
+
if (isBlob(file) && artifactOf(path) !== null && !existsSync(join(this.paths.data, path))) stuck.set(path, file.hash);
|
|
3247
|
+
}
|
|
3248
|
+
return stuck;
|
|
3249
|
+
}
|
|
2545
3250
|
saveRemote() {
|
|
2546
3251
|
if (this.remoteTimer) return;
|
|
2547
3252
|
this.remoteTimer = setTimeout(() => this.saveRemoteNow(), 200);
|
|
@@ -2550,9 +3255,10 @@ var SyncClient = class extends EventEmitter {
|
|
|
2550
3255
|
if (this.abandoned && !final) return;
|
|
2551
3256
|
clearTimeout(this.remoteTimer);
|
|
2552
3257
|
this.remoteTimer = void 0;
|
|
2553
|
-
|
|
3258
|
+
const listed = this.listedRemote();
|
|
3259
|
+
if (listed.size === 0 && !existsSync(this.paths.remote)) return;
|
|
2554
3260
|
try {
|
|
2555
|
-
writeRemote(this.root, Object.fromEntries([...
|
|
3261
|
+
writeRemote(this.root, Object.fromEntries([...listed].sort(([a], [b]) => comparePaths(a, b))));
|
|
2556
3262
|
} catch (error) {
|
|
2557
3263
|
this.log(`could not save local/sync/remote.json: ${error.message}`);
|
|
2558
3264
|
}
|
|
@@ -2566,15 +3272,18 @@ var SyncClient = class extends EventEmitter {
|
|
|
2566
3272
|
* not lost; it is read again shortly after, for a program that still held the old file open. A new file is linked
|
|
2567
3273
|
* into place, which fails rather than overwrite a file created meanwhile; a removed file is first moved aside and
|
|
2568
3274
|
* checked. `unsynced` names where the text of a never synced file the server no longer has was saved: the writer is
|
|
2569
|
-
* told of it once the file is removed, and it is deleted again when a write in the outbox keeps the file instead.
|
|
3275
|
+
* told of it once the file is removed, and it is deleted again when a write in the outbox keeps the file instead. A
|
|
3276
|
+
* binary file is downloaded into the prepared path first, checked against its sha256; one that cannot be downloaded
|
|
3277
|
+
* now is left as it is (download).
|
|
2570
3278
|
*/
|
|
2571
3279
|
async replace(path, target, unsynced) {
|
|
2572
3280
|
const file = join(this.paths.data, path);
|
|
2573
3281
|
const temp = this.tempName();
|
|
2574
|
-
this.trace(`replacing ${path} with ${short(target)}`);
|
|
3282
|
+
this.trace(`replacing ${path} with ${isBlob(target) ? `blob ${target.hash.slice(0, 7)}` : short(target)}`);
|
|
2575
3283
|
if (target !== null) {
|
|
2576
3284
|
await mkdir(dirname(file), { recursive: true });
|
|
2577
|
-
await writeFile(temp, target);
|
|
3285
|
+
if (typeof target === "string") await writeFile(temp, target);
|
|
3286
|
+
else if (!await this.download(path, target, temp)) return;
|
|
2578
3287
|
}
|
|
2579
3288
|
let step;
|
|
2580
3289
|
try {
|
|
@@ -2596,24 +3305,25 @@ var SyncClient = class extends EventEmitter {
|
|
|
2596
3305
|
saved: unsynced,
|
|
2597
3306
|
message: `${path} was never synced here and the sync server no longer has it, so it was removed here too; its text was saved to ${unsynced}`
|
|
2598
3307
|
});
|
|
2599
|
-
if (step.moved !== null && step.disk &&
|
|
2600
|
-
const saved = await this.saveConflict(path, step.moved);
|
|
3308
|
+
if (step.moved !== null && step.disk && !this.matches(step.disk, step.moved)) {
|
|
3309
|
+
const saved = await this.saveConflict(path, step.moved.kind === "text" ? step.moved.text : step.moved.bytes);
|
|
2601
3310
|
this.problem({
|
|
2602
3311
|
kind: "conflict",
|
|
2603
3312
|
path,
|
|
2604
3313
|
saved,
|
|
2605
|
-
message: `${path} was removed by another environment while it was being written here; the text written here was saved to ${saved}`
|
|
3314
|
+
message: `${path} was removed by another environment while it was being written here; the ${step.moved.kind === "text" ? "text" : "binary file"} written here was saved to ${saved}`
|
|
2606
3315
|
});
|
|
2607
3316
|
}
|
|
2608
3317
|
return;
|
|
2609
3318
|
}
|
|
2610
3319
|
const { displaced, swapped, from } = step;
|
|
2611
|
-
this.trace(`wrote ${path} ${short(target)}${swapped ? " (exchanged)" : ""}; displaced ${short(displaced)}`);
|
|
2612
|
-
const changed = displaced !== null &&
|
|
2613
|
-
const later = swapped && changed && displaced === "";
|
|
3320
|
+
this.trace(`wrote ${path} ${isBlob(target) ? `blob ${target.hash.slice(0, 7)}` : short(target)}${swapped ? " (exchanged)" : ""}; displaced ${displaced?.kind === "binary" ? `blob ${displaced.hash.slice(0, 7)}` : short(displaced?.text ?? null)}`);
|
|
3321
|
+
const changed = displaced !== null && !this.matches(from.over, displaced);
|
|
3322
|
+
const later = swapped && changed && displaced.kind === "text" && displaced.text === "";
|
|
2614
3323
|
if (changed && !later) {
|
|
2615
|
-
this.log(`${path}: a local write landed while the server's version was written; sending it`);
|
|
2616
|
-
await this.queueLocal(path, displaced, { displaced: from });
|
|
3324
|
+
this.log(`${path}: a local write landed while the server's version was written; ${displaced.kind === "text" ? "sending it" : "saving it"}`);
|
|
3325
|
+
if (displaced.kind === "text") await this.queueLocal(path, displaced.text, { displaced: from });
|
|
3326
|
+
else await this.displacedBinary(path, displaced);
|
|
2617
3327
|
}
|
|
2618
3328
|
if (swapped) {
|
|
2619
3329
|
const recheck = this.recheck(path, temp, later ? null : displaced, from);
|
|
@@ -2674,11 +3384,16 @@ var SyncClient = class extends EventEmitter {
|
|
|
2674
3384
|
const disk = this.expected.get(path);
|
|
2675
3385
|
const read = (at) => {
|
|
2676
3386
|
try {
|
|
2677
|
-
return readFileSync(at
|
|
3387
|
+
return contentOf(readFileSync(at));
|
|
2678
3388
|
} catch {
|
|
2679
3389
|
return null;
|
|
2680
3390
|
}
|
|
2681
3391
|
};
|
|
3392
|
+
/** Whether the file at `at` holds the server's version. */
|
|
3393
|
+
const holdsTarget = (at) => {
|
|
3394
|
+
const content = read(at);
|
|
3395
|
+
return content !== null && target !== null && holdsServer(content, target);
|
|
3396
|
+
};
|
|
2682
3397
|
if (target === null) {
|
|
2683
3398
|
try {
|
|
2684
3399
|
renameSync(file, temp);
|
|
@@ -2728,7 +3443,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2728
3443
|
if (answer === "ok") this.swapFailures = 0;
|
|
2729
3444
|
else {
|
|
2730
3445
|
this.swapFailed(path, answer === "unsupported" ? "the file system refused it" : answer.message);
|
|
2731
|
-
if (answer !== "unsupported" &&
|
|
3446
|
+
if (answer !== "unsupported" && holdsTarget(file) && !holdsTarget(temp)) answer = "ok";
|
|
2732
3447
|
}
|
|
2733
3448
|
if (answer === "ok") {
|
|
2734
3449
|
swapped = true;
|
|
@@ -2761,34 +3476,119 @@ var SyncClient = class extends EventEmitter {
|
|
|
2761
3476
|
*/
|
|
2762
3477
|
async recheck(path, temp, taken, from) {
|
|
2763
3478
|
await sleep$1(taken === null ? EMPTY_RECHECK_MS : LATE_WRITE_MS);
|
|
2764
|
-
const later = await readFile(temp
|
|
3479
|
+
const later = await readFile(temp).then(contentOf, () => null);
|
|
2765
3480
|
await rm(temp, { force: true });
|
|
2766
|
-
if (later !== null &&
|
|
2767
|
-
this.log(`${path}: a late write into the replaced file; sending it`);
|
|
2768
|
-
|
|
3481
|
+
if (later !== null && (taken === null || !sameLocal(later, taken)) && !this.matches(from.over, later)) {
|
|
3482
|
+
this.log(`${path}: a late write into the replaced file; ${later.kind === "text" ? "sending it" : "saving it"}`);
|
|
3483
|
+
if (later.kind === "binary") return this.withChain(path, () => this.displacedBinary(path, later));
|
|
3484
|
+
const held = taken?.kind !== "text" ? from : {
|
|
2769
3485
|
...from,
|
|
2770
|
-
among: /* @__PURE__ */ new Set([...from.among, taken])
|
|
3486
|
+
among: /* @__PURE__ */ new Set([...from.among, taken.text])
|
|
2771
3487
|
};
|
|
2772
|
-
await this.withChain(path, () => this.queueLocal(path, later, { displaced: held }));
|
|
3488
|
+
await this.withChain(path, () => this.queueLocal(path, later.text, { displaced: held }));
|
|
3489
|
+
}
|
|
3490
|
+
}
|
|
3491
|
+
/**
|
|
3492
|
+
* A binary file a program wrote while the server's version took its place (layer 3), now off disk: it cannot be merged
|
|
3493
|
+
* with the server's version, so it is saved and the writer told, as a refused binary file is.
|
|
3494
|
+
*/
|
|
3495
|
+
async displacedBinary(path, content) {
|
|
3496
|
+
const saved = await this.saveConflict(path, content.bytes);
|
|
3497
|
+
this.problem({
|
|
3498
|
+
kind: "conflict",
|
|
3499
|
+
path,
|
|
3500
|
+
saved,
|
|
3501
|
+
message: `${path}: a binary file was written here while the server's version took its place; the file written here was saved to ${saved} and the file now has the server's version`
|
|
3502
|
+
});
|
|
3503
|
+
}
|
|
3504
|
+
/**
|
|
3505
|
+
* Download the content of `blob`, the server's version of `path`, into `temp`: true once it is there and of its sha256.
|
|
3506
|
+
* Offline, or when the download fails, nothing is written and false comes back, and downloadErrors keeps why for
|
|
3507
|
+
* fetch. While online it is tried again after DOWNLOAD_BACKOFF_MS, doubling, with one timer for each path. A server
|
|
3508
|
+
* that could not be reached or failed is tried for as long as the connection holds, up to BLOB_BACKOFF_MAX_MS between
|
|
3509
|
+
* tries. Content of another sha256, or a file this machine could not write, as on a full disk, is tried DOWNLOAD_TRIES
|
|
3510
|
+
* times in a row, when the writer is told and the file waits for the next connection's snapshot. So does a file whose
|
|
3511
|
+
* download the connection dropped under: the snapshot after reconnecting names its version again, which receive takes
|
|
3512
|
+
* for one it already holds, so the file is reconciled again after that snapshot. Content the server no longer holds,
|
|
3513
|
+
* as when the file changed again meanwhile and the old content was cleaned up, reloads the snapshot, once per content
|
|
3514
|
+
* on a connection.
|
|
3515
|
+
*/
|
|
3516
|
+
async download(path, blob, temp) {
|
|
3517
|
+
clearTimeout(this.downloadRetries.get(path));
|
|
3518
|
+
this.downloadRetries.delete(path);
|
|
3519
|
+
try {
|
|
3520
|
+
if (await this.api.downloadBlob(blob.hash, temp)) {
|
|
3521
|
+
this.downloadFailures.delete(blob.hash);
|
|
3522
|
+
if (this.downloadErrors.delete(path)) this.saveRemote();
|
|
3523
|
+
return true;
|
|
3524
|
+
}
|
|
3525
|
+
await rm(temp, { force: true });
|
|
3526
|
+
this.downloadErrors.set(path, `the sync server no longer holds its content ${blob.hash.slice(0, 7)}`);
|
|
3527
|
+
this.saveRemote();
|
|
3528
|
+
this.log(`${path}: the sync server no longer holds its content ${blob.hash.slice(0, 7)}; reloading the snapshot`);
|
|
3529
|
+
if (!this.missingDownloads.has(blob.hash) && this.online) {
|
|
3530
|
+
this.missingDownloads.add(blob.hash);
|
|
3531
|
+
this.track(this.resync().catch((error) => this.log(`could not reload the snapshot: ${error.message}`)));
|
|
3532
|
+
}
|
|
3533
|
+
return false;
|
|
3534
|
+
} catch (error) {
|
|
3535
|
+
await rm(temp, { force: true });
|
|
3536
|
+
const reason = error.message;
|
|
3537
|
+
this.downloadErrors.set(path, reason);
|
|
3538
|
+
this.saveRemote();
|
|
3539
|
+
this.log(`${path}: could not download its content ${blob.hash.slice(0, 7)}: ${reason}`);
|
|
3540
|
+
if (this.closing) return false;
|
|
3541
|
+
if (!this.online) {
|
|
3542
|
+
this.stalledDownloads.add(path);
|
|
3543
|
+
return false;
|
|
3544
|
+
}
|
|
3545
|
+
const failures = (this.downloadFailures.get(blob.hash) ?? 0) + 1;
|
|
3546
|
+
this.downloadFailures.set(blob.hash, failures);
|
|
3547
|
+
if (!!(error instanceof BlobMismatch || error instanceof LocalWriteError) && failures >= DOWNLOAD_TRIES) {
|
|
3548
|
+
this.stalledDownloads.add(path);
|
|
3549
|
+
this.tellOnce(`download:${path}:${blob.hash}`, {
|
|
3550
|
+
kind: "download-failed",
|
|
3551
|
+
path,
|
|
3552
|
+
message: `${path}: the sync server's version could not be downloaded ${failures} times in a row (${reason}); the file stays as it is here, and the next connection tries again`
|
|
3553
|
+
});
|
|
3554
|
+
return false;
|
|
3555
|
+
}
|
|
3556
|
+
const timer = setTimeout(() => {
|
|
3557
|
+
this.downloadRetries.delete(path);
|
|
3558
|
+
this.schedule(path);
|
|
3559
|
+
}, Math.min((this.opts.downloadBackoffMs ?? DOWNLOAD_BACKOFF_MS) * 2 ** Math.min(failures - 1, 30), BLOB_BACKOFF_MAX_MS));
|
|
3560
|
+
this.downloadRetries.set(path, timer);
|
|
3561
|
+
return false;
|
|
2773
3562
|
}
|
|
2774
3563
|
}
|
|
2775
3564
|
withChain(path, fn) {
|
|
2776
3565
|
const next = (this.chains.get(path) ?? Promise.resolve()).then(fn).catch((error) => this.log(`${path}: ${String(error)}`));
|
|
2777
3566
|
this.chains.set(path, next);
|
|
2778
|
-
return this.track(next);
|
|
2779
|
-
}
|
|
2780
|
-
|
|
3567
|
+
return this.track(next, path);
|
|
3568
|
+
}
|
|
3569
|
+
/** The server's version `target` was written at `path`: what the client leaves there, and confirmed. */
|
|
3570
|
+
wrote(path, target) {
|
|
3571
|
+
if (isBlob(target)) {
|
|
3572
|
+
const disk = {
|
|
3573
|
+
hash: target.hash,
|
|
3574
|
+
blob: true
|
|
3575
|
+
};
|
|
3576
|
+
this.expected.set(path, disk);
|
|
3577
|
+
this.history.delete(path);
|
|
3578
|
+
this.confirm(path, disk);
|
|
3579
|
+
return;
|
|
3580
|
+
}
|
|
2781
3581
|
this.expected.set(path, {
|
|
2782
|
-
hash: contentHash(
|
|
2783
|
-
text
|
|
3582
|
+
hash: contentHash(target),
|
|
3583
|
+
text: target
|
|
2784
3584
|
});
|
|
2785
3585
|
this.remember(path, {
|
|
2786
|
-
text,
|
|
2787
|
-
server:
|
|
3586
|
+
text: target,
|
|
3587
|
+
server: target
|
|
2788
3588
|
});
|
|
2789
3589
|
this.confirm(path, {
|
|
2790
|
-
hash: contentHash(
|
|
2791
|
-
text
|
|
3590
|
+
hash: contentHash(target),
|
|
3591
|
+
text: target
|
|
2792
3592
|
});
|
|
2793
3593
|
}
|
|
2794
3594
|
/**
|
|
@@ -2831,7 +3631,8 @@ var SyncClient = class extends EventEmitter {
|
|
|
2831
3631
|
* disk, is not on disk: it was written over the version the client had left there, and on a version the old file held
|
|
2832
3632
|
* (layer 3); with none of those left, it is based on the text the client had left there, so layer 2 still tells the
|
|
2833
3633
|
* writer's lines from the ones the server changed meanwhile. `start` overrides layer 1: the version the writer started
|
|
2834
|
-
* from, or null for a new file.
|
|
3634
|
+
* from, or null for a new file. A text written over a binary file the client left there is based on that file's sha256,
|
|
3635
|
+
* with no base text: the server replaces the binary file with it when it still holds that one (PatchRequest).
|
|
2835
3636
|
*/
|
|
2836
3637
|
async queueLocal(path, text, options = {}) {
|
|
2837
3638
|
const { displaced } = options;
|
|
@@ -2843,7 +3644,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2843
3644
|
hash: contentHash(text),
|
|
2844
3645
|
text
|
|
2845
3646
|
});
|
|
2846
|
-
let start = options.start !== void 0 ? options.start ?? void 0 : before === null ? void 0 : this.startOf(path, text, before, {
|
|
3647
|
+
let start = options.start !== void 0 ? options.start ?? void 0 : before === null || before?.blob ? void 0 : this.startOf(path, text, before, {
|
|
2847
3648
|
entries,
|
|
2848
3649
|
among: displaced?.among
|
|
2849
3650
|
});
|
|
@@ -2879,6 +3680,74 @@ var SyncClient = class extends EventEmitter {
|
|
|
2879
3680
|
this.trace(`new ${path} changed before it was queued; reconciling it again`);
|
|
2880
3681
|
this.schedule(path);
|
|
2881
3682
|
}
|
|
3683
|
+
/**
|
|
3684
|
+
* Queue the binary file at `path`, which held `local` when it was read, as a write of its own after every write queued
|
|
3685
|
+
* already. In the hold of the write lock that queues it, the file is read again, and its content goes into blobs/
|
|
3686
|
+
* (stageHeld), so the entry uploads what the file held as it was queued (I2). The base is the hash of what the client
|
|
3687
|
+
* expected there before; `base: null` sends it as new. A file that changed since it was read, or that holds what the
|
|
3688
|
+
* client expects already, is reconciled again instead. With `unheld` the client expects that content there already, a
|
|
3689
|
+
* file the server never held (unheld), as after the client uploaded the hub; it is sent as new while the server still
|
|
3690
|
+
* has no file there.
|
|
3691
|
+
*/
|
|
3692
|
+
async queueBlob(path, local, options = {}) {
|
|
3693
|
+
if (await this.enqueue(() => {
|
|
3694
|
+
const before = this.expected.get(path);
|
|
3695
|
+
const expected = before?.blob === true && before.hash === local.hash;
|
|
3696
|
+
if (options.unheld ? !expected || !this.unheld.has(path) || (this.cloud.get(path) ?? null) !== null : expected) return null;
|
|
3697
|
+
if (!this.stageHeld(path, local.hash)) return null;
|
|
3698
|
+
this.expected.set(path, {
|
|
3699
|
+
hash: local.hash,
|
|
3700
|
+
blob: true
|
|
3701
|
+
});
|
|
3702
|
+
const base = options.base === null || options.unheld ? null : before?.hash ?? null;
|
|
3703
|
+
this.trace(`local ${path} blob ${local.hash.slice(0, 7)} base ${base?.slice(0, 7) ?? "null"}`);
|
|
3704
|
+
return {
|
|
3705
|
+
kind: "blob",
|
|
3706
|
+
path,
|
|
3707
|
+
hash: local.hash,
|
|
3708
|
+
size: local.size,
|
|
3709
|
+
base
|
|
3710
|
+
};
|
|
3711
|
+
}) !== null) return;
|
|
3712
|
+
this.trace(`binary ${path} changed before it was queued; reconciling it again`);
|
|
3713
|
+
this.schedule(path);
|
|
3714
|
+
}
|
|
3715
|
+
/**
|
|
3716
|
+
* Keep the content of sha256 `hash` in blobs/ when the file at `path` holds it now: false when it holds anything else.
|
|
3717
|
+
* Called under the write lock, so the content leaves blobs/ only once no entry of the outbox names it (pruneStaged).
|
|
3718
|
+
* The file is copied first, a clone of it where the file system makes one (APFS), and the copy is hashed in chunks, so
|
|
3719
|
+
* a file of up to MAX_BLOB_BYTES is never held in memory whole, and what blobs/ keeps is what was hashed. With the
|
|
3720
|
+
* content kept already, the file is hashed in chunks alone.
|
|
3721
|
+
*/
|
|
3722
|
+
stageHeld(path, hash) {
|
|
3723
|
+
const file = join(this.paths.data, path);
|
|
3724
|
+
const staged = join(this.paths.blobs, hash);
|
|
3725
|
+
if (existsSync(staged)) return sha256File(file) === hash;
|
|
3726
|
+
mkdirSync(this.paths.blobs, { recursive: true });
|
|
3727
|
+
const temp = `${staged}.tmp`;
|
|
3728
|
+
try {
|
|
3729
|
+
copyFileSync(file, temp);
|
|
3730
|
+
} catch {
|
|
3731
|
+
return false;
|
|
3732
|
+
}
|
|
3733
|
+
if (sha256File(temp) !== hash) {
|
|
3734
|
+
rmSync(temp, { force: true });
|
|
3735
|
+
return false;
|
|
3736
|
+
}
|
|
3737
|
+
renameSync(temp, staged);
|
|
3738
|
+
return true;
|
|
3739
|
+
}
|
|
3740
|
+
/** Remove from blobs/ the content no binary file's write in `entries`, the outbox, names. Called under the write lock. */
|
|
3741
|
+
pruneStaged(entries) {
|
|
3742
|
+
let names;
|
|
3743
|
+
try {
|
|
3744
|
+
names = readdirSync(this.paths.blobs);
|
|
3745
|
+
} catch {
|
|
3746
|
+
return;
|
|
3747
|
+
}
|
|
3748
|
+
const named = new Set(entries.flatMap((entry) => entry.kind === "blob" ? [entry.hash] : []));
|
|
3749
|
+
for (const name of names) if (!named.has(name)) rmSync(join(this.paths.blobs, name), { force: true });
|
|
3750
|
+
}
|
|
2882
3751
|
/** What the response to queued edit `entry` means for the writer (layer 2), then wait for the broadcast it caused. */
|
|
2883
3752
|
async settleEdit(entry, response, context, sent) {
|
|
2884
3753
|
const { path } = entry;
|
|
@@ -2901,15 +3770,16 @@ var SyncClient = class extends EventEmitter {
|
|
|
2901
3770
|
await this.refuse(entry, [{
|
|
2902
3771
|
kind: conflict ? "conflict" : "refused",
|
|
2903
3772
|
path,
|
|
2904
|
-
|
|
3773
|
+
saves: true,
|
|
2905
3774
|
message: (saved) => `${path}: the server refused the change (${response.message}); the text written here was saved to ${saved}${conflict ? " and the file now has the server's version" : ""}`
|
|
2906
3775
|
}]);
|
|
2907
3776
|
if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
|
|
2908
3777
|
}
|
|
2909
3778
|
/**
|
|
2910
3779
|
* Layer 2. Report only when the writer's own content may be lost: lines it added that the result lacks, or lines
|
|
2911
|
-
* it removed that the result still has. A writer that rewrote an older version unchanged, while the disk had
|
|
2912
|
-
*
|
|
3780
|
+
* it removed that the result still has. A writer that rewrote an older version unchanged, while the disk had lines
|
|
3781
|
+
* added since, here or in another environment, is taken as deleting those lines, which were kept: content alone does
|
|
3782
|
+
* not tell it from a program that read the older version and wrote it back. A result that only adds
|
|
2913
3783
|
* other environments' changes to the writer's text is information.
|
|
2914
3784
|
*/
|
|
2915
3785
|
classify(path, context, result) {
|
|
@@ -2950,7 +3820,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2950
3820
|
if (reverted.length) return this.problem({
|
|
2951
3821
|
kind: "deletion-undone",
|
|
2952
3822
|
path,
|
|
2953
|
-
message: `${path}: the write
|
|
3823
|
+
message: `${path}: the write put back an earlier version of the file without ${reverted.length} line(s) added since, here or in another environment, such as "${reverted[0]}"; they were kept, as a program holding the earlier version writes it back the same way. Delete them through a mainflow command or the Edit tool`
|
|
2954
3824
|
});
|
|
2955
3825
|
}
|
|
2956
3826
|
this.info(path, "merged with changes from other environments");
|
|
@@ -2978,7 +3848,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
2978
3848
|
if (!response.ok) return this.refuse(entry, [{
|
|
2979
3849
|
kind: "refused",
|
|
2980
3850
|
path,
|
|
2981
|
-
|
|
3851
|
+
saves: false,
|
|
2982
3852
|
message: () => `${path}: the server refused the deletion (${response.message})`
|
|
2983
3853
|
}]);
|
|
2984
3854
|
if (response.files.some((f) => f.path === path && f.kind === "removed")) this.confirmWritten(path, null);
|
|
@@ -2990,12 +3860,45 @@ var SyncClient = class extends EventEmitter {
|
|
|
2990
3860
|
if (response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
|
|
2991
3861
|
}
|
|
2992
3862
|
/**
|
|
3863
|
+
* What the server answered to binary file `entry`, its content uploaded first: confirmed when applied; refused, as a
|
|
3864
|
+
* conflict when the server holds another version than its base, its content saved and the writer told, as a refused
|
|
3865
|
+
* edit's text is, and the server's version takes its place.
|
|
3866
|
+
*/
|
|
3867
|
+
async settleBlob(entry, response) {
|
|
3868
|
+
const { path } = entry;
|
|
3869
|
+
this.trace(`response ${path} ${response.ok ? `${response.value} blob ${entry.hash.slice(0, 7)}` : `${response.reason}: ${response.message}`}`);
|
|
3870
|
+
if (response.ok) {
|
|
3871
|
+
const disk = this.expected.get(path);
|
|
3872
|
+
if (disk?.blob && disk.hash === entry.hash) this.confirm(path, {
|
|
3873
|
+
hash: entry.hash,
|
|
3874
|
+
blob: true
|
|
3875
|
+
});
|
|
3876
|
+
if (response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
|
|
3877
|
+
return;
|
|
3878
|
+
}
|
|
3879
|
+
const conflict = response.reason === "conflict";
|
|
3880
|
+
await this.refuse(entry, [{
|
|
3881
|
+
kind: conflict ? "conflict" : "refused",
|
|
3882
|
+
path,
|
|
3883
|
+
saves: true,
|
|
3884
|
+
message: (saved) => `${path}: the server refused the binary file written here (${response.message}); it was saved to ${saved}${conflict ? " and the file now has the server's version" : ""}`
|
|
3885
|
+
}]);
|
|
3886
|
+
if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
|
|
3887
|
+
}
|
|
3888
|
+
/**
|
|
2993
3889
|
* What an outbox entry left on disk is what the client expects there; a displaced text left nothing there, since the
|
|
2994
3890
|
* server's version took it off. The versions it holds need nothing more: the outbox is where versions() reads them.
|
|
2995
3891
|
*/
|
|
2996
3892
|
restoreEntry(entry) {
|
|
2997
3893
|
if (entry.refused) {
|
|
2998
3894
|
for (const path of entry.refused.paths) {
|
|
3895
|
+
if (entry.kind === "blob") {
|
|
3896
|
+
if (path === entry.path) this.expected.set(path, {
|
|
3897
|
+
hash: entry.hash,
|
|
3898
|
+
blob: true
|
|
3899
|
+
});
|
|
3900
|
+
continue;
|
|
3901
|
+
}
|
|
2999
3902
|
const text = wroteAt(entry, path);
|
|
3000
3903
|
if (text !== void 0) this.expected.set(path, text === null ? null : {
|
|
3001
3904
|
hash: contentHash(text),
|
|
@@ -3013,7 +3916,11 @@ var SyncClient = class extends EventEmitter {
|
|
|
3013
3916
|
hash: contentHash(entry.text),
|
|
3014
3917
|
text: entry.text
|
|
3015
3918
|
});
|
|
3016
|
-
} else this.expected.set(entry.path,
|
|
3919
|
+
} else if (entry.kind === "blob") this.expected.set(entry.path, {
|
|
3920
|
+
hash: entry.hash,
|
|
3921
|
+
blob: true
|
|
3922
|
+
});
|
|
3923
|
+
else this.expected.set(entry.path, null);
|
|
3017
3924
|
}
|
|
3018
3925
|
/**
|
|
3019
3926
|
* Hold the write lock for `fn`; work stop() abandoned gets an error ('abandoned after stop') instead, so it leaves
|
|
@@ -3060,7 +3967,10 @@ var SyncClient = class extends EventEmitter {
|
|
|
3060
3967
|
changed = true;
|
|
3061
3968
|
this.resolveBases(entries);
|
|
3062
3969
|
}
|
|
3063
|
-
if (changed)
|
|
3970
|
+
if (changed) {
|
|
3971
|
+
writeQueue(this.root, entries);
|
|
3972
|
+
this.pruneStaged(entries);
|
|
3973
|
+
}
|
|
3064
3974
|
this.queue = entries;
|
|
3065
3975
|
return entries;
|
|
3066
3976
|
}
|
|
@@ -3144,6 +4054,12 @@ var SyncClient = class extends EventEmitter {
|
|
|
3144
4054
|
text: before,
|
|
3145
4055
|
base: null
|
|
3146
4056
|
};
|
|
4057
|
+
else if (seen.blob) write = {
|
|
4058
|
+
kind: "patch",
|
|
4059
|
+
path,
|
|
4060
|
+
text: before,
|
|
4061
|
+
base: seen.hash
|
|
4062
|
+
};
|
|
3147
4063
|
else if (seen.hash !== contentHash(before)) write = editOn(path, before, this.startOf(path, before, seen, {
|
|
3148
4064
|
entries,
|
|
3149
4065
|
upto: at
|
|
@@ -3234,7 +4150,10 @@ var SyncClient = class extends EventEmitter {
|
|
|
3234
4150
|
* outbox under the write lock; while another writer holds the lock longer than a writer waits for it, the entry
|
|
3235
4151
|
* stays and is taken out on a later try, without being sent again. Before an entry is first sent, the request it
|
|
3236
4152
|
* sends and its firstSentSeq are written to the outbox (`sent`), so it is sent again as it was even after a restart,
|
|
3237
|
-
* and nothing is merged into it. An edit whose base cannot be moved onto the server's version is not sent.
|
|
4153
|
+
* and nothing is merged into it. An edit whose base cannot be moved onto the server's version is not sent. A binary
|
|
4154
|
+
* file's content is uploaded before its write is first sent, so the server never gets a `file.blob` of content it was
|
|
4155
|
+
* not given (uploadEntry); on a server that keeps no binary files it is not sent, and stays as it is here (dropBlob).
|
|
4156
|
+
* While its upload waits, the writes of other paths go ahead of it, each path's writes still in order (nextEntry).
|
|
3238
4157
|
*/
|
|
3239
4158
|
flushQueue() {
|
|
3240
4159
|
if (this.flushing || this.closing) return Promise.resolve();
|
|
@@ -3249,13 +4168,19 @@ var SyncClient = class extends EventEmitter {
|
|
|
3249
4168
|
try {
|
|
3250
4169
|
while (this.caughtUp) {
|
|
3251
4170
|
let entry;
|
|
4171
|
+
let wake;
|
|
3252
4172
|
let again = false;
|
|
3253
4173
|
let unmergeable = false;
|
|
4174
|
+
let upload = false;
|
|
3254
4175
|
this.editQueue((entries) => {
|
|
3255
|
-
entry =
|
|
4176
|
+
({entry, wake} = this.nextEntry(entries));
|
|
3256
4177
|
if (!entry || this.answered.has(entry.id)) return false;
|
|
3257
4178
|
again = entry.firstSentSeq !== void 0;
|
|
3258
4179
|
if (entry.sent) return false;
|
|
4180
|
+
if (entry.kind === "blob" && (this.blobSupport !== true || !this.uploadedBlobs.has(entry.hash))) {
|
|
4181
|
+
upload = true;
|
|
4182
|
+
return false;
|
|
4183
|
+
}
|
|
3259
4184
|
const request = this.prepare(entry);
|
|
3260
4185
|
if (!request) {
|
|
3261
4186
|
unmergeable = true;
|
|
@@ -3264,7 +4189,14 @@ var SyncClient = class extends EventEmitter {
|
|
|
3264
4189
|
entry.sent = request;
|
|
3265
4190
|
entry.firstSentSeq ??= this.hubSeq;
|
|
3266
4191
|
});
|
|
3267
|
-
if (!entry || this.closing)
|
|
4192
|
+
if (!entry || this.closing) {
|
|
4193
|
+
if (!entry && wake !== void 0) this.retryQueue(Math.max(0, wake - Date.now()));
|
|
4194
|
+
break;
|
|
4195
|
+
}
|
|
4196
|
+
if (upload) {
|
|
4197
|
+
await this.uploadEntry(entry);
|
|
4198
|
+
continue;
|
|
4199
|
+
}
|
|
3268
4200
|
if (unmergeable) await this.refuseUnmergeable(entry);
|
|
3269
4201
|
else if (!this.answered.has(entry.id)) {
|
|
3270
4202
|
this.sending = entry.id;
|
|
@@ -3279,9 +4211,199 @@ var SyncClient = class extends EventEmitter {
|
|
|
3279
4211
|
this.flushing = false;
|
|
3280
4212
|
}
|
|
3281
4213
|
}
|
|
3282
|
-
|
|
4214
|
+
/**
|
|
4215
|
+
* The entry the outbox sends next: the first one waiting, or answered but not taken out yet (a refused write is
|
|
4216
|
+
* answered and waits for nothing). A binary file's write whose content cannot be uploaded now (uploadHeld) waits, and
|
|
4217
|
+
* so does every write after it about one of the paths of a write that waits, so the writes of each path still go in
|
|
4218
|
+
* order; the writes of the other paths go ahead of it. An operation that waits, as one about such a path, holds every
|
|
4219
|
+
* write after it: it may give out ids the writes after it name (renumber). `wake` is when the first write that waits on
|
|
4220
|
+
* a time may go, undefined when none does.
|
|
4221
|
+
*/
|
|
4222
|
+
nextEntry(entries) {
|
|
4223
|
+
const now = Date.now();
|
|
4224
|
+
const held = /* @__PURE__ */ new Set();
|
|
4225
|
+
let wake;
|
|
4226
|
+
for (const queued of entries) {
|
|
4227
|
+
if (this.answered.has(queued.id)) return {
|
|
4228
|
+
entry: queued,
|
|
4229
|
+
wake
|
|
4230
|
+
};
|
|
4231
|
+
if (!waiting(queued)) continue;
|
|
4232
|
+
const paths = entryPaths(queued);
|
|
4233
|
+
if (paths.some((path) => held.has(path))) {
|
|
4234
|
+
if (queued.kind === "op") return { wake };
|
|
4235
|
+
for (const path of paths) held.add(path);
|
|
4236
|
+
continue;
|
|
4237
|
+
}
|
|
4238
|
+
const until = queued.kind === "blob" ? this.uploadHeld(queued) : 0;
|
|
4239
|
+
if (until > now) {
|
|
4240
|
+
for (const path of paths) held.add(path);
|
|
4241
|
+
if (until !== Infinity) wake = Math.min(wake ?? until, until);
|
|
4242
|
+
continue;
|
|
4243
|
+
}
|
|
4244
|
+
return {
|
|
4245
|
+
entry: queued,
|
|
4246
|
+
wake
|
|
4247
|
+
};
|
|
4248
|
+
}
|
|
4249
|
+
return { wake };
|
|
4250
|
+
}
|
|
4251
|
+
/**
|
|
4252
|
+
* Until when binary file `entry`'s write waits for its content to be uploaded: 0 when it may go now, as one uploaded
|
|
4253
|
+
* already; Infinity while the client does not know whether the server keeps binary files, until the snapshot after
|
|
4254
|
+
* the probe sends the outbox again. On a server that keeps none it goes at once, to be dropped (uploadEntry).
|
|
4255
|
+
*/
|
|
4256
|
+
uploadHeld(entry) {
|
|
4257
|
+
if (entry.sent !== void 0 || this.blobSupport === false) return 0;
|
|
4258
|
+
if (this.blobSupport === null) return Infinity;
|
|
4259
|
+
if (this.uploadedBlobs.has(entry.hash)) return 0;
|
|
4260
|
+
return Math.max(this.uploadHolds.get(entry.id) ?? 0, this.uploadAfter.get(entry.hash) ?? 0);
|
|
4261
|
+
}
|
|
4262
|
+
/** Send the outbox again in `after` ms, or shortly. */
|
|
4263
|
+
retryQueue(after) {
|
|
3283
4264
|
clearTimeout(this.queueTimer);
|
|
3284
|
-
if (!this.stopped && !this.closing) this.queueTimer = setTimeout(() => void this.track(this.flushQueue()), 1e3 + Math.random() * 1e3);
|
|
4265
|
+
if (!this.stopped && !this.closing) this.queueTimer = setTimeout(() => void this.track(this.flushQueue()), after ?? 1e3 + Math.random() * 1e3);
|
|
4266
|
+
}
|
|
4267
|
+
/**
|
|
4268
|
+
* Upload the content of binary file `entry` from blobs/, before its write is sent. Once uploaded, the write goes next.
|
|
4269
|
+
* One that cannot be uploaded now waits (uploadHeld) while the writes of other paths go ahead: the time the server
|
|
4270
|
+
* asked for while it deletes that content (Retry-After), or after a failure (uploadFailed). Content blobs/ no longer has
|
|
4271
|
+
* whole, or that the server found not of its sha256, is read again from the file each time, when the file still holds
|
|
4272
|
+
* it (stageHeld); when it does not, the write is dropped as superseded (supersede). On a server that keeps no binary
|
|
4273
|
+
* files, or that refuses the content as too large, the write is dropped and the file left as it is here (dropBlob).
|
|
4274
|
+
*/
|
|
4275
|
+
async uploadEntry(entry) {
|
|
4276
|
+
if (this.blobSupport === false) return this.dropBlob(entry, () => this.skipBinary(entry.path, entry.hash));
|
|
4277
|
+
if (this.blobSupport === null) return;
|
|
4278
|
+
this.uploadAfter.delete(entry.hash);
|
|
4279
|
+
const staged = join(this.paths.blobs, entry.hash);
|
|
4280
|
+
if ((await stat(staged).catch(() => null))?.size !== entry.size) {
|
|
4281
|
+
rmSync(staged, { force: true });
|
|
4282
|
+
if (!await this.restage(entry)) return this.supersede(entry);
|
|
4283
|
+
}
|
|
4284
|
+
let result;
|
|
4285
|
+
try {
|
|
4286
|
+
result = await this.api.putBlob(entry.hash, staged, entry.size);
|
|
4287
|
+
} catch (error) {
|
|
4288
|
+
if (!(error instanceof Unreachable || error instanceof ServerError)) throw error;
|
|
4289
|
+
return this.uploadFailed(entry, error.message);
|
|
4290
|
+
}
|
|
4291
|
+
if (result.ok) {
|
|
4292
|
+
this.uploadedBlobs.add(entry.hash);
|
|
4293
|
+
this.uploadFailures.delete(entry.id);
|
|
4294
|
+
this.uploadHolds.delete(entry.id);
|
|
4295
|
+
this.trace(`uploaded ${entry.path} blob ${entry.hash.slice(0, 7)}`);
|
|
4296
|
+
return;
|
|
4297
|
+
}
|
|
4298
|
+
if ("retryAfter" in result) {
|
|
4299
|
+
this.log(`${entry.path}: the sync server is deleting the content ${entry.hash.slice(0, 7)}; uploading it again in ${result.retryAfter} s`);
|
|
4300
|
+
this.uploadAfter.set(entry.hash, Date.now() + result.retryAfter * 1e3);
|
|
4301
|
+
return;
|
|
4302
|
+
}
|
|
4303
|
+
this.log(`${entry.path}: the sync server refused its content ${entry.hash.slice(0, 7)}: ${result.message}`);
|
|
4304
|
+
switch (result.refused) {
|
|
4305
|
+
case "hash-mismatch":
|
|
4306
|
+
rmSync(staged, { force: true });
|
|
4307
|
+
if (!await this.restage(entry)) return this.supersede(entry);
|
|
4308
|
+
return this.uploadFailed(entry, result.message, true);
|
|
4309
|
+
case "too-large":
|
|
4310
|
+
this.oversized.add(`${entry.path}\0${entry.hash}`);
|
|
4311
|
+
return this.dropBlob(entry, () => this.skipLarge(entry.path, entry.hash, true));
|
|
4312
|
+
case "unsupported":
|
|
4313
|
+
this.blobSupport = false;
|
|
4314
|
+
return this.dropBlob(entry, () => this.skipBinary(entry.path, entry.hash));
|
|
4315
|
+
default: return this.uploadFailed(entry, result.message);
|
|
4316
|
+
}
|
|
4317
|
+
}
|
|
4318
|
+
/**
|
|
4319
|
+
* The upload of binary file `entry`'s content failed for `reason`: its write waits, while the writes of other paths go
|
|
4320
|
+
* ahead (nextEntry). The first BLOB_FAILURES - 1 failures in a row are tried again shortly, or `atOnce`, as content
|
|
4321
|
+
* read again after a hash-mismatch is; from the BLOB_FAILURES-th on, the wait doubles up to BLOB_BACKOFF_MAX_MS, and
|
|
4322
|
+
* the writer is told once for the write.
|
|
4323
|
+
*/
|
|
4324
|
+
uploadFailed(entry, reason, atOnce = false) {
|
|
4325
|
+
const failures = (this.uploadFailures.get(entry.id) ?? 0) + 1;
|
|
4326
|
+
this.uploadFailures.set(entry.id, failures);
|
|
4327
|
+
if (failures < BLOB_FAILURES) {
|
|
4328
|
+
this.log(`${entry.path}: could not upload its content ${entry.hash.slice(0, 7)}: ${reason}; trying again`);
|
|
4329
|
+
if (!atOnce) this.uploadHolds.set(entry.id, Date.now() + BLOB_RETRY_MS * (1 + Math.random()));
|
|
4330
|
+
return;
|
|
4331
|
+
}
|
|
4332
|
+
const wait = Math.min(BLOB_BACKOFF_MS * 2 ** (failures - BLOB_FAILURES), BLOB_BACKOFF_MAX_MS);
|
|
4333
|
+
this.uploadHolds.set(entry.id, Date.now() + wait);
|
|
4334
|
+
this.log(`${entry.path}: could not upload its content ${entry.hash.slice(0, 7)} ${failures} times in a row: ${reason}; trying again in ${Math.round(wait / 1e3)} s`);
|
|
4335
|
+
this.tellOnce(`upload:${entry.id}`, {
|
|
4336
|
+
kind: "upload-failing",
|
|
4337
|
+
path: entry.path,
|
|
4338
|
+
message: `${entry.path}: the binary file written here could not be uploaded ${failures} times in a row (${reason}); it stays here, its write waits in the outbox and is tried again less and less often, while the writes of other files go ahead`
|
|
4339
|
+
});
|
|
4340
|
+
}
|
|
4341
|
+
/** Put the content of `entry` into blobs/ again from its file, when the file still holds it: false when it does not. */
|
|
4342
|
+
restage(entry) {
|
|
4343
|
+
return this.whenFree(() => this.locked(() => this.queue.some((queued) => queued.id === entry.id) && this.stageHeld(entry.path, entry.hash)));
|
|
4344
|
+
}
|
|
4345
|
+
/**
|
|
4346
|
+
* Take binary file `entry` out of the outbox without sending it, its file left as it is here: a server that keeps no
|
|
4347
|
+
* binary files, or not one this large. The client expects what the entry's base names there again (unqueueBlob), so
|
|
4348
|
+
* the file is a change still to send: the reconcile after it leaves it as it is (binarySkip), and sends it once the
|
|
4349
|
+
* server keeps binary files, as the snapshot of a later connection finds, or once it is small enough. `tell` tells the
|
|
4350
|
+
* writer, once for each content.
|
|
4351
|
+
*/
|
|
4352
|
+
async dropBlob(entry, tell) {
|
|
4353
|
+
await this.unqueueBlob(entry);
|
|
4354
|
+
tell();
|
|
4355
|
+
this.schedule(entry.path);
|
|
4356
|
+
}
|
|
4357
|
+
/**
|
|
4358
|
+
* Take binary file `entry` out of the outbox once its content is nowhere: blobs/ lost it and the file holds another
|
|
4359
|
+
* since. The file's later content supersedes it, as a binary file keeps only its latest version: the reconcile after
|
|
4360
|
+
* it sends the file as it is now, on the entry's base (unqueueBlob). Recorded as information.
|
|
4361
|
+
*/
|
|
4362
|
+
async supersede(entry) {
|
|
4363
|
+
await this.unqueueBlob(entry);
|
|
4364
|
+
this.info(entry.path, `the content ${entry.hash.slice(0, 7)} written here was no longer anywhere to upload, and the file holds another since; the file is sent as it is now`);
|
|
4365
|
+
this.schedule(entry.path);
|
|
4366
|
+
}
|
|
4367
|
+
/**
|
|
4368
|
+
* Take binary file `entry` out of the outbox unsent, as if it had never been queued: what the client expects at its
|
|
4369
|
+
* path goes back to the entry's base, and so do the bases of the writes after it on its content, in one hold of the
|
|
4370
|
+
* write lock.
|
|
4371
|
+
*/
|
|
4372
|
+
async unqueueBlob(entry) {
|
|
4373
|
+
const back = entry.base === null ? null : BLOB_HASH.test(entry.base) ? {
|
|
4374
|
+
hash: entry.base,
|
|
4375
|
+
blob: true
|
|
4376
|
+
} : { hash: entry.base };
|
|
4377
|
+
await this.whenFree(() => this.editQueue((entries) => {
|
|
4378
|
+
const at = entries.findIndex((queued) => queued.id === entry.id);
|
|
4379
|
+
if (at < 0) return false;
|
|
4380
|
+
entries.splice(at, 1);
|
|
4381
|
+
for (const later of entries.slice(at)) {
|
|
4382
|
+
if (later.kind === "op" || later.path !== entry.path || later.sent !== void 0 || later.base !== entry.hash) continue;
|
|
4383
|
+
if (later.kind === "remove" && entry.base === null) continue;
|
|
4384
|
+
later.base = entry.base;
|
|
4385
|
+
}
|
|
4386
|
+
const disk = this.expected.get(entry.path);
|
|
4387
|
+
if (disk?.blob && disk.hash === entry.hash) this.expected.set(entry.path, back);
|
|
4388
|
+
}));
|
|
4389
|
+
this.uploadFailures.delete(entry.id);
|
|
4390
|
+
this.uploadHolds.delete(entry.id);
|
|
4391
|
+
}
|
|
4392
|
+
/**
|
|
4393
|
+
* The server refused binary file `entry` as `missing-blob`: it holds no such content, never given or cleaned up since.
|
|
4394
|
+
* The content is uploaded again and the write sent again as a new request, under a requestId of its own (requestIdOf),
|
|
4395
|
+
* since the server keeps the refusal under the one it was sent with.
|
|
4396
|
+
*/
|
|
4397
|
+
async retryBlob(entry) {
|
|
4398
|
+
this.uploadedBlobs.delete(entry.hash);
|
|
4399
|
+
this.log(`${entry.path}: the sync server holds no content ${entry.hash.slice(0, 7)}; uploading it again`);
|
|
4400
|
+
await this.whenFree(() => this.editQueue((entries) => {
|
|
4401
|
+
const queued = entries.find((e) => e.id === entry.id);
|
|
4402
|
+
if (queued?.kind !== "blob") return false;
|
|
4403
|
+
queued.retry = (queued.retry ?? 0) + 1;
|
|
4404
|
+
delete queued.sent;
|
|
4405
|
+
delete queued.firstSentSeq;
|
|
4406
|
+
}));
|
|
3285
4407
|
}
|
|
3286
4408
|
/**
|
|
3287
4409
|
* Take an answered entry out of the outbox, and give the ids the server renumbered to the files written here and
|
|
@@ -3352,15 +4474,16 @@ var SyncClient = class extends EventEmitter {
|
|
|
3352
4474
|
*/
|
|
3353
4475
|
async request(entry, request, again, fresh = false) {
|
|
3354
4476
|
const firstSentSeq = again ? entry.firstSentSeq : void 0;
|
|
4477
|
+
const requestId = requestIdOf(entry);
|
|
3355
4478
|
this.trace(`send ${request.op} ${entry.id}${firstSentSeq === void 0 ? "" : ` again, first sent at seq ${firstSentSeq}`}`);
|
|
3356
4479
|
let response;
|
|
3357
4480
|
try {
|
|
3358
4481
|
response = await this.api.send(firstSentSeq === void 0 ? {
|
|
3359
4482
|
...request,
|
|
3360
|
-
requestId
|
|
4483
|
+
requestId
|
|
3361
4484
|
} : {
|
|
3362
4485
|
...request,
|
|
3363
|
-
requestId
|
|
4486
|
+
requestId,
|
|
3364
4487
|
firstSentSeq
|
|
3365
4488
|
}, { fresh });
|
|
3366
4489
|
} catch (error) {
|
|
@@ -3398,6 +4521,14 @@ var SyncClient = class extends EventEmitter {
|
|
|
3398
4521
|
},
|
|
3399
4522
|
now: requestTime()
|
|
3400
4523
|
};
|
|
4524
|
+
if (entry.kind === "blob") return {
|
|
4525
|
+
protocol: 3,
|
|
4526
|
+
op: "file.blob",
|
|
4527
|
+
path: entry.path,
|
|
4528
|
+
hash: entry.hash,
|
|
4529
|
+
size: entry.size,
|
|
4530
|
+
base: entry.base
|
|
4531
|
+
};
|
|
3401
4532
|
if (entry.replace) return {
|
|
3402
4533
|
protocol: 3,
|
|
3403
4534
|
op: "text.replace",
|
|
@@ -3414,7 +4545,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3414
4545
|
await this.refuse(entry, [{
|
|
3415
4546
|
kind: "conflict",
|
|
3416
4547
|
path: entry.path,
|
|
3417
|
-
|
|
4548
|
+
saves: true,
|
|
3418
4549
|
message: (saved) => `${entry.path} was edited on top of a write the server did not take as it was, or the edit overlaps what the server merged into it; the text written here was saved to ${saved}`
|
|
3419
4550
|
}]);
|
|
3420
4551
|
}
|
|
@@ -3424,11 +4555,12 @@ var SyncClient = class extends EventEmitter {
|
|
|
3424
4555
|
* the entry: what the writer is told, with the name each text is saved under, and the files that hold what the entry
|
|
3425
4556
|
* wrote there as the client left it. Then each text is saved and the writer told. The entry is sent no more; it leaves
|
|
3426
4557
|
* the outbox once no file holds what it wrote (releaseHeld), so a restart before the server's version is on disk puts
|
|
3427
|
-
* that version back rather than send the refused text again. An operation's writer is answered with `outcome`.
|
|
4558
|
+
* that version back rather than send the refused text again. An operation's writer is answered with `outcome`. `saves`
|
|
4559
|
+
* says whether the entry wrote something at the path to save, a text or a binary file's content (tell).
|
|
3428
4560
|
*/
|
|
3429
4561
|
async refuse(entry, told, outcome) {
|
|
3430
|
-
const problems = told.map(({ kind, path,
|
|
3431
|
-
const saved =
|
|
4562
|
+
const problems = told.map(({ kind, path, saves, message }) => {
|
|
4563
|
+
const saved = saves ? this.conflictName(path) : void 0;
|
|
3432
4564
|
return {
|
|
3433
4565
|
kind,
|
|
3434
4566
|
path,
|
|
@@ -3466,7 +4598,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
3466
4598
|
for (const problem of refusal.told) {
|
|
3467
4599
|
if (before(problem)) continue;
|
|
3468
4600
|
const text = textAt(entry, problem.path);
|
|
3469
|
-
|
|
4601
|
+
const failed = (error) => this.log(`${problem.path}: could not save ${problem.saved}: ${String(error)}`);
|
|
4602
|
+
if (problem.saved !== void 0 && entry.kind === "blob") await this.saveStaged(problem.saved, entry.hash).catch(failed);
|
|
4603
|
+
else if (problem.saved !== void 0 && typeof text === "string") await this.saveText(problem.saved, text).catch(failed);
|
|
3470
4604
|
this.problem(problem, entry.id);
|
|
3471
4605
|
}
|
|
3472
4606
|
if (refusal.outcome) this.settled(entry.id, {
|
|
@@ -3478,8 +4612,9 @@ var SyncClient = class extends EventEmitter {
|
|
|
3478
4612
|
}
|
|
3479
4613
|
/** Whether `path` holds what `entry` wrote there, as the client left it. */
|
|
3480
4614
|
leftHere(entry, path) {
|
|
3481
|
-
const text = wroteAt(entry, path);
|
|
3482
4615
|
const disk = this.expected.get(path);
|
|
4616
|
+
if (entry.kind === "blob") return path === entry.path && disk?.blob === true && disk.hash === entry.hash;
|
|
4617
|
+
const text = wroteAt(entry, path);
|
|
3483
4618
|
if (text === void 0 || disk === void 0) return false;
|
|
3484
4619
|
return text === null ? disk === null : disk !== null && disk.hash === contentHash(text);
|
|
3485
4620
|
}
|
|
@@ -3526,6 +4661,10 @@ var SyncClient = class extends EventEmitter {
|
|
|
3526
4661
|
try {
|
|
3527
4662
|
const response = await this.request(entry, request, again);
|
|
3528
4663
|
if (response === null) return false;
|
|
4664
|
+
if (entry.kind === "blob" && !response.ok && response.reason === "missing-blob") {
|
|
4665
|
+
await this.retryBlob(entry);
|
|
4666
|
+
return false;
|
|
4667
|
+
}
|
|
3529
4668
|
this.answer(entry);
|
|
3530
4669
|
if (entry.kind === "op") await this.settleQueuedOp(entry, response);
|
|
3531
4670
|
else if (entry.kind === "remove") await this.settleRemove(entry, response);
|
|
@@ -3535,6 +4674,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3535
4674
|
base: entry.baseText,
|
|
3536
4675
|
last: entry.replaced
|
|
3537
4676
|
}, request.text);
|
|
4677
|
+
else if (entry.kind === "blob") await this.settleBlob(entry, response);
|
|
3538
4678
|
return true;
|
|
3539
4679
|
} finally {
|
|
3540
4680
|
for (const path of paths) {
|
|
@@ -3550,7 +4690,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3550
4690
|
const told = Object.entries(entry.written).flatMap(([path, text]) => text === null ? [] : [{
|
|
3551
4691
|
kind: "refused",
|
|
3552
4692
|
path,
|
|
3553
|
-
|
|
4693
|
+
saves: true,
|
|
3554
4694
|
message: (saved) => `the server refused ${entry.request.op} (${response.message}); ${path} as it was written here was saved to ${saved} and the file has the server's version again`
|
|
3555
4695
|
}]);
|
|
3556
4696
|
return this.refuse(entry, told, {
|
|
@@ -3580,7 +4720,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3580
4720
|
* The server gave some of the entry's provisional ids away: rewrite the files written offline, the queued requests
|
|
3581
4721
|
* and edits after it, and what the client expects on disk (docs/sync-protocol.md). Called under the write lock with
|
|
3582
4722
|
* the outbox as read, so a writer appending meanwhile sees either the old ids everywhere or the new. The outbox is
|
|
3583
|
-
* sent in order, so every entry after it is still to be sent; one sent already (`sent`), as an older sync client left
|
|
4723
|
+
* sent in order for each path, so every entry after it about its files is still to be sent; one sent already (`sent`), as an older sync client left
|
|
3584
4724
|
* behind an operation when it sent edits outside the outbox, stays as the server may have taken it, and so do the
|
|
3585
4725
|
* files only it wrote; the server's version of them comes back once it is answered. An edit queued on top of it
|
|
3586
4726
|
* keeps that text as its base, so its renumbering is what it changes, and it puts the server's id there when it
|
|
@@ -3590,7 +4730,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3590
4730
|
* the sent write wrote it. A refused write is no write to send.
|
|
3591
4731
|
*/
|
|
3592
4732
|
renumber(entries, entry, renumbered) {
|
|
3593
|
-
const rest = entries.
|
|
4733
|
+
const rest = entries.slice(entries.findIndex((queued) => queued.id === entry.id) + 1).filter((queued) => queued.sent === void 0 && waiting(queued));
|
|
3594
4734
|
const offlinePaths = [...entryPaths(entry), ...rest.flatMap(entryPaths)];
|
|
3595
4735
|
this.renumberDisk(offlinePaths, renumbered);
|
|
3596
4736
|
const files = (record) => Object.fromEntries(Object.entries(record).map(([path, text]) => {
|
|
@@ -3649,6 +4789,22 @@ var SyncClient = class extends EventEmitter {
|
|
|
3649
4789
|
lock: (fn) => fn()
|
|
3650
4790
|
};
|
|
3651
4791
|
const seen = new Map([...new Set(paths)].filter((path) => this.syncs(path)).map((path) => [path, this.expected.get(path)]));
|
|
4792
|
+
const binary = [...seen].filter(([path]) => {
|
|
4793
|
+
try {
|
|
4794
|
+
return contentOf(readFileSync(join(this.paths.data, path))).kind === "binary";
|
|
4795
|
+
} catch {
|
|
4796
|
+
return false;
|
|
4797
|
+
}
|
|
4798
|
+
});
|
|
4799
|
+
for (const [path] of binary) seen.delete(path);
|
|
4800
|
+
for (const [path, disk] of binary) {
|
|
4801
|
+
const to = renumberPath(path, renumbered);
|
|
4802
|
+
if (to === path) continue;
|
|
4803
|
+
mkdirSync(dirname(join(this.paths.data, to)), { recursive: true });
|
|
4804
|
+
renameSync(join(this.paths.data, path), join(this.paths.data, to));
|
|
4805
|
+
this.expected.set(path, null);
|
|
4806
|
+
if (disk !== void 0) this.expected.set(to, disk);
|
|
4807
|
+
}
|
|
3652
4808
|
const changes = renumberFiles(storage, seen.keys(), renumbered);
|
|
3653
4809
|
const unknown = /* @__PURE__ */ new Set();
|
|
3654
4810
|
for (const [path, disk] of seen) {
|
|
@@ -3684,30 +4840,204 @@ var SyncClient = class extends EventEmitter {
|
|
|
3684
4840
|
this.reported = this.problems.length;
|
|
3685
4841
|
return out;
|
|
3686
4842
|
}
|
|
3687
|
-
/**
|
|
4843
|
+
/**
|
|
4844
|
+
* Send every change now, wait for the server, and return what went wrong: `mainflow sync flush`, and a hook of an
|
|
4845
|
+
* older CLI that names no tool. No change waits for an Edit's report (awaitEdit) any longer, nor starts to meanwhile.
|
|
4846
|
+
*/
|
|
3688
4847
|
async flush(timeoutMs = 2e4) {
|
|
3689
|
-
|
|
3690
|
-
|
|
3691
|
-
|
|
3692
|
-
|
|
3693
|
-
|
|
3694
|
-
|
|
3695
|
-
|
|
4848
|
+
this.sendingAll++;
|
|
4849
|
+
try {
|
|
4850
|
+
for (const wait of this.awaitingEdit.values()) wait.end(null);
|
|
4851
|
+
await this.scan(true);
|
|
4852
|
+
this.outboxChanged();
|
|
4853
|
+
await this.idle(timeoutMs);
|
|
4854
|
+
return {
|
|
4855
|
+
problems: this.drain(),
|
|
4856
|
+
queued: this.queued,
|
|
4857
|
+
online: this.online
|
|
4858
|
+
};
|
|
4859
|
+
} finally {
|
|
4860
|
+
this.sendingAll--;
|
|
4861
|
+
}
|
|
4862
|
+
}
|
|
4863
|
+
/**
|
|
4864
|
+
* After a tool other than an Edit ran, as its PostToolUse hook reports it: find every change now, wait for the server,
|
|
4865
|
+
* and return what went wrong. `path`, the file the tool wrote when it names one (Write), is no Edit: its change waiting
|
|
4866
|
+
* for an Edit's report is sent now, and one found meanwhile does not wait. `call`, the tool call when the hook names
|
|
4867
|
+
* it, ends the announcements of its session (settleCaller). A change of any other file waiting for an Edit's report,
|
|
4868
|
+
* which only an announced Edit's change does, as of another session sharing the sync client, goes on waiting for it:
|
|
4869
|
+
* the tool reported here did not write it, or may not have, as a Bash command, and the flush does not wait for it. Its
|
|
4870
|
+
* problems go to the caller after it ends (drain).
|
|
4871
|
+
*/
|
|
4872
|
+
async flushTool(path, timeoutMs = 2e4, call) {
|
|
4873
|
+
this.settleCaller(call);
|
|
4874
|
+
if (path !== void 0) {
|
|
4875
|
+
this.sendingPaths.set(path, (this.sendingPaths.get(path) ?? 0) + 1);
|
|
4876
|
+
this.awaitingEdit.get(path)?.end(null);
|
|
4877
|
+
}
|
|
4878
|
+
try {
|
|
4879
|
+
await this.scan(true, false, false);
|
|
4880
|
+
this.outboxChanged();
|
|
4881
|
+
await this.idle(timeoutMs, true);
|
|
4882
|
+
return {
|
|
4883
|
+
problems: this.drain(),
|
|
4884
|
+
queued: this.queued,
|
|
4885
|
+
online: this.online
|
|
4886
|
+
};
|
|
4887
|
+
} finally {
|
|
4888
|
+
if (path !== void 0) {
|
|
4889
|
+
const left = (this.sendingPaths.get(path) ?? 1) - 1;
|
|
4890
|
+
if (left > 0) this.sendingPaths.set(path, left);
|
|
4891
|
+
else this.sendingPaths.delete(path);
|
|
4892
|
+
}
|
|
4893
|
+
}
|
|
4894
|
+
}
|
|
4895
|
+
/**
|
|
4896
|
+
* Before an Edit runs, as its PreToolUse hook announces it, with the tool call when the hook names it: the change it
|
|
4897
|
+
* makes, its one replacement of the version on disk, may wait for its report (awaitEdit). A change no Edit announced,
|
|
4898
|
+
* as a Bash command's, an editor's, or an Edit's whose hook does not announce it (an older CLI or forwarding script),
|
|
4899
|
+
* is sent at once. Several Edits of one file may be announced at a time, as by sessions sharing the sync client.
|
|
4900
|
+
* Announcements past announceTtlMs go here.
|
|
4901
|
+
*/
|
|
4902
|
+
announceEdit(edit, call) {
|
|
4903
|
+
const now = Date.now();
|
|
4904
|
+
for (const [path, list] of this.announced) {
|
|
4905
|
+
const live = list.filter((a) => a.expires > now);
|
|
4906
|
+
if (live.length) this.announced.set(path, live);
|
|
4907
|
+
else this.announced.delete(path);
|
|
4908
|
+
}
|
|
4909
|
+
if (this.closing || !this.syncs(edit.path) || edit.old === "") return;
|
|
4910
|
+
const announcement = {
|
|
4911
|
+
edit: {
|
|
4912
|
+
path: edit.path,
|
|
4913
|
+
old: edit.old,
|
|
4914
|
+
new: edit.new
|
|
4915
|
+
},
|
|
4916
|
+
...call ? { call } : {},
|
|
4917
|
+
left: this.opts.editReportMs ?? 3e3,
|
|
4918
|
+
expires: now + (this.opts.announceTtlMs ?? 6e5)
|
|
3696
4919
|
};
|
|
4920
|
+
this.announced.set(edit.path, [...this.announced.get(edit.path) ?? [], announcement]);
|
|
4921
|
+
this.trace(`announce edit ${edit.path}`);
|
|
4922
|
+
}
|
|
4923
|
+
/** The announcement of `path`, live and with time left to wait, of an Edit that makes `to` of `from`. */
|
|
4924
|
+
announcementFor(path, from, to) {
|
|
4925
|
+
const now = Date.now();
|
|
4926
|
+
return this.announced.get(path)?.find((a) => a.expires > now && a.left > 0 && replaces(a.edit, from, to));
|
|
4927
|
+
}
|
|
4928
|
+
dropAnnouncement(path, announcement) {
|
|
4929
|
+
const list = this.announced.get(path)?.filter((a) => a !== announcement) ?? [];
|
|
4930
|
+
if (list.length) this.announced.set(path, list);
|
|
4931
|
+
else this.announced.delete(path);
|
|
4932
|
+
}
|
|
4933
|
+
/**
|
|
4934
|
+
* A tool call of `call`'s session, or of the same subagent in it, was reported: the Edits that session announced
|
|
4935
|
+
* before have run, and were reported, or never will, as one refused or failed, which runs no PostToolUse hook. A
|
|
4936
|
+
* session runs one such tool at a time, so their announcements are dropped, and a change waiting on one is sent now,
|
|
4937
|
+
* before the flush of this report waits for what it finds: a Bash command that made the change a refused Edit
|
|
4938
|
+
* announced hears of it at its own hook. Another session's announcements, or the session's subagents', stay.
|
|
4939
|
+
*/
|
|
4940
|
+
settleCaller(call) {
|
|
4941
|
+
if (call === void 0) return;
|
|
4942
|
+
const dropped = /* @__PURE__ */ new Set();
|
|
4943
|
+
for (const [path, list] of this.announced) for (const announcement of list.filter((a) => sameCaller(a.call, call))) {
|
|
4944
|
+
dropped.add(announcement);
|
|
4945
|
+
this.dropAnnouncement(path, announcement);
|
|
4946
|
+
}
|
|
4947
|
+
for (const wait of this.awaitingEdit.values()) if (dropped.has(wait.announcement)) wait.end(null);
|
|
4948
|
+
if (dropped.size) this.trace(`dropped ${dropped.size} announcement(s) of ${call.session}${call.agent ? `/${call.agent}` : ""}`);
|
|
3697
4949
|
}
|
|
3698
4950
|
/**
|
|
3699
4951
|
* After an Edit: replay the replacement on the server's latest version, so an Edit made while a broadcast was on
|
|
3700
|
-
* its way is not mistaken for a rewrite of the older file
|
|
4952
|
+
* its way is not mistaken for a rewrite of the older file, nor one of lines added a moment ago for a write of the
|
|
4953
|
+
* version before them. The report takes the Edit's announcement: the one of its tool call when the hook names it,
|
|
4954
|
+
* else the oldest of the same replacement. The rest of the session's announcements go (settleCaller). The change the
|
|
4955
|
+
* Edit made, waiting for this report (awaitEdit), is replayed where it waits. One not waiting yet, as when the
|
|
4956
|
+
* watcher's reconcile is still on its way to awaitEdit, takes the report from reportedEdits when it gets there, ahead
|
|
4957
|
+
* of the replay queued here on the path's chain; the report is dropped once that replay ran. Then the files are
|
|
4958
|
+
* flushed as for any tool that wrote this one.
|
|
3701
4959
|
*/
|
|
3702
|
-
async flushEdit(edit, timeoutMs = 2e4) {
|
|
4960
|
+
async flushEdit(edit, timeoutMs = 2e4, call) {
|
|
3703
4961
|
const { path } = edit;
|
|
3704
|
-
|
|
3705
|
-
|
|
4962
|
+
const list = this.announced.get(path);
|
|
4963
|
+
const announcement = call?.toolUseId !== void 0 ? list?.find((a) => sameCaller(a.call, call) && a.call?.toolUseId === call.toolUseId) : list?.find((a) => a.edit.old === edit.old && a.edit.new === edit.new);
|
|
4964
|
+
if (announcement) this.dropAnnouncement(path, announcement);
|
|
4965
|
+
const replays = this.caughtUp && this.syncs(path) && edit.old !== "";
|
|
4966
|
+
const waiting = replays ? this.awaitingEdit.get(path) : void 0;
|
|
4967
|
+
const handed = waiting !== void 0 && replaces(edit, waiting.from, waiting.to);
|
|
4968
|
+
if (handed) waiting.end(edit);
|
|
4969
|
+
this.settleCaller(call);
|
|
4970
|
+
if (replays && !handed) {
|
|
4971
|
+
this.reportedEdits.set(path, edit);
|
|
4972
|
+
await this.withChain(path, () => this.replayEdit(edit));
|
|
4973
|
+
if (this.reportedEdits.get(path) === edit) this.reportedEdits.delete(path);
|
|
4974
|
+
}
|
|
4975
|
+
return this.flushTool(path, timeoutMs, call);
|
|
4976
|
+
}
|
|
4977
|
+
/**
|
|
4978
|
+
* The Edit reported for the local `text` of `path`, which the client left holding `disk`, or null. An Edit starts
|
|
4979
|
+
* from the version on disk, and the watcher sees its write before the PostToolUse hook reports it. Layer 1 bases a
|
|
4980
|
+
* change on the recent version nearest to it, which for an Edit that removes or rewrites lines added a moment ago is
|
|
4981
|
+
* the version before them, as for a program that read that version and wrote it back: the lines would be kept, or
|
|
4982
|
+
* the change refused as overlapping them.
|
|
4983
|
+
* So a change that is exactly the replacement of an Edit announced before it ran (announceEdit), when layer 1 would
|
|
4984
|
+
* base it on another version than the one on disk, waits for the report: the Edit's (flushEdit) is replayed in its
|
|
4985
|
+
* place; a report of the tool that wrote this file (flushTool), flush(), the announcement's time running out or stop()
|
|
4986
|
+
* sends it as layer 1 bases it, as does a later report of the session that announced it (settleCaller). The waits on
|
|
4987
|
+
* one announcement add up to editReportMs at most: once used up, it is dropped. Any other change does not wait: no
|
|
4988
|
+
* Edit named it before it ran, so no report is due for it.
|
|
4989
|
+
* A report that came first (reportedEdits) is taken at once when the Edit it names made this text; while there is one,
|
|
4990
|
+
* no change of the path starts to wait, as its report came already.
|
|
4991
|
+
* Nor does a change wait unless it may be an Edit's replay: the client caught up, as flushEdit replays only then, and
|
|
4992
|
+
* no write about the file waiting for the server (pending). An answer to one changes the versions layer 1 chooses
|
|
4993
|
+
* from, so a change that waited past it would be based on another version than the one it was read against. Nor does
|
|
4994
|
+
* a change found while every change is being sent (flush), or one of a file a tool reported as written meanwhile.
|
|
4995
|
+
*/
|
|
4996
|
+
async awaitEdit(path, text, disk) {
|
|
4997
|
+
if (disk.text === void 0 || this.closing) return null;
|
|
4998
|
+
const from = disk.text;
|
|
4999
|
+
const reported = this.reportedEdits.get(path);
|
|
5000
|
+
if (reported) {
|
|
5001
|
+
if (!replaces(reported, from, text)) return null;
|
|
5002
|
+
this.reportedEdits.delete(path);
|
|
5003
|
+
this.trace(`await edit ${path} ${short(text)} reported before it waited`);
|
|
5004
|
+
return reported;
|
|
5005
|
+
}
|
|
5006
|
+
const announcement = this.announcementFor(path, from, text);
|
|
5007
|
+
if (!announcement || !this.caughtUp || this.pending(path) || this.sendingAll > 0 || this.sendingPaths.has(path)) return null;
|
|
5008
|
+
const start = this.startOf(path, text, disk);
|
|
5009
|
+
if (start === void 0 || start.text === from) return null;
|
|
5010
|
+
this.trace(`await edit ${path} ${short(text)} on disk ${short(from)} nearest ${short(start.text)} up to ${announcement.left} ms`);
|
|
5011
|
+
const started = Date.now();
|
|
5012
|
+
let timedOut = false;
|
|
5013
|
+
const edit = await new Promise((resolve) => {
|
|
5014
|
+
const wait = {
|
|
5015
|
+
from,
|
|
5016
|
+
to: text,
|
|
5017
|
+
announcement,
|
|
5018
|
+
end: (edit) => {
|
|
5019
|
+
clearTimeout(timer);
|
|
5020
|
+
if (this.awaitingEdit.get(path) === wait) this.awaitingEdit.delete(path);
|
|
5021
|
+
resolve(edit);
|
|
5022
|
+
}
|
|
5023
|
+
};
|
|
5024
|
+
const timer = setTimeout(() => {
|
|
5025
|
+
timedOut = true;
|
|
5026
|
+
wait.end(null);
|
|
5027
|
+
}, announcement.left);
|
|
5028
|
+
this.awaitingEdit.get(path)?.end(null);
|
|
5029
|
+
this.awaitingEdit.set(path, wait);
|
|
5030
|
+
});
|
|
5031
|
+
announcement.left -= Date.now() - started;
|
|
5032
|
+
if (timedOut || announcement.left <= 0) this.dropAnnouncement(path, announcement);
|
|
5033
|
+
this.trace(`await edit ${path} ${short(text)} ${edit ? "reported as an Edit" : "over"}`);
|
|
5034
|
+
return edit;
|
|
3706
5035
|
}
|
|
3707
5036
|
/**
|
|
3708
5037
|
* Queue an Edit as its replacement, which the server makes again on its latest version, based as layer 1 knows the
|
|
3709
|
-
* version the Edit changed, the one on disk.
|
|
3710
|
-
*
|
|
5038
|
+
* version the Edit changed, the one on disk. The replacement sent is the text the Edit replaced there (editTarget),
|
|
5039
|
+
* with the line break after a deleted line when the Edit took it. One the watcher queued as a whole text already, or
|
|
5040
|
+
* that is not that replacement of the version on disk, is left to the flush. The Edit's result is adopted in the hold of the write lock
|
|
3711
5041
|
* that queues it (I2); one changed there since it was read, as by an operation written on it, which then has the Edit
|
|
3712
5042
|
* queued ahead of it as a direct edit, is reconciled again.
|
|
3713
5043
|
*/
|
|
@@ -3715,7 +5045,11 @@ var SyncClient = class extends EventEmitter {
|
|
|
3715
5045
|
const disk = this.expected.get(path);
|
|
3716
5046
|
const local = await this.readLocal(path);
|
|
3717
5047
|
if (local.kind !== "text" || disk?.text === void 0 || contentHash(local.text) === disk.hash) return;
|
|
3718
|
-
|
|
5048
|
+
const target = editTarget(disk.text, {
|
|
5049
|
+
old,
|
|
5050
|
+
new: replacement
|
|
5051
|
+
});
|
|
5052
|
+
if (target === null || disk.text.replace(target, () => replacement) !== local.text) return;
|
|
3719
5053
|
const edited = local.text;
|
|
3720
5054
|
const { hash, text } = disk;
|
|
3721
5055
|
const id = await this.enqueue((entries) => {
|
|
@@ -3731,7 +5065,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3731
5065
|
return {
|
|
3732
5066
|
...editOn(path, edited, start, hash),
|
|
3733
5067
|
replace: {
|
|
3734
|
-
old,
|
|
5068
|
+
old: target,
|
|
3735
5069
|
new: replacement
|
|
3736
5070
|
}
|
|
3737
5071
|
};
|
|
@@ -3759,7 +5093,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3759
5093
|
await this.refuse(entry, [{
|
|
3760
5094
|
kind: "edit-rejected",
|
|
3761
5095
|
path,
|
|
3762
|
-
|
|
5096
|
+
saves: true,
|
|
3763
5097
|
message: (saved) => `${path}: the Edit could not be replayed on the latest version (${response.message}); the edited text was saved to ${saved} and the file now has the latest version. Read it and edit again`
|
|
3764
5098
|
}]);
|
|
3765
5099
|
if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
|
|
@@ -3826,9 +5160,10 @@ var SyncClient = class extends EventEmitter {
|
|
|
3826
5160
|
/**
|
|
3827
5161
|
* Compare every synced file's size, modification time and inode with the last scan and reconcile what changed,
|
|
3828
5162
|
* plus every file expected on disk that is gone. A file modified in the last moment may still be being written,
|
|
3829
|
-
* so it waits for the next scan unless `immediate`. `baseline` only records what is there.
|
|
5163
|
+
* so it waits for the next scan unless `immediate`. `baseline` only records what is there. An immediate scan waits for
|
|
5164
|
+
* the reconciles it started unless `settle` is false, for a caller that waits for them itself (flushTool).
|
|
3830
5165
|
*/
|
|
3831
|
-
async scan(immediate, baseline = false) {
|
|
5166
|
+
async scan(immediate, baseline = false, settle = immediate) {
|
|
3832
5167
|
const seen = /* @__PURE__ */ new Set();
|
|
3833
5168
|
for (const path of await this.walk()) {
|
|
3834
5169
|
seen.add(path);
|
|
@@ -3845,7 +5180,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3845
5180
|
this.signatures.delete(path);
|
|
3846
5181
|
this.schedule(path);
|
|
3847
5182
|
}
|
|
3848
|
-
if (
|
|
5183
|
+
if (settle) await Promise.allSettled(this.chains.values());
|
|
3849
5184
|
}
|
|
3850
5185
|
/**
|
|
3851
5186
|
* Watch the directories that may hold synced files, and local/sync/ for the outbox. The client's own files in the
|
|
@@ -3930,7 +5265,7 @@ var SyncClient = class extends EventEmitter {
|
|
|
3930
5265
|
const name = `${path.split("/").join("__")}.${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.${Math.random().toString(36).slice(2, 6)}`;
|
|
3931
5266
|
return relative(this.paths.data, join(this.paths.conflicts, name)).split(sep).join("/");
|
|
3932
5267
|
}
|
|
3933
|
-
/** Save `text
|
|
5268
|
+
/** Save `text`, or a binary file's bytes, as `saved`, relative to .mainflow/; a text saved there already is kept. */
|
|
3934
5269
|
async saveText(saved, text) {
|
|
3935
5270
|
await mkdir(this.paths.conflicts, { recursive: true });
|
|
3936
5271
|
await writeFile(join(this.paths.data, saved), text, { flag: "wx" }).catch((error) => {
|
|
@@ -3942,16 +5277,30 @@ var SyncClient = class extends EventEmitter {
|
|
|
3942
5277
|
await this.saveText(saved, text);
|
|
3943
5278
|
return saved;
|
|
3944
5279
|
}
|
|
5280
|
+
/** Save the content of sha256 `hash` that blobs/ keeps as `saved`, relative to .mainflow/; one saved there already is kept. */
|
|
5281
|
+
async saveStaged(saved, hash) {
|
|
5282
|
+
await mkdir(this.paths.conflicts, { recursive: true });
|
|
5283
|
+
try {
|
|
5284
|
+
copyFileSync(join(this.paths.blobs, hash), join(this.paths.data, saved), COPYFILE_EXCL);
|
|
5285
|
+
} catch (error) {
|
|
5286
|
+
if (error.code !== "EEXIST") throw error;
|
|
5287
|
+
}
|
|
5288
|
+
}
|
|
3945
5289
|
saveKnown() {
|
|
3946
5290
|
if (this.knownTimer) return;
|
|
3947
5291
|
this.knownTimer = setTimeout(() => this.saveKnownNow(), 200);
|
|
3948
5292
|
}
|
|
5293
|
+
/** What known.json names: each confirmed version, a binary file's sha256 with BLOB_PREFIX. */
|
|
5294
|
+
knownFiles() {
|
|
5295
|
+
const files = {};
|
|
5296
|
+
for (const [path, disk] of this.confirmed) if (disk) files[path] = disk.blob ? { hash: `${BLOB_PREFIX}${disk.hash}` } : disk;
|
|
5297
|
+
return files;
|
|
5298
|
+
}
|
|
3949
5299
|
saveKnownNow(final = false) {
|
|
3950
5300
|
if (this.abandoned && !final) return;
|
|
3951
5301
|
clearTimeout(this.knownTimer);
|
|
3952
5302
|
this.knownTimer = void 0;
|
|
3953
|
-
const files =
|
|
3954
|
-
for (const [path, disk] of this.confirmed) if (disk) files[path] = disk;
|
|
5303
|
+
const files = this.knownFiles();
|
|
3955
5304
|
try {
|
|
3956
5305
|
underLock(this.root, () => writeKnown(this.root, files));
|
|
3957
5306
|
} catch (error) {
|
|
@@ -4049,6 +5398,26 @@ function forgetDaemon(root, gone) {
|
|
|
4049
5398
|
rmSync(syncPaths(root).daemon, { force: true });
|
|
4050
5399
|
return true;
|
|
4051
5400
|
}
|
|
5401
|
+
/** The Edit a request's body names, its file and its one replacement, or null. */
|
|
5402
|
+
function editIn(body) {
|
|
5403
|
+
const edit = body.edit;
|
|
5404
|
+
if (typeof edit?.path !== "string" || typeof edit.old !== "string" || typeof edit.new !== "string") return null;
|
|
5405
|
+
return {
|
|
5406
|
+
path: edit.path,
|
|
5407
|
+
old: edit.old,
|
|
5408
|
+
new: edit.new
|
|
5409
|
+
};
|
|
5410
|
+
}
|
|
5411
|
+
/** The tool call a request's body names, its session at least, or undefined. */
|
|
5412
|
+
function callIn(body) {
|
|
5413
|
+
const call = body.call;
|
|
5414
|
+
if (typeof call?.session !== "string" || call.session === "") return void 0;
|
|
5415
|
+
return {
|
|
5416
|
+
session: call.session,
|
|
5417
|
+
...typeof call.agent === "string" ? { agent: call.agent } : {},
|
|
5418
|
+
...typeof call.toolUseId === "string" ? { toolUseId: call.toolUseId } : {}
|
|
5419
|
+
};
|
|
5420
|
+
}
|
|
4052
5421
|
/** Serve `client` to this machine's mainflow commands until closed. */
|
|
4053
5422
|
async function serveIpc(root, client) {
|
|
4054
5423
|
const secret = randomBytes(16).toString("hex");
|
|
@@ -4070,8 +5439,14 @@ async function serveIpc(root, client) {
|
|
|
4070
5439
|
return client.outboxChanged();
|
|
4071
5440
|
}
|
|
4072
5441
|
if (req.url === "/flush") {
|
|
4073
|
-
const edit = body
|
|
4074
|
-
|
|
5442
|
+
const edit = editIn(body);
|
|
5443
|
+
const call = callIn(body);
|
|
5444
|
+
return reply(200, edit ? await client.flushEdit(edit, void 0, call) : body.tool === true ? await client.flushTool(typeof body.path === "string" ? body.path : void 0, void 0, call) : await client.flush());
|
|
5445
|
+
}
|
|
5446
|
+
if (req.url === "/announce") {
|
|
5447
|
+
const edit = editIn(body);
|
|
5448
|
+
if (edit) client.announceEdit(edit, callIn(body));
|
|
5449
|
+
return reply(200, {});
|
|
4075
5450
|
}
|
|
4076
5451
|
if (req.url === "/status") return reply(200, {
|
|
4077
5452
|
online: client.online,
|
|
@@ -4142,6 +5517,18 @@ async function startDaemon(options) {
|
|
|
4142
5517
|
}
|
|
4143
5518
|
//#endregion
|
|
4144
5519
|
//#region ../sync/dist/hook.js
|
|
5520
|
+
/**
|
|
5521
|
+
* The tool call `payload` is of, as the sync client tells sessions apart: the session, the subagent in it for a
|
|
5522
|
+
* subagent's call (agent_id), and the call's id (tool_use_id), which both hooks of one call name. None without a session.
|
|
5523
|
+
*/
|
|
5524
|
+
function toolCallOf(payload) {
|
|
5525
|
+
if (typeof payload.session_id !== "string" || payload.session_id === "") return void 0;
|
|
5526
|
+
return {
|
|
5527
|
+
session: payload.session_id,
|
|
5528
|
+
...typeof payload.agent_id === "string" && payload.agent_id !== "" ? { agent: payload.agent_id } : {},
|
|
5529
|
+
...typeof payload.tool_use_id === "string" && payload.tool_use_id !== "" ? { toolUseId: payload.tool_use_id } : {}
|
|
5530
|
+
};
|
|
5531
|
+
}
|
|
4145
5532
|
/** Tools that write the one file named in file_path. */
|
|
4146
5533
|
const FILE_TOOLS = /* @__PURE__ */ new Set([
|
|
4147
5534
|
"Edit",
|
|
@@ -4166,26 +5553,37 @@ function hubPath(root, payload) {
|
|
|
4166
5553
|
return path;
|
|
4167
5554
|
}
|
|
4168
5555
|
/**
|
|
5556
|
+
* The one replacement an Edit of `path` makes, as both its hooks name it, the PreToolUse one announcing it and the
|
|
5557
|
+
* PostToolUse one reporting it; null for an Edit of every occurrence, which the sync client does not replay.
|
|
5558
|
+
*/
|
|
5559
|
+
function editOf(path, payload) {
|
|
5560
|
+
const input = payload.tool_input ?? {};
|
|
5561
|
+
if (payload.tool_name !== "Edit" || input.replace_all === true || typeof input.old_string !== "string" || typeof input.new_string !== "string") return null;
|
|
5562
|
+
return {
|
|
5563
|
+
path,
|
|
5564
|
+
old: input.old_string,
|
|
5565
|
+
new: input.new_string
|
|
5566
|
+
};
|
|
5567
|
+
}
|
|
5568
|
+
/**
|
|
4169
5569
|
* What to flush after the tool in `payload` ran in the hub at `root`. A file tool outside the hub's synced files
|
|
4170
|
-
* needs nothing; an Edit of one replacement is replayed on the latest version;
|
|
4171
|
-
* command that may have written any file, rescans every file.
|
|
5570
|
+
* needs nothing; an Edit of one replacement is replayed on the latest version; another file tool names the file it
|
|
5571
|
+
* wrote; anything else, such as a Bash command that may have written any file, rescans every file and names none.
|
|
4172
5572
|
*/
|
|
4173
5573
|
function planFlush(root, payload) {
|
|
4174
5574
|
const tool = payload.tool_name ?? "";
|
|
4175
5575
|
if (!FILE_TOOLS.has(tool)) return { skip: false };
|
|
4176
|
-
|
|
4177
|
-
if (typeof input.file_path !== "string") return { skip: false };
|
|
5576
|
+
if (typeof (payload.tool_input ?? {}).file_path !== "string") return { skip: false };
|
|
4178
5577
|
const path = hubPath(root, payload);
|
|
4179
5578
|
if (path === null || !isSynced(path)) return { skip: true };
|
|
4180
|
-
|
|
5579
|
+
const edit = editOf(path, payload);
|
|
5580
|
+
return edit ? {
|
|
4181
5581
|
skip: false,
|
|
4182
|
-
edit
|
|
4183
|
-
|
|
4184
|
-
|
|
4185
|
-
|
|
4186
|
-
}
|
|
5582
|
+
edit
|
|
5583
|
+
} : {
|
|
5584
|
+
skip: false,
|
|
5585
|
+
path
|
|
4187
5586
|
};
|
|
4188
|
-
return { skip: false };
|
|
4189
5587
|
}
|
|
4190
5588
|
/** The PostToolUse answer: block with the problems as the reason, or nothing when the change went through as written. */
|
|
4191
5589
|
function hookAnswer(result) {
|
|
@@ -4214,6 +5612,17 @@ function planFetch(root, payload) {
|
|
|
4214
5612
|
write: tool !== "Read"
|
|
4215
5613
|
};
|
|
4216
5614
|
}
|
|
5615
|
+
/**
|
|
5616
|
+
* The Edit to announce to the sync client before it runs in the hub at `root`: one replacement of a synced file, whose
|
|
5617
|
+
* change then waits for the Edit's PostToolUse report (docs/sync.md "等 Edit 的回報"). Null for anything else, which
|
|
5618
|
+
* announces nothing: a change no Edit announced is sent at once.
|
|
5619
|
+
*/
|
|
5620
|
+
function planAnnounce(root, payload) {
|
|
5621
|
+
const path = hubPath(root, payload);
|
|
5622
|
+
if (path === null || !isSynced(path)) return null;
|
|
5623
|
+
const edit = editOf(path, payload);
|
|
5624
|
+
return edit && edit.old !== "" ? edit : null;
|
|
5625
|
+
}
|
|
4217
5626
|
/** The PreToolUse answer that stops the tool, with `reason` for the agent. */
|
|
4218
5627
|
const fetchDenial = (reason) => JSON.stringify({ hookSpecificOutput: {
|
|
4219
5628
|
hookEventName: "PreToolUse",
|
|
@@ -4493,4 +5902,4 @@ function hubWriter(root, env = process.env, options = {}) {
|
|
|
4493
5902
|
};
|
|
4494
5903
|
}
|
|
4495
5904
|
//#endregion
|
|
4496
|
-
export {
|
|
5905
|
+
export { syncPaths as A, ENV as C, SETTINGS_FILE as D, syncSettings as E, SYNC_DIR as O, ServerError as S, hubName as T, readDaemon as _, remoteMode as a, oversizedFiles as b, planAnnounce as c, toolCallOf as d, startDaemon as f, probeDaemon as g, callDaemon as h, queuedCount as i, isSynced as k, planFetch as l, DaemonUnavailable as m, hubWriter as n, fetchDenial as o, DaemonRunning as p, writtenLines as r, hookAnswer as s, CONFIRM_MS as t, planFlush as u, silentHint as v, earlierToken as w, ServerApi as x, readRemote as y };
|