mainflow 0.32.1 → 0.33.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,6 +1,6 @@
1
- import { $ as ACCESS_HEADERS, F as renumberFiles, I as renumberPath, L as renumberText, N as applyOffline, P as renumberRequest, Q as nodeStorage, R as scopeOfPath, X as nowMinute, at as requestTime, dt as DATA_DIR, et as MAX_FILE_BYTES, ft as WriteError, it as exceedsFileLimit, mt as splitFrontMatter, nt as SOCKET_MAX_AGE_CLOSE, ot as textBytes, pt as contentHash, rt as comparePaths, tt as ROUTES, ut as ConflictError, z as applyOperation } from "./host-C7GKIjSc.js";
1
+ import { $ as MAX_FILE_BYTES, F as renumberFiles, I as renumberPath, L as renumberText, N as applyOffline, P as renumberRequest, Q as MAX_BLOB_BYTES, R as scopeOfPath, X as ACCESS_HEADERS, Y as nodeStorage, Z as BLOB_HASH, _t as splitFrontMatter, at as requestTime, et as ROUTES, gt as contentHash, ht as WriteError, it as readsAsText, mt as DATA_DIR, nt as comparePaths, ot as textBytes, pt as ConflictError, q as nowMinute, rt as exceedsFileLimit, tt as SOCKET_MAX_AGE_CLOSE, z as applyOperation } from "./host-CObkJQmr.js";
2
2
  import { createHash, randomBytes } from "node:crypto";
3
- import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, watch, writeFileSync } from "node:fs";
3
+ import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, createReadStream, createWriteStream, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readSync, readdirSync, renameSync, rmSync, statSync, watch, writeFileSync } from "node:fs";
4
4
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
5
5
  import { fileURLToPath } from "node:url";
6
6
  import { execFileSync, spawn, spawnSync } from "node:child_process";
@@ -9,8 +9,11 @@ import { homedir } from "node:os";
9
9
  import { createBrotliDecompress, createGunzip, createInflate, createZstdDecompress } from "node:zlib";
10
10
  import { Agent, createServer, request } from "node:http";
11
11
  import { Agent as Agent$1, request as request$1 } from "node:https";
12
+ import { Readable, Transform } from "node:stream";
13
+ import { pipeline } from "node:stream/promises";
12
14
  import { EventEmitter } from "node:events";
13
- import { appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
15
+ import { COPYFILE_EXCL } from "node:constants";
16
+ import { appendFile, mkdir, open, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
14
17
  import { diff3Merge } from "node-diff3";
15
18
  import ignore from "ignore";
16
19
  //#region ../sync/dist/paths.js
@@ -27,10 +30,17 @@ const syncPaths = (root) => {
27
30
  tmp: join(own, "tmp"),
28
31
  conflicts: join(own, "conflicts"),
29
32
  known: join(own, "known.json"),
30
- /** The files only on the server, the artifacts of closed mainlines not downloaded here, by their content hash */
33
+ /** The files only on the server, the artifacts of closed mainlines not downloaded here, by their content hash: a binary file's sha256 */
31
34
  remote: join(own, "remote.json"),
32
35
  /** The text of each version known.json names, in a file named by its content hash */
33
36
  knownTexts: join(own, "known"),
37
+ /** The content of each binary file a write in the outbox uploads, in a file named by its sha256 */
38
+ blobs: join(own, "blobs"),
39
+ /**
40
+ * Present while a hub taken without known.json has had no snapshot that named binary files, as when the first probe
41
+ * failed: a binary file such a snapshot names removed was removed on the server since the copy
42
+ */
43
+ copyBlobs: join(own, "copy-blobs"),
34
44
  queue: join(own, "queue.json"),
35
45
  notices: join(own, "notices.jsonl"),
36
46
  daemon: join(own, "daemon.json"),
@@ -140,6 +150,13 @@ function earlierToken(root, env = process.env) {
140
150
  }
141
151
  //#endregion
142
152
  //#region ../sync/dist/api.js
153
+ /** The sha256 of `bytes` in lowercase hex, as a binary file is named (BLOB_HASH). */
154
+ const sha256 = (bytes) => createHash("sha256").update(bytes).digest("hex");
155
+ /**
156
+ * The sha256 the probe of the binary files' route names (blobRoute): not the sha256 of the empty content it sends, so a
157
+ * server with the route refuses it and keeps nothing.
158
+ */
159
+ const PROBE_HASH = "0".repeat(64);
143
160
  /**
144
161
  * The server could not be reached, or the connection broke. `sent` says whether the request may have reached the
145
162
  * server: a refused connection did not, a reset or timeout after sending may have, so its result is unknown.
@@ -165,6 +182,19 @@ var ServerError = class extends Error {
165
182
  };
166
183
  /** Cloudflare Access refused the request: no service token, a wrong or revoked one, or a policy that does not admit it. */
167
184
  var AccessRefused = class extends ServerError {};
185
+ /** Content downloaded as one sha256 turned out to be of another (downloadBlob): damaged on its way, or by the server. */
186
+ var BlobMismatch = class extends Error {};
187
+ /**
188
+ * The downloaded content could not be written here (downloadBlob), as on a full disk (ENOSPC): an error of this machine,
189
+ * not of the server or the network. `code` is the file system's error code.
190
+ */
191
+ var LocalWriteError = class extends Error {
192
+ code;
193
+ constructor(message, code) {
194
+ super(message);
195
+ this.code = code;
196
+ }
197
+ };
168
198
  /** Error codes of a connection that never carried the request. */
169
199
  const NOT_SENT = /* @__PURE__ */ new Set([
170
200
  "ECONNREFUSED",
@@ -198,6 +228,16 @@ const ACCEPT_ENCODING = Object.keys(DECODERS).join(", ");
198
228
  const causeCode = (error) => {
199
229
  for (let cause = error; cause; cause = cause.cause) if (typeof cause.code === "string") return cause.code;
200
230
  };
231
+ /** What an answer's body is once decompressed, as Content-Encoding says; throws for an encoding this client cannot read. */
232
+ function decoded(res) {
233
+ const encoding = String(res.headers["content-encoding"] ?? "identity").toLowerCase();
234
+ const decode = DECODERS[encoding];
235
+ if (!decode && encoding !== "identity") {
236
+ res.resume();
237
+ throw new Error(`the answer came in content-encoding ${encoding}, which this client cannot read`);
238
+ }
239
+ return decode ? res.pipe(decode()) : res;
240
+ }
201
241
  var ServerApi = class {
202
242
  settings;
203
243
  timeoutMs;
@@ -208,6 +248,11 @@ var ServerApi = class {
208
248
  agent;
209
249
  /** The connections snapshots go on, apart from the writes', so a snapshot never waits behind a write or holds one up */
210
250
  reads;
251
+ /**
252
+ * The connections binary files' contents go up and down on, apart from the writes' and the snapshots', so a large file
253
+ * holds neither up; a few at a time, as the files of a snapshot are downloaded side by side.
254
+ */
255
+ transfers;
211
256
  /** Requests on `agent` not answered yet */
212
257
  busy = 0;
213
258
  transfer = {
@@ -232,6 +277,11 @@ var ServerApi = class {
232
277
  maxSockets: 2,
233
278
  maxFreeSockets: 1
234
279
  });
280
+ this.transfers = new Agent$2({
281
+ keepAlive: true,
282
+ maxSockets: 4,
283
+ maxFreeSockets: 1
284
+ });
235
285
  }
236
286
  /** The service token's headers, which every request and the WebSocket's upgrade carry. Never logged. */
237
287
  get access() {
@@ -240,6 +290,18 @@ var ServerApi = class {
240
290
  [ACCESS_HEADERS.clientSecret]: this.settings.clientSecret
241
291
  };
242
292
  }
293
+ /** A failure to reach the server or of the connection, as the Unreachable it is. */
294
+ unreachable(error) {
295
+ const code = causeCode(error);
296
+ const detail = [code, error.cause?.message ?? error.message].filter(Boolean).join(": ");
297
+ const sent = !(code && NOT_SENT.has(code));
298
+ return new Unreachable(`cannot reach the sync server at ${this.settings.url}${detail ? ` (${detail})` : ""}`, sent, sent && STALE.has(code ?? ""));
299
+ }
300
+ /** Access answers for the server, with a page rather than JSON: 401 or 403, or a redirect to its login page. */
301
+ refusedByAccess(status, location, url) {
302
+ if (status !== 401 && status !== 403 && !(status >= 300 && status < 400)) return null;
303
+ return new AccessRefused(`Cloudflare Access in front of the sync server refused the request (HTTP ${status}${location ? ` to ${new URL(location, url).origin}` : ""}); check the service token in ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}`, status);
304
+ }
243
305
  async call(method, route, body, connection) {
244
306
  const url = `${this.settings.url}${route}`;
245
307
  const text = JSON.stringify(body);
@@ -247,15 +309,10 @@ var ServerApi = class {
247
309
  try {
248
310
  answer = connection === "read" ? await this.read(method, url, text) : await this.write(method, url, text, connection === "fresh");
249
311
  } catch (error) {
250
- const code = causeCode(error);
251
- const detail = [code, error.cause?.message ?? error.message].filter(Boolean).join(": ");
252
- const sent = !(code && NOT_SENT.has(code));
253
- throw new Unreachable(`cannot reach the sync server at ${this.settings.url}${detail ? ` (${detail})` : ""}`, sent, sent && STALE.has(code ?? ""));
254
- }
255
- if (answer.status === 401 || answer.status === 403 || answer.status >= 300 && answer.status < 400) {
256
- const to = answer.location ? ` to ${new URL(answer.location, url).origin}` : "";
257
- throw new AccessRefused(`Cloudflare Access in front of the sync server refused the request (HTTP ${answer.status}${to}); check the service token in ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}`, answer.status);
312
+ throw this.unreachable(error);
258
313
  }
314
+ const access = this.refusedByAccess(answer.status, answer.location, url);
315
+ if (access) throw access;
259
316
  let parsed;
260
317
  try {
261
318
  parsed = JSON.parse(answer.text);
@@ -293,50 +350,171 @@ var ServerApi = class {
293
350
  write(method, url, body, fresh) {
294
351
  return this.exchange(method, url, body, fresh ? false : this.agent, this.timeoutMs);
295
352
  }
296
- exchange(method, url, body, agent, timeoutMs) {
353
+ /**
354
+ * Send one request with the service token and resolve with the answer as it starts to come. `body` is a JSON text,
355
+ * the raw bytes of a stream, or nothing; `headers` add to the token's and say what the body is.
356
+ */
357
+ open(method, url, body, headers, agent, timeoutMs) {
297
358
  const send = url.startsWith("https:") ? request$1 : request;
298
- const headers = {
299
- ...this.access,
300
- "accept-encoding": ACCEPT_ENCODING
301
- };
302
- if (body !== void 0) Object.assign(headers, {
303
- "content-type": "application/json",
304
- "content-length": String(Buffer.byteLength(body))
305
- });
306
- if (agent === this.agent) this.busy++;
307
359
  return new Promise((resolve, reject) => {
308
360
  const req = send(url, {
309
361
  method,
310
362
  agent,
311
- headers,
363
+ headers: {
364
+ ...this.access,
365
+ "accept-encoding": ACCEPT_ENCODING,
366
+ ...headers
367
+ },
312
368
  timeout: timeoutMs
313
- }, (res) => {
369
+ }, resolve);
370
+ req.on("timeout", () => req.destroy(Object.assign(/* @__PURE__ */ new Error("no answer in time"), { code: req.socket?.connecting === false ? "ETIMEDOUT" : "ECONNECT_TIMEOUT" })));
371
+ req.on("error", reject);
372
+ if (body instanceof Readable) {
373
+ body.on("error", (error) => req.destroy(error));
374
+ body.pipe(req);
375
+ } else req.end(body);
376
+ });
377
+ }
378
+ /** One request and its whole answer as text. */
379
+ async exchange(method, url, body, agent, timeoutMs, headers) {
380
+ const kind = typeof body === "string" ? {
381
+ "content-type": "application/json",
382
+ "content-length": String(Buffer.byteLength(body))
383
+ } : {};
384
+ if (agent === this.agent) this.busy++;
385
+ try {
386
+ const res = await this.open(method, url, body, {
387
+ ...kind,
388
+ ...headers
389
+ }, agent, timeoutMs);
390
+ return await new Promise((resolve, reject) => {
314
391
  let wire = 0;
315
392
  res.on("data", (chunk) => wire += chunk.length);
316
- const encoding = String(res.headers["content-encoding"] ?? "identity").toLowerCase();
317
- const decode = DECODERS[encoding];
318
- if (!decode && encoding !== "identity") {
319
- res.resume();
320
- return reject(/* @__PURE__ */ new Error(`the answer came in content-encoding ${encoding}, which this client cannot read`));
393
+ res.on("error", reject);
394
+ let stream;
395
+ try {
396
+ stream = decoded(res);
397
+ } catch (error) {
398
+ return reject(error);
321
399
  }
322
- const stream = decode ? res.pipe(decode()) : res;
323
400
  const chunks = [];
324
401
  stream.on("data", (chunk) => chunks.push(chunk));
325
402
  stream.on("end", () => resolve({
326
403
  status: res.statusCode ?? 0,
327
404
  text: Buffer.concat(chunks).toString("utf8"),
328
405
  location: res.headers.location,
406
+ retryAfter: res.headers["retry-after"],
329
407
  wire
330
408
  }));
331
409
  stream.on("error", reject);
332
- res.on("error", reject);
333
410
  });
334
- req.on("timeout", () => req.destroy(Object.assign(/* @__PURE__ */ new Error("no answer in time"), { code: req.socket?.connecting === false ? "ETIMEDOUT" : "ECONNECT_TIMEOUT" })));
335
- req.on("error", reject);
336
- req.end(body);
337
- }).finally(() => {
411
+ } finally {
338
412
  if (agent === this.agent) this.busy--;
339
- });
413
+ }
414
+ }
415
+ /**
416
+ * Whether the server has the route of binary files' contents (ROUTES.blob), as a server from before them does not:
417
+ * the server's version (GET /version) tells only its protocol, which binary files left as it was, so the route is
418
+ * asked. The probe uploads empty content as PROBE_HASH, which is not its sha256: a server with the route refuses it
419
+ * and keeps nothing, with 400 `hash-mismatch`, or 4xx for any other reason it refuses an upload; one without the route
420
+ * answers 404 `not-found`, from the Worker's router itself. A 5xx comes from a server that got as far as the route, as
421
+ * one whose R2 is failing: it keeps binary files, and the uploads wait until it answers them. Asked once per connection,
422
+ * before its snapshot, since a snapshot names the binary files only to a server that keeps them. It goes on a
423
+ * connection of the reads, so one the server closed meanwhile is asked again.
424
+ */
425
+ async blobRoute() {
426
+ const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, PROBE_HASH)}`;
427
+ let answer;
428
+ try {
429
+ answer = await this.read("PUT", url, "");
430
+ } catch (error) {
431
+ throw this.unreachable(error);
432
+ }
433
+ const access = this.refusedByAccess(answer.status, answer.location, url);
434
+ if (access) throw access;
435
+ return answer.status !== 404;
436
+ }
437
+ /**
438
+ * Upload the content of a binary file, the `size` bytes of `file`, as sha256 `hash` (ROUTES.blob), on a connection of
439
+ * the transfers. The server out of reach, or failing, throws Unreachable; Access refusing it throws AccessRefused.
440
+ */
441
+ async putBlob(hash, file, size) {
442
+ const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, hash)}`;
443
+ let answer;
444
+ try {
445
+ answer = await this.exchange("PUT", url, createReadStream(file), this.transfers, this.timeoutMs, {
446
+ "content-type": "application/octet-stream",
447
+ "content-length": String(size)
448
+ });
449
+ } catch (error) {
450
+ throw this.unreachable(error);
451
+ }
452
+ const access = this.refusedByAccess(answer.status, answer.location, url);
453
+ if (access) throw access;
454
+ let body;
455
+ try {
456
+ body = JSON.parse(answer.text);
457
+ } catch {
458
+ throw new Unreachable(`the sync server answered ${answer.status} without a readable body`, true);
459
+ }
460
+ if (answer.status === 503 && body.reason === "blob-deleting") return {
461
+ ok: false,
462
+ retryAfter: Math.max(1, Number(answer.retryAfter) || 1)
463
+ };
464
+ if (answer.status >= 500) throw new Unreachable(`the sync server failed (${answer.status})`, true);
465
+ if (body.ok === true) return { ok: true };
466
+ const message = body.message ?? `HTTP ${answer.status}`;
467
+ if (body.reason === "hash-mismatch" || body.reason === "too-large") return {
468
+ ok: false,
469
+ refused: body.reason,
470
+ message
471
+ };
472
+ return {
473
+ ok: false,
474
+ refused: answer.status === 404 ? "unsupported" : "other",
475
+ message
476
+ };
477
+ }
478
+ /**
479
+ * Download the content of sha256 `hash` into `file`, checking it is of that hash as it comes: true once it is there,
480
+ * false when the server holds no such content. Content of another hash throws BlobMismatch, and `file` that cannot be
481
+ * written here throws LocalWriteError; both leave `file` to the caller to remove, and so do the server out of reach
482
+ * (Unreachable) and Access refusing it (AccessRefused).
483
+ */
484
+ async downloadBlob(hash, file) {
485
+ const url = `${this.settings.url}${ROUTES.blob(this.settings.hub, hash)}`;
486
+ let res;
487
+ try {
488
+ res = await this.open("GET", url, void 0, {}, this.transfers, this.timeoutMs);
489
+ } catch (error) {
490
+ throw this.unreachable(error);
491
+ }
492
+ const status = res.statusCode ?? 0;
493
+ const access = this.refusedByAccess(status, res.headers.location, url);
494
+ if (status !== 200) {
495
+ res.resume();
496
+ if (access) throw access;
497
+ if (status === 404) return false;
498
+ if (status >= 500) throw new Unreachable(`the sync server failed (${status})`, true);
499
+ throw new ServerError(`the sync server refused the download of ${hash} (HTTP ${status})`, status);
500
+ }
501
+ const digest = createHash("sha256");
502
+ const tap = new Transform({ transform(chunk, _encoding, done) {
503
+ digest.update(chunk);
504
+ done(null, chunk);
505
+ } });
506
+ const out = createWriteStream(file);
507
+ let local;
508
+ out.once("error", (error) => local = error);
509
+ try {
510
+ await pipeline(decoded(res), tap, out);
511
+ } catch (error) {
512
+ if (local !== void 0) throw new LocalWriteError(`could not write the download here: ${local.message}`, local.code);
513
+ throw this.unreachable(error);
514
+ }
515
+ const got = digest.digest("hex");
516
+ if (got !== hash) throw new BlobMismatch(`the content downloaded as sha256 ${hash} is of ${got}`);
517
+ return true;
340
518
  }
341
519
  /**
342
520
  * Keep the write connection open: a request that needs no hub, on it, unless a request is on it already. Opens it
@@ -352,6 +530,7 @@ var ServerApi = class {
352
530
  close() {
353
531
  this.agent.destroy();
354
532
  this.reads.destroy();
533
+ this.transfers.destroy();
355
534
  }
356
535
  /** Send one request; the response is the hub's answer, applied or refused. */
357
536
  async send(request, options = {}) {
@@ -377,7 +556,8 @@ var ServerApi = class {
377
556
  * is below the batch before, as when the server's storage was replaced meanwhile, starts the snapshot over. A path of
378
557
  * `have` the server never held a file at comes as `unknown` rather than `removed`, from a server that tells them apart.
379
558
  * `listOnly` asks for the files under those prefixes as `listed`, without their text. A part of the hub alone is taken
380
- * from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of.
559
+ * from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of. `blobs` takes
560
+ * the binary files too, each that differs from `have` as `blob`; `have` names a binary file by its sha256 then.
381
561
  */
382
562
  async snapshot(have = /* @__PURE__ */ new Map(), options = {}) {
383
563
  const paths = [...have.keys()].sort(comparePaths);
@@ -394,7 +574,8 @@ var ServerApi = class {
394
574
  have: range,
395
575
  unknown: true,
396
576
  ...after === void 0 ? {} : { after },
397
- ...options.listOnly?.length ? { listOnly: options.listOnly } : {}
577
+ ...options.listOnly?.length ? { listOnly: options.listOnly } : {},
578
+ ...options.blobs ? { blobs: true } : {}
398
579
  });
399
580
  if (parts.length > 0 && seq < parts.at(-1).seq) break;
400
581
  parts.push({
@@ -431,13 +612,32 @@ var ServerApi = class {
431
612
  }, "fresh");
432
613
  return body;
433
614
  }
434
- /** Open the hub's broadcast connection. */
615
+ /**
616
+ * Open the hub's broadcast connection, taking the binary files too (ROUTES.socket). It opens before the client knows
617
+ * whether the server keeps binary files (blobRoute), and a server from before them reads nothing of `blobs`.
618
+ */
435
619
  socket() {
436
- const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=3`;
620
+ const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=3&blobs=true`;
437
621
  return new WebSocket(url, { headers: this.access });
438
622
  }
439
623
  };
440
624
  //#endregion
625
+ //#region ../sync/dist/edit.js
626
+ /**
627
+ * The text an Edit of `edit` replaces in `text`: old_string, with the line break after it when new_string is empty and
628
+ * old_string ends in none (Claude Code 2.1.285). Null when old_string is empty or not in `text` exactly once, as the
629
+ * Edit then fails.
630
+ */
631
+ function editTarget(text, edit) {
632
+ if (edit.old === "" || text.split(edit.old).length !== 2) return null;
633
+ return edit.new === "" && !edit.old.endsWith("\n") && text.includes(`${edit.old}\n`) ? `${edit.old}\n` : edit.old;
634
+ }
635
+ /** `text` with the Edit of `edit` made as Claude Code makes it, or null when the Edit fails on it. */
636
+ function applyEdit(text, edit) {
637
+ const target = editTarget(text, edit);
638
+ return target === null ? null : text.replace(target, () => edit.new);
639
+ }
640
+ //#endregion
441
641
  //#region ../sync/dist/lines.js
442
642
  const split = (text) => text.split("\n");
443
643
  /**
@@ -694,6 +894,8 @@ function oversizedFiles(root) {
694
894
  }
695
895
  //#endregion
696
896
  //#region ../sync/dist/state.js
897
+ /** The requestId an outbox entry goes under: its id, with the number of the try for a binary file's write sent again. */
898
+ const requestIdOf = (entry) => entry.kind === "blob" && entry.retry ? `${entry.id}-${entry.retry}` : entry.id;
697
899
  /** Whether `entry` waits for the server: not one it refused. */
698
900
  const waiting = (entry) => entry.refused === void 0;
699
901
  /** A new entry's id. */
@@ -752,6 +954,8 @@ function writeJson(file, value) {
752
954
  renameSync(temp, file);
753
955
  }
754
956
  const readKnown = (root) => readJson(syncPaths(root).known, {}).files ?? {};
957
+ /** What known.json writes before a binary file's sha256, apart from a text's content hash. */
958
+ const BLOB_PREFIX = "blob:";
755
959
  /** The text of the version known.json names by `hash`; undefined when it was not kept, as by a sync client before known/. */
756
960
  function readKnownText(root, hash) {
757
961
  try {
@@ -765,7 +969,7 @@ function readKnownText(root, hash) {
765
969
  * Write known.json, and into known/ the text of each version it names that is not
766
970
  * there yet; the texts of versions it no longer names are removed. A version whose text the client does not have keeps
767
971
  * the text known/ holds, if any. The new texts go first, then known.json, then the removals, so a client killed at any
768
- * step leaves a known.json whose every text is in known/.
972
+ * step leaves a known.json whose every text is in known/. A binary file is given as its sha256 with BLOB_PREFIX, and no text.
769
973
  */
770
974
  function writeKnown(root, files) {
771
975
  const dir = syncPaths(root).knownTexts;
@@ -1108,6 +1312,13 @@ function exists(file) {
1108
1312
  return false;
1109
1313
  }
1110
1314
  }
1315
+ const isBlob = (file) => typeof file === "object" && file !== null && file.kind === "blob";
1316
+ /** The hash the server names a file of its by: a text's content hash, a binary file's sha256. */
1317
+ const serverHash = (file) => typeof file === "string" ? contentHash(file) : file.hash;
1318
+ /** Whether two tool calls are of one session, the same subagent in it or both of the session itself. */
1319
+ const sameCaller = (a, b) => a !== void 0 && b !== void 0 && a.session === b.session && a.agent === b.agent;
1320
+ /** Whether `edit` makes `to` of `from`, as the Edit tool makes it (applyEdit). */
1321
+ const replaces = (edit, from, to) => applyEdit(from, edit) === to;
1111
1322
  const HISTORY = 20;
1112
1323
  /** The statuses of a closed mainline, whose artifacts stay on the server until they are used */
1113
1324
  const CLOSED = /* @__PURE__ */ new Set(["done", "dropped"]);
@@ -1125,6 +1336,26 @@ const SETTLE_LIMIT_MS = 1e3;
1125
1336
  * ones before the last are each followed by a pause before the next
1126
1337
  */
1127
1338
  const STALLED_SNAPSHOTS = 4;
1339
+ /** How long after a failed probe of binary files (probeBlobs) the snapshot is taken again, at first and at most */
1340
+ const PROBE_RETRY_MS = 3e4;
1341
+ const PROBE_RETRY_MAX_MS = 6e5;
1342
+ /**
1343
+ * How long an upload of a binary file's content waits after a failure (uploadEntry): the first two failures are tried
1344
+ * again after BLOB_RETRY_MS, with up to as much again at random; from the third on, the wait doubles from
1345
+ * BLOB_BACKOFF_MS up to BLOB_BACKOFF_MAX_MS
1346
+ */
1347
+ const BLOB_RETRY_MS = 1e3;
1348
+ const BLOB_BACKOFF_MS = 2e3;
1349
+ const BLOB_BACKOFF_MAX_MS = 6e5;
1350
+ /** Failures of an upload of the same write in a row before it waits longer each time and the writer is told */
1351
+ const BLOB_FAILURES = 3;
1352
+ /**
1353
+ * How long a download of a binary file's content waits after it failed (download), doubling from DOWNLOAD_BACKOFF_MS up to
1354
+ * BLOB_BACKOFF_MAX_MS. Content that does not match or cannot be written here is tried DOWNLOAD_TRIES times; then the file
1355
+ * is left as it is until the next connection.
1356
+ */
1357
+ const DOWNLOAD_BACKOFF_MS = 2e3;
1358
+ const DOWNLOAD_TRIES = 5;
1128
1359
  /** How long after a read that is not UTF-8 the file is read again before it counts as binary */
1129
1360
  const BINARY_RECHECK_MS = 150;
1130
1361
  /**
@@ -1147,6 +1378,91 @@ const OUTBOX = `${SYNC_DIR}/queue.json`;
1147
1378
  const sleep$1 = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
1148
1379
  const decoder = new TextDecoder("utf-8", { fatal: true });
1149
1380
  const sameLocal = (a, b) => a.kind === b.kind && (a.kind !== "text" || a.text === b.text) && (a.kind !== "binary" || a.hash === b.hash);
1381
+ /** What `bytes` read from a file are: a text when they are UTF-8, a binary file otherwise. */
1382
+ function contentOf(bytes) {
1383
+ try {
1384
+ return {
1385
+ kind: "text",
1386
+ text: decoder.decode(bytes)
1387
+ };
1388
+ } catch {
1389
+ return {
1390
+ kind: "binary",
1391
+ hash: sha256(bytes),
1392
+ size: bytes.length,
1393
+ bytes
1394
+ };
1395
+ }
1396
+ }
1397
+ /** The hash of `content`, as the server names it: a text's content hash, a binary file's sha256. */
1398
+ const hashOfContent = (content) => content.kind === "text" ? contentHash(content.text) : content.hash;
1399
+ /** Whether `content` is what the server holds as `file`. */
1400
+ const holdsServer = (content, file) => typeof file === "string" ? content.kind === "text" && content.text === file : isBlob(file) ? content.kind === "binary" && content.hash === file.hash : false;
1401
+ /**
1402
+ * The sha256 and size of the file at `file`, and whether it is UTF-8, read in chunks of a MiB: a file of up to
1403
+ * MAX_BLOB_BYTES is never held in memory whole.
1404
+ */
1405
+ async function scanFile(file) {
1406
+ const handle = await open(file, "r");
1407
+ try {
1408
+ const digest = createHash("sha256");
1409
+ const utf8 = new TextDecoder("utf-8", { fatal: true });
1410
+ const chunk = Buffer.allocUnsafe(1 << 20);
1411
+ let size = 0;
1412
+ let text = true;
1413
+ for (;;) {
1414
+ const { bytesRead } = await handle.read(chunk, 0, chunk.length, null);
1415
+ if (bytesRead === 0) break;
1416
+ const part = chunk.subarray(0, bytesRead);
1417
+ digest.update(part);
1418
+ size += bytesRead;
1419
+ if (text) try {
1420
+ utf8.decode(part, { stream: true });
1421
+ } catch {
1422
+ text = false;
1423
+ }
1424
+ }
1425
+ if (text) try {
1426
+ utf8.decode();
1427
+ } catch {
1428
+ text = false;
1429
+ }
1430
+ return {
1431
+ hash: digest.digest("hex"),
1432
+ size,
1433
+ text
1434
+ };
1435
+ } finally {
1436
+ await handle.close();
1437
+ }
1438
+ }
1439
+ /** The sha256 of the file at `file`, read in chunks; null when it cannot be read. */
1440
+ function sha256File(file) {
1441
+ let fd;
1442
+ try {
1443
+ fd = openSync(file, "r");
1444
+ } catch {
1445
+ return null;
1446
+ }
1447
+ try {
1448
+ const digest = createHash("sha256");
1449
+ const chunk = Buffer.allocUnsafe(1 << 20);
1450
+ for (let read = readSync(fd, chunk); read > 0; read = readSync(fd, chunk)) digest.update(chunk.subarray(0, read));
1451
+ return digest.digest("hex");
1452
+ } catch {
1453
+ return null;
1454
+ } finally {
1455
+ closeSync(fd);
1456
+ }
1457
+ }
1458
+ /** What the client leaves on disk when a file holds `content`: a text with its hash, or a binary file's sha256. */
1459
+ const diskOf = (content) => content.kind === "text" ? {
1460
+ hash: contentHash(content.text),
1461
+ text: content.text
1462
+ } : {
1463
+ hash: content.hash,
1464
+ blob: true
1465
+ };
1150
1466
  /**
1151
1467
  * The queued edit of `path` to the writer's `text` that started from `start` (src/state.ts): on the server's version
1152
1468
  * that holds a confirmed start, waiting for an unconfirmed one, or, without a start, on `base`, the hash of the version
@@ -1194,7 +1510,7 @@ function heldBase(entry) {
1194
1510
  */
1195
1511
  function textAt(entry, path) {
1196
1512
  if (entry.kind === "op") return entry.written[path];
1197
- if (entry.path !== path) return void 0;
1513
+ if (entry.path !== path || entry.kind === "blob") return void 0;
1198
1514
  if (entry.kind === "remove") return null;
1199
1515
  return entry.text;
1200
1516
  }
@@ -1233,7 +1549,7 @@ function patchRequest(entry) {
1233
1549
  function holdsFiles(parts, held) {
1234
1550
  const missing = /* @__PURE__ */ new Set();
1235
1551
  for (const part of parts) for (const event of part.files) {
1236
- if (event.kind === "file" || event.kind === "listed") return true;
1552
+ if (event.kind === "file" || event.kind === "listed" || event.kind === "blob") return true;
1237
1553
  if (event.kind === "removed" || event.kind === "unknown") missing.add(event.path);
1238
1554
  }
1239
1555
  return [...held.keys()].some((path) => !missing.has(path));
@@ -1245,6 +1561,18 @@ var SyncClient = class extends EventEmitter {
1245
1561
  api;
1246
1562
  problems = [];
1247
1563
  reported = 0;
1564
+ /** The local change of each path waiting for an Edit's report (awaitEdit) */
1565
+ awaitingEdit = /* @__PURE__ */ new Map();
1566
+ /** The Edits announced before they ran (announceEdit) and not reported yet, by path, oldest first */
1567
+ announced = /* @__PURE__ */ new Map();
1568
+ /** Edits reported before the change they made waited, by path, until the report's own replay on the path's chain ran */
1569
+ reportedEdits = /* @__PURE__ */ new Map();
1570
+ /** Flushes under way that send every change at once (flush): no change waits for an Edit's report meanwhile */
1571
+ sendingAll = 0;
1572
+ /** The files the tools reported by flushes under way wrote, by count: a change of them does not wait for an Edit's report */
1573
+ sendingPaths = /* @__PURE__ */ new Map();
1574
+ /** The path the reconcile or step on a path's chain that `work` tracks is about (schedule, withChain) */
1575
+ workPaths = /* @__PURE__ */ new Map();
1248
1576
  /** What each path holds on disk as the client last left it, confirmed or not */
1249
1577
  expected = /* @__PURE__ */ new Map();
1250
1578
  /**
@@ -1263,17 +1591,65 @@ var SyncClient = class extends EventEmitter {
1263
1591
  /** The content hash last worked out for a text the client named in a snapshot request, by path */
1264
1592
  hashes = /* @__PURE__ */ new Map();
1265
1593
  /**
1266
- * The text the last snapshot request named for each path, with what the server made of it: left out, as the same text
1267
- * it holds, or named removed. A file this client never synced, as one in a hub taken with git, that still holds that
1268
- * text holds the server's version, or one the server removed since; not a text written here (reconcile).
1594
+ * The content the last snapshot request named for each path, by its hash, with what the server made of it: left out,
1595
+ * as the same content it holds, or named removed. A file this client never synced, as one in a hub taken with git, that
1596
+ * still holds that content holds the server's version, or one the server removed since; not one written here (reconcile).
1269
1597
  */
1270
1598
  snapshotAnswers = /* @__PURE__ */ new Map();
1271
1599
  /**
1600
+ * Whether the server keeps binary files, as the probe of its route told (ServerApi.blobRoute); null until a connection
1601
+ * asked. Only then does a snapshot name binary files, and the outbox send one: a server from before binary files names
1602
+ * a path of `have` it holds no file at removed, which would remove the binary file here. While it is null, a binary file
1603
+ * waits as it is here, neither sent nor written over: the snapshot after the probe reconciles every file again.
1604
+ */
1605
+ blobSupport = null;
1606
+ /** Whether the current connection asked the server whether it keeps binary files; cleared when a connection opens */
1607
+ probed = false;
1608
+ /** The contents this run uploaded, by sha256: a write of one is sent without uploading it again */
1609
+ uploadedBlobs = /* @__PURE__ */ new Set();
1610
+ /** The binary files the server refused as too large, as `path\0hash`: left as they are here, as larger than MAX_BLOB_BYTES */
1611
+ oversized = /* @__PURE__ */ new Set();
1612
+ /** The contents a download found missing on the server on this connection, by sha256, so a snapshot is reloaded once for each */
1613
+ missingDownloads = /* @__PURE__ */ new Set();
1614
+ /** The failed uploads of each binary file's write in a row this run, by outbox id (uploadEntry) */
1615
+ uploadFailures = /* @__PURE__ */ new Map();
1616
+ /**
1617
+ * Until when each binary file's write waits after its upload failed, by outbox id: meanwhile the writes of other paths
1618
+ * behind it go ahead (sendOutbox)
1619
+ */
1620
+ uploadHolds = /* @__PURE__ */ new Map();
1621
+ /**
1622
+ * When the content of each sha256 the server is deleting may be uploaded again, as its Retry-After said: the outbox is
1623
+ * sent again on any change to queue.json, which would otherwise upload it again at once
1624
+ */
1625
+ uploadAfter = /* @__PURE__ */ new Map();
1626
+ /** The download to try again shortly, after one failed while online: at most one timer for each path */
1627
+ downloadRetries = /* @__PURE__ */ new Map();
1628
+ /**
1629
+ * The failed downloads of each content in a row on this connection, by sha256, for the ones that came of another sha256
1630
+ * or could not be written here (download)
1631
+ */
1632
+ downloadFailures = /* @__PURE__ */ new Map();
1633
+ /** Why the last download of each path failed, until one succeeds: what fetch tells its caller */
1634
+ downloadErrors = /* @__PURE__ */ new Map();
1635
+ /** The paths whose download failed DOWNLOAD_TRIES times in a row, reconciled again after the next snapshot */
1636
+ stalledDownloads = /* @__PURE__ */ new Set();
1637
+ /** The snapshot taken again after a failed probe (probeBlobs), and how long after the next failure it is */
1638
+ probeTimer;
1639
+ probeRetry = PROBE_RETRY_MS;
1640
+ /**
1272
1641
  * Whether this run started without known.json, as in a hub taken with git that never synced here. Only then does the
1273
1642
  * first snapshot's `removed` for a file the client named tell of a file the server removed since the copy: the server
1274
1643
  * names removed every path it does not have, so later, or with known.json, such a file is one created here.
1275
1644
  */
1276
1645
  copied = false;
1646
+ /**
1647
+ * Whether the binary files of a copy (copied) were compared with the server's yet: false until a snapshot that named
1648
+ * binary files ends, which the first snapshot does not when the client does not know yet that the server keeps them.
1649
+ * Until then a binary file that snapshot names removed was removed on the server since the copy, as `copied` tells of
1650
+ * a text in the first snapshot. Kept across runs by copyBlobs, since the first snapshot saves known.json.
1651
+ */
1652
+ blobsCompared = true;
1277
1653
  /** The server has no copy of the hub, which this environment, downloading on demand, does not upload (snapshotNow) */
1278
1654
  uploadWanted = false;
1279
1655
  /** Why the server refused the upload of this environment's copy, as last reported (snapshotNow); null once it holds the hub */
@@ -1425,15 +1801,17 @@ var SyncClient = class extends EventEmitter {
1425
1801
  /**
1426
1802
  * The file at `path` as the server last had it to this client's knowledge: its text, null when the server has no such
1427
1803
  * file, or undefined when the client does not know it, such as before the first snapshot, or for a file only on the
1428
- * server, whose text is not here.
1804
+ * server, whose text is not here, and for a binary file, which has no text.
1429
1805
  */
1430
1806
  serverText(path) {
1431
1807
  if (this.remote.has(path) || this.awaiting.has(path)) return void 0;
1432
- return this.cloud.has(path) ? this.cloud.get(path) : this.synced ? null : void 0;
1808
+ if (!this.cloud.has(path)) return this.synced ? null : void 0;
1809
+ const held = this.cloud.get(path);
1810
+ return isBlob(held) ? void 0 : held;
1433
1811
  }
1434
1812
  /** How many files are only on the server, not downloaded here (onDemand). */
1435
1813
  get remoteCount() {
1436
- return this.remote.size;
1814
+ return this.listedRemote().size;
1437
1815
  }
1438
1816
  /**
1439
1817
  * Whether the sync server has no copy of the hub and this environment, downloading on demand, waits for one that keeps
@@ -1446,6 +1824,7 @@ var SyncClient = class extends EventEmitter {
1446
1824
  async start() {
1447
1825
  await mkdir(this.paths.tmp, { recursive: true });
1448
1826
  await mkdir(this.paths.conflicts, { recursive: true });
1827
+ await mkdir(this.paths.blobs, { recursive: true });
1449
1828
  if (this.opts.swap !== false) {
1450
1829
  const swapper = startSwapper(this.paths.tmp);
1451
1830
  this.swapper = swapper.helper;
@@ -1459,6 +1838,7 @@ var SyncClient = class extends EventEmitter {
1459
1838
  await this.restoreKnown();
1460
1839
  this.restoreRemote();
1461
1840
  await this.whenFree(() => this.syncQueue());
1841
+ await this.whenFree(() => this.locked(() => this.pruneStaged(this.queue)));
1462
1842
  await this.resumeRefusals();
1463
1843
  if (this.opts.watch !== false) this.startWatcher();
1464
1844
  await this.scan(true, true);
@@ -1474,10 +1854,21 @@ var SyncClient = class extends EventEmitter {
1474
1854
  * its write based on that change and undo it, and a change made while no client ran would have no base text to tell
1475
1855
  * the writer's lines from another environment's (layer 2). Without its text, as known.json of an older client, only
1476
1856
  * a file that still holds it gives it. What the outbox wrote, refused writes included, is read after it (restoreEntry).
1857
+ * A binary file's version is its sha256 alone (BLOB_PREFIX): it is no base for a write, which is never merged.
1477
1858
  */
1478
1859
  async restoreKnown() {
1479
1860
  this.copied = !existsSync(this.paths.known);
1861
+ this.blobsCompared = !this.copied && !existsSync(this.paths.copyBlobs);
1480
1862
  for (const [path, hash] of Object.entries(readKnown(this.root))) {
1863
+ if (hash.startsWith("blob:")) {
1864
+ const disk = {
1865
+ hash: hash.slice(5),
1866
+ blob: true
1867
+ };
1868
+ this.expected.set(path, disk);
1869
+ this.confirmed.set(path, disk);
1870
+ continue;
1871
+ }
1481
1872
  const local = await this.readLocal(path);
1482
1873
  const text = local.kind === "text" && contentHash(local.text) === hash ? local.text : readKnownText(this.root, hash);
1483
1874
  const disk = text === void 0 ? { hash } : {
@@ -1535,6 +1926,12 @@ var SyncClient = class extends EventEmitter {
1535
1926
  clearTimeout(this.queueTimer);
1536
1927
  for (const timer of this.watchTimers.values()) clearTimeout(timer);
1537
1928
  this.watchTimers.clear();
1929
+ for (const timer of this.downloadRetries.values()) clearTimeout(timer);
1930
+ this.downloadRetries.clear();
1931
+ clearTimeout(this.probeTimer);
1932
+ for (const wait of this.awaitingEdit.values()) wait.end(null);
1933
+ this.reportedEdits.clear();
1934
+ this.announced.clear();
1538
1935
  this.watcher?.close();
1539
1936
  this.ws?.close();
1540
1937
  const settled = Promise.allSettled([
@@ -1566,14 +1963,19 @@ var SyncClient = class extends EventEmitter {
1566
1963
  this.connect();
1567
1964
  });
1568
1965
  }
1569
- /** Resolve once nothing is being sent, applied or waited for, and the queue is empty or cannot be sent. */
1570
- async idle(timeoutMs = 3e4) {
1966
+ /**
1967
+ * Resolve once nothing is being sent, applied or waited for, and the queue is empty or cannot be sent now. With
1968
+ * `besidesAwaiting`, the work on a path waiting for an Edit's report (awaitEdit) is left out, as is the work behind it
1969
+ * on that path's chain: a tool's flush does not wait for an Edit of another session to be reported.
1970
+ */
1971
+ async idle(timeoutMs = 3e4, besidesAwaiting = false) {
1571
1972
  const deadline = Date.now() + timeoutMs;
1572
1973
  this.outboxChanged();
1573
1974
  for (;;) {
1574
- if (!(this.work.size > 0 || [...this.inflight.values()].some((n) => n > 0) || this.flushing || this.resyncing !== null || this.online && this.queued > 0)) return true;
1975
+ const work = besidesAwaiting ? [...this.work].filter((promise) => !this.awaitingEdit.has(this.workPaths.get(promise) ?? "")) : [...this.work];
1976
+ if (!(work.length > 0 || [...this.inflight.values()].some((n) => n > 0) || this.flushing || this.resyncing !== null || this.online && this.nextEntry(this.queue).entry !== void 0)) return true;
1575
1977
  if (Date.now() > deadline) return false;
1576
- await (this.work.size > 0 ? Promise.race([Promise.allSettled(this.work), sleep$1(50)]) : sleep$1(50));
1978
+ await (work.length > 0 ? Promise.race([Promise.allSettled(work), sleep$1(50)]) : sleep$1(50));
1577
1979
  }
1578
1980
  }
1579
1981
  connect() {
@@ -1591,6 +1993,9 @@ var SyncClient = class extends EventEmitter {
1591
1993
  this.connected = true;
1592
1994
  this.rejoining = false;
1593
1995
  this.current = false;
1996
+ this.probed = false;
1997
+ this.missingDownloads.clear();
1998
+ this.downloadFailures.clear();
1594
1999
  if (!this.uploadWanted && this.uploadRefusal === null) this.backoff = 500;
1595
2000
  this.lastPong = Date.now();
1596
2001
  clearInterval(this.pingTimer);
@@ -1725,7 +2130,9 @@ var SyncClient = class extends EventEmitter {
1725
2130
  * One file as the server has it as of the hub's `seq`, from a broadcast or a snapshot. A version no newer than the
1726
2131
  * one the client holds, such as a broadcast that arrives after a later snapshot, is left out. A file only on the
1727
2132
  * server whose mainline is still closed gets its new hash, or leaves `remote` when removed; nothing is written here.
1728
- * A mainline's file that opens it again brings its artifacts here.
2133
+ * A mainline's file that opens it again brings its artifacts here. A binary file comes as its hash (`blob`), and one
2134
+ * that may stay on the server, as a new screenshot of a closed mainline, goes into `remote` instead of being downloaded,
2135
+ * as a listed text does in a snapshot.
1729
2136
  */
1730
2137
  receive(event, seq) {
1731
2138
  if (event.kind === "dir") {
@@ -1733,26 +2140,42 @@ var SyncClient = class extends EventEmitter {
1733
2140
  return;
1734
2141
  }
1735
2142
  if (!this.syncs(event.path)) return;
1736
- const text = event.kind === "file" ? event.text : null;
2143
+ const file = event.kind === "file" ? event.text : event.kind === "blob" ? {
2144
+ kind: "blob",
2145
+ hash: event.hash,
2146
+ size: event.size
2147
+ } : null;
2148
+ const label = isBlob(file) ? `blob ${file.hash.slice(0, 7)}` : short(file);
1737
2149
  const held = this.fileSeq.get(event.path) ?? -1;
1738
2150
  if (seq <= held) {
1739
- this.trace(`older ${event.path} ${short(text)} seq ${seq}, holding seq ${held}`);
2151
+ this.trace(`older ${event.path} ${label} seq ${seq}, holding seq ${held}`);
1740
2152
  return;
1741
2153
  }
1742
2154
  this.fileSeq.set(event.path, seq);
2155
+ if (this.downloadErrors.has(event.path)) {
2156
+ const before = this.cloud.get(event.path);
2157
+ if (!(isBlob(file) && isBlob(before) && before.hash === file.hash) && this.downloadErrors.delete(event.path)) this.saveRemote();
2158
+ }
1743
2159
  if (event.kind === "unknown") this.unheld.add(event.path);
1744
2160
  else this.unheld.delete(event.path);
1745
- if (this.remote.has(event.path) && !this.awaiting.has(event.path) && (text === null || this.staysRemote(event.path))) {
1746
- this.trace(`remote ${event.path} ${short(text)} seq ${seq}`);
1747
- if (text === null) this.remote.delete(event.path);
1748
- else this.remote.set(event.path, contentHash(text));
2161
+ if (this.remote.has(event.path) && !this.awaiting.has(event.path) && (file === null || this.staysRemote(event.path))) {
2162
+ this.trace(`remote ${event.path} ${label} seq ${seq}`);
2163
+ if (file === null) this.remote.delete(event.path);
2164
+ else this.remote.set(event.path, serverHash(file));
2165
+ this.saveRemote();
2166
+ return;
2167
+ }
2168
+ if (isBlob(file) && !this.awaiting.has(event.path) && this.mayStay(event.path) && !existsSync(join(this.paths.data, event.path))) {
2169
+ this.trace(`remote ${event.path} ${label} seq ${seq}`);
2170
+ this.remote.set(event.path, file.hash);
2171
+ this.cloud.delete(event.path);
1749
2172
  this.saveRemote();
1750
2173
  return;
1751
2174
  }
1752
2175
  if (this.remote.delete(event.path)) this.saveRemote();
1753
- this.trace(`receive ${event.path} ${short(text)} seq ${seq}`);
2176
+ this.trace(`receive ${event.path} ${label} seq ${seq}`);
1754
2177
  this.awaiting.delete(event.path);
1755
- this.cloud.set(event.path, text);
2178
+ this.cloud.set(event.path, file);
1756
2179
  this.schedule(event.path);
1757
2180
  const mainline = MAINLINE_FILE.exec(event.path)?.[1];
1758
2181
  if (mainline !== void 0 && this.remote.size > 0) this.fetchOpened(mainline);
@@ -1865,6 +2288,14 @@ var SyncClient = class extends EventEmitter {
1865
2288
  * downloaded before the snapshot ends, with the files only on the server whose mainline is open now. The first
1866
2289
  * snapshot of a hub without known.json lists all of mainlines/, since the client may not know a mainline or its
1867
2290
  * status yet: the mainlines' own files are downloaded after, then the artifacts of the open ones.
2291
+ *
2292
+ * Each connection first asks whether the server keeps binary files (blobSupport). One that does is sent the binary
2293
+ * files the client holds by their sha256, and tells of its own as `blob`; one that does not is told of none, and tells
2294
+ * of none, nor any path it holds one at: such a path is neither named nor taken for removed. A server the client
2295
+ * uploaded the hub to holds none of them, since an upload holds texts alone: known.json stops naming them before the
2296
+ * upload (forgetBinaries), so they are files never synced here, sent as new even when this snapshot fails or the
2297
+ * client is killed; and in the snapshot after the upload, a binary file of the client's the server names removed, as
2298
+ * when it once held one there, is taken as one it never held (unheld), also in a run that started without known.json.
1868
2299
  */
1869
2300
  takeSnapshot() {
1870
2301
  return this.exclusive(() => this.snapshotNow());
@@ -1872,6 +2303,7 @@ var SyncClient = class extends EventEmitter {
1872
2303
  async snapshotNow() {
1873
2304
  let parts;
1874
2305
  let held;
2306
+ let heldBlobs;
1875
2307
  let have;
1876
2308
  let before;
1877
2309
  /** Whether the hub was uploaded, and why the server refused the upload; asked again, the snapshot may hold the hub */
@@ -1879,14 +2311,26 @@ var SyncClient = class extends EventEmitter {
1879
2311
  let refusal = null;
1880
2312
  /** The rules' generation the snapshot is asked for under: what it holds is taken in as they leave it in */
1881
2313
  let asked;
2314
+ /** Whether the snapshot is asked for with the binary files */
2315
+ let withBlobs = false;
2316
+ if (!this.probed) await this.probeBlobs();
1882
2317
  for (;;) {
1883
2318
  before = this.hubSeq;
1884
- held = await this.holdings();
1885
- for (const path of this.remote.keys()) if (held.has(path) || this.expected.get(path) !== void 0) this.remote.delete(path);
2319
+ ({texts: held, blobs: heldBlobs} = await this.holdings());
2320
+ for (const path of this.remote.keys()) if (held.has(path) || heldBlobs.has(path) || this.expected.get(path) !== void 0) this.remote.delete(path);
1886
2321
  have = new Map([...held].map(([path, text]) => [path, this.hashOf(path, text)]));
2322
+ for (const [path, blob] of heldBlobs) have.set(path, blob.hash);
1887
2323
  for (const [path, hash] of this.remote) have.set(path, hash);
1888
2324
  asked = this.scope.generation;
1889
- parts = await this.api.snapshot(have, { listOnly: this.listOnly(this.copied && !this.synced) });
2325
+ withBlobs = this.blobSupport === true;
2326
+ if (!withBlobs && !this.blobsCompared && !existsSync(this.paths.copyBlobs)) {
2327
+ mkdirSync(this.paths.own, { recursive: true });
2328
+ writeFileSync(this.paths.copyBlobs, "");
2329
+ }
2330
+ parts = await this.api.snapshot(have, {
2331
+ listOnly: this.listOnly(this.copied && !this.synced),
2332
+ blobs: this.blobSupport === true
2333
+ });
1890
2334
  if (uploaded || holdsFiles(parts, have)) break;
1891
2335
  if (this.opts.settings.onDemand === true) {
1892
2336
  if (!this.uploadWanted) {
@@ -1927,6 +2371,7 @@ var SyncClient = class extends EventEmitter {
1927
2371
  let sent = 0;
1928
2372
  this.snapshotAnswers.clear();
1929
2373
  const fromCopy = this.copied && !this.synced;
2374
+ const blobsFromCopy = withBlobs && !this.blobsCompared;
1930
2375
  for (const part of parts) for (const event of part.files) {
1931
2376
  if (event.kind !== "dir" && !this.syncs(event.path)) continue;
1932
2377
  if (event.kind === "listed") {
@@ -1940,21 +2385,31 @@ var SyncClient = class extends EventEmitter {
1940
2385
  continue;
1941
2386
  }
1942
2387
  named.add(event.path);
1943
- if (event.kind === "file") {
2388
+ if (event.kind === "file" || event.kind === "blob") {
1944
2389
  present.add(event.path);
1945
- sent++;
2390
+ if (event.kind === "file") sent++;
2391
+ }
2392
+ if (uploaded && event.kind === "removed" && heldBlobs.has(event.path)) {
2393
+ this.receive({
2394
+ kind: "unknown",
2395
+ path: event.path
2396
+ }, part.seq);
2397
+ continue;
2398
+ }
2399
+ if (event.kind === "removed" && (held.has(event.path) ? fromCopy : blobsFromCopy && heldBlobs.has(event.path))) {
2400
+ const text = held.get(event.path);
2401
+ this.snapshotAnswers.set(event.path, {
2402
+ hash: text === void 0 ? heldBlobs.get(event.path).hash : this.hashOf(event.path, text),
2403
+ removed: true
2404
+ });
1946
2405
  }
1947
- if (event.kind === "removed" && fromCopy && held.has(event.path)) this.snapshotAnswers.set(event.path, {
1948
- text: held.get(event.path),
1949
- removed: true
1950
- });
1951
2406
  this.receive(event, part.seq);
1952
2407
  }
1953
2408
  for (const [path, text] of held) {
1954
2409
  if (named.has(path)) continue;
1955
2410
  present.add(path);
1956
2411
  this.snapshotAnswers.set(path, {
1957
- text,
2412
+ hash: this.hashOf(path, text),
1958
2413
  removed: false
1959
2414
  });
1960
2415
  this.receive({
@@ -1963,37 +2418,123 @@ var SyncClient = class extends EventEmitter {
1963
2418
  text
1964
2419
  }, seq);
1965
2420
  }
1966
- for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) if (!present.has(path) && !named.has(path) && this.syncs(path)) this.receive({
1967
- kind: "removed",
1968
- path
1969
- }, seq);
2421
+ for (const [path, blob] of heldBlobs) {
2422
+ if (named.has(path)) continue;
2423
+ present.add(path);
2424
+ this.snapshotAnswers.set(path, {
2425
+ hash: blob.hash,
2426
+ removed: false
2427
+ });
2428
+ this.receive({
2429
+ kind: "blob",
2430
+ path,
2431
+ hash: blob.hash,
2432
+ size: blob.size ?? 0
2433
+ }, seq);
2434
+ }
2435
+ const blind = this.blobSupport !== true;
2436
+ for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) {
2437
+ if (present.has(path) || named.has(path) || !this.syncs(path)) continue;
2438
+ if (blind && this.knownBinary(path)) continue;
2439
+ this.receive({
2440
+ kind: "removed",
2441
+ path
2442
+ }, seq);
2443
+ }
1970
2444
  this.scopeTaken = asked;
1971
2445
  this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
1972
2446
  this.takenUpTo(seq);
1973
2447
  this.synced = true;
2448
+ if (withBlobs && !this.blobsCompared) {
2449
+ this.blobsCompared = true;
2450
+ rmSync(this.paths.copyBlobs, { force: true });
2451
+ }
1974
2452
  this.saveRemote();
1975
2453
  await this.fetchWanted(wanted);
1976
2454
  await this.settleUnsynced();
1977
2455
  this.wake();
1978
2456
  for (const path of await this.walk()) this.schedule(path);
2457
+ for (const path of this.stalledDownloads) this.schedule(path);
2458
+ this.stalledDownloads.clear();
2459
+ }
2460
+ /**
2461
+ * Ask the server whether it keeps binary files (blobSupport), once per connection, before its snapshot. A probe that
2462
+ * fails leaves what the last one found, null on a first connection, so the texts sync all the same and binary files
2463
+ * wait as they are here: the snapshot is taken again after PROBE_RETRY_MS, doubling up to PROBE_RETRY_MAX_MS while it
2464
+ * keeps failing, and asks again.
2465
+ */
2466
+ async probeBlobs() {
2467
+ const had = this.blobSupport;
2468
+ try {
2469
+ await this.askBlobs();
2470
+ } finally {
2471
+ if (had !== true && this.blobSupport === true) this.fileSeq.clear();
2472
+ }
2473
+ }
2474
+ async askBlobs() {
2475
+ const had = this.blobSupport;
2476
+ try {
2477
+ this.blobSupport = await this.api.blobRoute();
2478
+ } catch (error) {
2479
+ if (!(error instanceof Unreachable)) throw error;
2480
+ this.log(`could not ask the sync server whether it keeps binary files (${error.message}); binary files wait for a later snapshot`);
2481
+ clearTimeout(this.probeTimer);
2482
+ this.probeTimer = setTimeout(() => {
2483
+ if (this.connected && !this.probed) this.track(this.resync().catch((failed) => this.log(`could not reload the snapshot: ${failed.message}`)));
2484
+ }, this.probeRetry);
2485
+ this.probeRetry = Math.min(this.probeRetry * 2, PROBE_RETRY_MAX_MS);
2486
+ return;
2487
+ }
2488
+ this.probed = true;
2489
+ this.probeRetry = PROBE_RETRY_MS;
2490
+ if (had !== this.blobSupport) this.log(this.blobSupport ? "the sync server keeps binary files" : "the sync server keeps no binary files, as one from before them: binary files are not synced");
1979
2491
  }
1980
2492
  /**
1981
2493
  * The text the client holds of each synced file, which a snapshot request names by hash so the server leaves out what
1982
2494
  * is the same (SnapshotRequest.have): the server's version this run last took in, else the version known.json names,
1983
2495
  * else the file on disk, as in a hub cloned with git that never synced here. Only a text the client has is named,
1984
2496
  * since a file the server leaves out holds the text named for it; a file larger than the server keeps is not synced,
1985
- * so it is not named either.
2497
+ * so it is not named either. The binary files (`blobs`) come in the same order, by their sha256, only for a server that
2498
+ * keeps them (blobSupport), and only ones the client sends (binarySkip).
1986
2499
  */
1987
2500
  async holdings() {
1988
- const held = /* @__PURE__ */ new Map();
1989
- for (const [path, text] of this.cloud) if (typeof text === "string" && this.syncs(path)) held.set(path, text);
1990
- for (const [path, disk] of this.confirmed) if (!held.has(path) && disk?.text !== void 0 && this.syncs(path)) held.set(path, disk.text);
2501
+ const texts = /* @__PURE__ */ new Map();
2502
+ const blobs = /* @__PURE__ */ new Map();
2503
+ const support = this.blobSupport === true;
2504
+ const unnamed = /* @__PURE__ */ new Set();
2505
+ for (const [path, file] of this.cloud) {
2506
+ if (!this.syncs(path)) continue;
2507
+ if (typeof file === "string") texts.set(path, file);
2508
+ else if (isBlob(file) && support) blobs.set(path, file);
2509
+ else if (isBlob(file)) unnamed.add(path);
2510
+ }
2511
+ for (const [path, disk] of this.confirmed) {
2512
+ if (texts.has(path) || blobs.has(path) || unnamed.has(path) || !this.syncs(path)) continue;
2513
+ if (disk?.text !== void 0) texts.set(path, disk.text);
2514
+ else if (disk?.blob && support) blobs.set(path, {
2515
+ kind: "blob",
2516
+ hash: disk.hash
2517
+ });
2518
+ else if (disk?.blob) unnamed.add(path);
2519
+ }
1991
2520
  for (const path of await this.walk()) {
1992
- if (held.has(path)) continue;
2521
+ if (texts.has(path) || blobs.has(path) || unnamed.has(path)) continue;
1993
2522
  const local = await this.readLocal(path);
1994
- if (local.kind === "text" && !exceedsFileLimit(local.text)) held.set(path, local.text);
2523
+ if (local.kind === "text" && !exceedsFileLimit(local.text)) texts.set(path, local.text);
2524
+ else if (local.kind === "binary" && support && !this.binarySkip(path, local, false)) blobs.set(path, {
2525
+ kind: "blob",
2526
+ hash: local.hash,
2527
+ size: local.size
2528
+ });
1995
2529
  }
1996
- return held;
2530
+ return {
2531
+ texts,
2532
+ blobs
2533
+ };
2534
+ }
2535
+ /** Whether the client knows `path` as a binary file: the server's version it took in, or what it left on disk. */
2536
+ knownBinary(path) {
2537
+ return isBlob(this.cloud.get(path)) || this.expected.get(path)?.blob === true;
1997
2538
  }
1998
2539
  /** The content hash of `text`, the one `path` was named with, worked out once per text. */
1999
2540
  hashOf(path, text) {
@@ -2032,6 +2573,7 @@ var SyncClient = class extends EventEmitter {
2032
2573
  }
2033
2574
  if (Object.keys(files).length === 0) return null;
2034
2575
  const dirs = await this.walkDirs();
2576
+ const forgotten = await this.forgetBinaries();
2035
2577
  let result;
2036
2578
  try {
2037
2579
  result = await this.api.upload(files, dirs);
@@ -2044,14 +2586,49 @@ var SyncClient = class extends EventEmitter {
2044
2586
  }
2045
2587
  if (!result.ok) {
2046
2588
  this.log(`upload refused: ${result.message}`);
2589
+ this.recallBinaries(forgotten);
2047
2590
  return {
2048
2591
  ok: false,
2049
2592
  message: result.message
2050
2593
  };
2051
2594
  }
2052
2595
  this.log(`uploaded the hub: ${result.files} files`);
2596
+ await this.whenFree(() => this.editQueue((entries) => {
2597
+ const stale = entries.filter((entry) => entry.kind === "blob" && entry.sent === void 0 && waiting(entry) && entry.base !== null);
2598
+ for (const entry of stale) if (entry.kind === "blob") entry.base = null;
2599
+ return stale.length > 0;
2600
+ }));
2053
2601
  return { ok: true };
2054
2602
  }
2603
+ /**
2604
+ * Before an upload of the hub, which holds texts alone: known.json stops naming the binary files the server confirmed,
2605
+ * and the client stops expecting them, saved before the upload is sent. Whatever happens after it, a failed snapshot
2606
+ * or a client killed, the client then takes each for a file never synced here, and sends it as new (reconcile), rather
2607
+ * than for one the server removed, which would remove it here. A run that started without known.json confirmed none,
2608
+ * and writes no known.json here: a refused upload would leave it taking a text the server removed since the copy for
2609
+ * one created here. Returns what it forgot, for recallBinaries.
2610
+ */
2611
+ async forgetBinaries() {
2612
+ const forgotten = /* @__PURE__ */ new Map();
2613
+ for (const [path, disk] of this.confirmed) if (disk?.blob) forgotten.set(path, disk);
2614
+ for (const [path, disk] of forgotten) {
2615
+ this.confirmed.delete(path);
2616
+ this.snapshotAnswers.delete(path);
2617
+ const expected = this.expected.get(path);
2618
+ if (expected?.blob && expected.hash === disk.hash) this.expected.delete(path);
2619
+ if (isBlob(this.cloud.get(path))) this.cloud.delete(path);
2620
+ }
2621
+ if (forgotten.size > 0) await this.whenFree(() => underLock(this.root, () => writeKnown(this.root, this.knownFiles())));
2622
+ return forgotten;
2623
+ }
2624
+ /** The server refused the upload, so it holds what it held: the binary files forgetBinaries forgot are known again. */
2625
+ recallBinaries(forgotten) {
2626
+ for (const [path, disk] of forgotten) {
2627
+ if (!this.confirmed.has(path)) this.confirmed.set(path, disk);
2628
+ if (!this.expected.has(path)) this.expected.set(path, disk);
2629
+ }
2630
+ if (forgotten.size > 0) this.saveKnown();
2631
+ }
2055
2632
  /** Tell the writer, once for each reason, that the server has no copy of the hub and refused this one (snapshotNow). */
2056
2633
  refusedUpload(message) {
2057
2634
  if (this.uploadRefusal === message) return;
@@ -2067,13 +2644,18 @@ var SyncClient = class extends EventEmitter {
2067
2644
  if (this.closing) return;
2068
2645
  const next = (this.chains.get(path) ?? Promise.resolve()).then(() => this.reconcile(path)).catch((error) => this.log(`${path}: ${error.stack ?? String(error)}`));
2069
2646
  this.chains.set(path, next);
2070
- this.track(next).then(() => {
2647
+ this.track(next, path).then(() => {
2071
2648
  if (this.chains.get(path) === next) this.chains.delete(path);
2072
2649
  });
2073
2650
  }
2074
- track(promise) {
2651
+ /** Count `promise` as work under way until it settles; `path` for one on that path's chain. */
2652
+ track(promise, path) {
2075
2653
  this.work.add(promise);
2076
- promise.finally(() => this.work.delete(promise)).catch(() => {});
2654
+ if (path !== void 0) this.workPaths.set(promise, path);
2655
+ promise.finally(() => {
2656
+ this.work.delete(promise);
2657
+ this.workPaths.delete(promise);
2658
+ }).catch(() => {});
2077
2659
  return promise;
2078
2660
  }
2079
2661
  /**
@@ -2084,26 +2666,34 @@ var SyncClient = class extends EventEmitter {
2084
2666
  pending(path) {
2085
2667
  return (this.inflight.get(path) ?? 0) > 0 || this.queue.some((entry) => waiting(entry) && entryPaths(entry).includes(path));
2086
2668
  }
2669
+ /**
2670
+ * What the file at `path` holds now. A file larger than any text the server keeps (MAX_FILE_BYTES) is read in chunks
2671
+ * first (scanFile): a binary file comes as its sha256 and size alone, so every reconcile after a snapshot, which reads
2672
+ * every file, holds at most a chunk of it, and its bytes are read again only to be saved (bytesOf). Such a file that is
2673
+ * UTF-8 is read whole, as a smaller file is, since the text is what reconcile compares; it is not sent either way
2674
+ * (exceedsFileLimit).
2675
+ */
2087
2676
  async readLocal(path) {
2088
- let bytes;
2677
+ const file = join(this.paths.data, path);
2089
2678
  try {
2090
- bytes = await readFile(join(this.paths.data, path));
2679
+ if ((await stat(file)).size > 19e5) {
2680
+ const scanned = await scanFile(file);
2681
+ if (!scanned.text) return {
2682
+ kind: "binary",
2683
+ hash: scanned.hash,
2684
+ size: scanned.size
2685
+ };
2686
+ }
2687
+ return contentOf(await readFile(file));
2091
2688
  } catch (error) {
2092
2689
  const code = error.code;
2093
2690
  if (code === "ENOENT" || code === "ENOTDIR" || code === "EISDIR") return { kind: "absent" };
2094
2691
  throw error;
2095
2692
  }
2096
- try {
2097
- return {
2098
- kind: "text",
2099
- text: decoder.decode(bytes)
2100
- };
2101
- } catch {
2102
- return {
2103
- kind: "binary",
2104
- hash: contentHash(bytes.toString("latin1"))
2105
- };
2106
- }
2693
+ }
2694
+ /** The bytes of binary file `content` at `path`: read again from the file when readLocal read it in chunks. */
2695
+ async bytesOf(path, content) {
2696
+ return content.bytes ?? await readFile(join(this.paths.data, path));
2107
2697
  }
2108
2698
  /**
2109
2699
  * Compare the local file with what the client last left there and with the server's version, and act: queue a
@@ -2119,9 +2709,9 @@ var SyncClient = class extends EventEmitter {
2119
2709
  await sleep$1(BINARY_RECHECK_MS);
2120
2710
  const again = await this.readLocal(path);
2121
2711
  if (again.kind !== "binary" || again.hash !== local.hash) return this.schedule(path);
2122
- return this.skipBinary(path, local.hash);
2712
+ if (this.binarySkip(path, local, true)) return;
2123
2713
  }
2124
- if (local.kind === "text" && exceedsFileLimit(local.text)) return this.skipLarge(path, local.text);
2714
+ if (local.kind === "text" && exceedsFileLimit(local.text)) return this.skipLarge(path, contentHash(local.text), false);
2125
2715
  const seen = this.expected.get(path);
2126
2716
  if (seen === void 0 ? local.kind !== "absent" : !this.matches(seen, local)) {
2127
2717
  await this.whenFree(() => this.syncQueue());
@@ -2130,48 +2720,49 @@ var SyncClient = class extends EventEmitter {
2130
2720
  const after = this.expected.get(path);
2131
2721
  if (after !== seen && after !== void 0 && this.matches(after, local)) return this.schedule(path);
2132
2722
  }
2133
- const target = this.cloud.has(path) ? this.cloud.get(path) : this.synced && !this.scopeStale ? null : void 0;
2723
+ const target = this.cloud.has(path) ? this.cloud.get(path) : this.synced && !this.scopeStale && (this.blobSupport === true || !this.knownBinary(path)) ? null : void 0;
2134
2724
  const disk = this.expected.get(path);
2135
2725
  const unheld = target === null && this.unheld.has(path);
2136
2726
  if (disk === void 0) {
2137
2727
  if (local.kind === "absent") {
2138
- if (typeof target === "string") await this.replace(path, target);
2728
+ if (target !== null && target !== void 0) await this.replace(path, target);
2139
2729
  return;
2140
2730
  }
2141
- if (target === local.text) return this.adopt(path, local.text);
2731
+ if (holdsServer(local, target)) return this.adoptContent(path, local);
2142
2732
  if (!this.synced || target === void 0) return;
2143
2733
  const answer = this.snapshotAnswers.get(path);
2144
- if (answer?.text === local.text && !this.pending(path)) {
2734
+ if (answer?.hash === hashOfContent(local) && !this.pending(path)) {
2145
2735
  this.snapshotAnswers.delete(path);
2146
2736
  if (!answer.removed) {
2147
- this.adopt(path, local.text);
2737
+ this.adoptContent(path, local);
2148
2738
  return this.schedule(path);
2149
2739
  }
2150
- const saved = await this.saveConflict(path, local.text);
2151
- this.expected.set(path, {
2152
- hash: contentHash(local.text),
2153
- text: local.text
2154
- });
2740
+ const saved = await this.saveConflict(path, local.kind === "text" ? local.text : await this.bytesOf(path, local));
2741
+ this.expected.set(path, diskOf(local));
2155
2742
  return this.replace(path, null, saved);
2156
2743
  }
2744
+ if (local.kind === "binary") return this.queueBlob(path, local, { base: null });
2157
2745
  return this.queueLocal(path, local.text, { start: target === null ? null : {
2158
2746
  text: local.text,
2159
2747
  server: local.text
2160
2748
  } });
2161
2749
  }
2162
2750
  if (!this.matches(disk, local)) {
2163
- if (local.kind === "text" && target === local.text && !this.pending(path)) {
2164
- this.adopt(path, local.text);
2751
+ if (holdsServer(local, target) && !this.pending(path)) {
2752
+ this.adoptContent(path, local);
2165
2753
  return this.release(path);
2166
2754
  }
2167
2755
  if (local.kind === "text") {
2168
- if (local.text === "" && disk?.text) {
2756
+ if (local.text === "" && (disk?.text || disk?.blob)) {
2169
2757
  await sleep$1(EMPTY_RECHECK_MS);
2170
2758
  const again = await this.readLocal(path);
2171
2759
  if (again.kind !== "text" || again.text !== "") return this.schedule(path);
2172
2760
  }
2761
+ const reported = !unheld && disk !== null ? await this.awaitEdit(path, local.text, disk) : null;
2762
+ if (reported) return this.replayEdit(reported);
2173
2763
  return this.queueLocal(path, local.text, unheld ? { start: null } : {});
2174
2764
  }
2765
+ if (local.kind === "binary") return this.queueBlob(path, local, unheld ? { base: null } : {});
2175
2766
  if (target === null || disk === null) {
2176
2767
  this.forget(path);
2177
2768
  this.confirm(path, null);
@@ -2179,7 +2770,7 @@ var SyncClient = class extends EventEmitter {
2179
2770
  }
2180
2771
  return this.queueRemove(path, disk);
2181
2772
  }
2182
- if (target === void 0 || (disk === null ? target === null : target !== null && disk.hash === contentHash(target))) {
2773
+ if (target === void 0 || (disk === null ? target === null : target !== null && disk.hash === serverHash(target))) {
2183
2774
  if (disk?.text !== void 0) this.remember(path, {
2184
2775
  text: disk.text,
2185
2776
  server: disk.text
@@ -2190,6 +2781,7 @@ var SyncClient = class extends EventEmitter {
2190
2781
  }
2191
2782
  if (this.pending(path)) return;
2192
2783
  if (unheld && local.kind === "text") return this.queueUnheld(path, local.text);
2784
+ if (unheld && local.kind === "binary") return this.queueBlob(path, local, { unheld: true });
2193
2785
  await this.replace(path, target);
2194
2786
  }
2195
2787
  /**
@@ -2214,6 +2806,7 @@ var SyncClient = class extends EventEmitter {
2214
2806
  }
2215
2807
  matches(disk, local) {
2216
2808
  if (disk === null) return local.kind === "absent";
2809
+ if (disk.blob) return local.kind === "binary" && disk.hash === local.hash;
2217
2810
  return local.kind === "text" && disk.hash === contentHash(local.text);
2218
2811
  }
2219
2812
  adopt(path, text) {
@@ -2230,6 +2823,17 @@ var SyncClient = class extends EventEmitter {
2230
2823
  text
2231
2824
  });
2232
2825
  }
2826
+ /** The file at `path` holds `content`, the server's version: adopted, a text as adopt does, a binary file by its sha256. */
2827
+ adoptContent(path, content) {
2828
+ if (content.kind === "text") return this.adopt(path, content.text);
2829
+ const disk = {
2830
+ hash: content.hash,
2831
+ blob: true
2832
+ };
2833
+ this.expected.set(path, disk);
2834
+ this.history.delete(path);
2835
+ this.confirm(path, disk);
2836
+ }
2233
2837
  /** A removed file starts over: its earlier versions are gone from the server and are no base for a new one. */
2234
2838
  forget(path) {
2235
2839
  this.expected.set(path, null);
@@ -2260,24 +2864,51 @@ var SyncClient = class extends EventEmitter {
2260
2864
  if (list.length > HISTORY) list.shift();
2261
2865
  this.history.set(path, list);
2262
2866
  }
2263
- skipBinary(path, hash) {
2264
- const key = `binary:${path}:${hash}`;
2867
+ /**
2868
+ * Whether the binary file at `path`, which holds `local`, stays as it is here: neither sent nor written over nor
2869
+ * removed. So it does at a path the hub reads as text (readsAsText), larger than MAX_BLOB_BYTES or than the server took,
2870
+ * on a server that keeps no binary files, and while the client does not know yet whether it does (blobSupport), which
2871
+ * the snapshot after the probe reconciles again. With `tell`, the writer is told once for each content.
2872
+ */
2873
+ binarySkip(path, local, tell) {
2874
+ if (readsAsText(path)) {
2875
+ if (tell) this.tellOnce(`text-only:${path}:${local.hash}`, {
2876
+ kind: "binary-text-file",
2877
+ path,
2878
+ message: `${path} is not UTF-8 text, and the hub reads it as text (config.yaml, .md, .yaml, .yml, ops/*.log, .gitignore), so it is not synced and not overwritten; write it as UTF-8 text, or rename it if it is meant to be a binary file`
2879
+ });
2880
+ return true;
2881
+ }
2882
+ if (local.size > 52428800 || this.oversized.has(`${path}\0${local.hash}`)) {
2883
+ if (tell) this.skipLarge(path, local.hash, true);
2884
+ return true;
2885
+ }
2886
+ if (this.blobSupport === false) {
2887
+ if (tell) this.skipBinary(path, local.hash);
2888
+ return true;
2889
+ }
2890
+ return this.blobSupport === null;
2891
+ }
2892
+ /** Tell the writer of `problem` once for `key`, whatever runs into it again. */
2893
+ tellOnce(key, problem) {
2265
2894
  if (this.skipped.has(key)) return;
2266
2895
  this.skipped.add(key);
2267
- this.problem({
2896
+ this.problem(problem);
2897
+ }
2898
+ skipBinary(path, hash) {
2899
+ this.tellOnce(`binary:${path}:${hash}`, {
2268
2900
  kind: "skipped-binary",
2269
2901
  path,
2270
- message: `${path} is not UTF-8 text, so it is not synced and not overwritten; other environments do not see it`
2902
+ message: `${path} is not UTF-8 text, and the sync server keeps no binary files, as one from before them: it stays as it is here, not overwritten, and is sent once the sync server keeps binary files; other environments do not see it until then`
2271
2903
  });
2272
2904
  }
2273
- skipLarge(path, text) {
2274
- const key = `large:${path}:${contentHash(text)}`;
2275
- if (this.skipped.has(key)) return;
2276
- this.skipped.add(key);
2277
- this.problem({
2905
+ /** A text of `path` larger than MAX_FILE_BYTES, or a binary file larger than the server keeps, of `hash`. */
2906
+ skipLarge(path, hash, binary) {
2907
+ const limit = binary ? `the ${MAX_BLOB_BYTES} bytes the sync server keeps of a binary file` : `the ${MAX_FILE_BYTES} bytes the sync server keeps of a file`;
2908
+ this.tellOnce(`large:${path}:${hash}`, {
2278
2909
  kind: "skipped-large",
2279
2910
  path,
2280
- message: `${path} is larger than the ${MAX_FILE_BYTES} bytes the sync server keeps of a file, so it is not synced and not overwritten; other environments do not see it. Split it, or add it to .mainflow/.gitignore`
2911
+ message: `${path} is larger than ${limit}, so it is not synced and not overwritten; other environments do not see it. ${binary ? "Make it smaller" : "Split it"}, or add it to .mainflow/.gitignore`
2281
2912
  });
2282
2913
  }
2283
2914
  /**
@@ -2324,7 +2955,7 @@ var SyncClient = class extends EventEmitter {
2324
2955
  */
2325
2956
  mayStay(path, held) {
2326
2957
  if (!this.staysRemote(path)) return false;
2327
- return !(held?.has(path) ?? false) && typeof this.cloud.get(path) !== "string" && this.expected.get(path) === void 0 && !this.pending(path);
2958
+ return !(held?.has(path) ?? false) && (this.cloud.get(path) ?? null) === null && this.expected.get(path) === void 0 && !this.pending(path);
2328
2959
  }
2329
2960
  /** Note `path` as only on the server, with the hash of the server's text as of `seq`, unless a newer version came. */
2330
2961
  keepRemote(path, hash, seq) {
@@ -2386,22 +3017,34 @@ var SyncClient = class extends EventEmitter {
2386
3017
  * file the client knows by its hash, so the server sends the targets and only what changed of the rest; the artifacts
2387
3018
  * of closed mainlines other than the targets' stay listed. What comes is taken in as a broadcast of its batch's seq
2388
3019
  * is: a target's text is written here, and a target named nowhere is not on the server, since the range covers it
2389
- * and `have` does not name it. Returns, besides, the files of other paths listed that may not stay on the server.
3020
+ * and `have` does not name it. Returns, besides, the files of other paths listed that may not stay on the server. A
3021
+ * binary file only on the server is not asked for while the client does not know that the server keeps binary files
3022
+ * (blobSupport): such a snapshot would name it nowhere, as if it were gone. Nor is a target such a snapshot names
3023
+ * nowhere taken for gone, since the server may hold a binary file there now. Both stay only on the server,
3024
+ * `unavailable`.
2390
3025
  */
2391
3026
  async fetchNow(targets) {
2392
- const sorted = [...new Set(targets)].filter((path) => this.syncs(path)).sort(comparePaths);
3027
+ const asked = [...new Set(targets)].filter((path) => this.syncs(path));
3028
+ const unavailable = this.blobSupport === true ? [] : asked.filter((path) => BLOB_HASH.test(this.remote.get(path) ?? ""));
3029
+ const sorted = asked.filter((path) => !unavailable.includes(path)).sort(comparePaths);
2393
3030
  const again = /* @__PURE__ */ new Set();
2394
3031
  if (sorted.length === 0 || this.closing) return {
2395
3032
  fetched: [],
2396
3033
  gone: [],
2397
- again
3034
+ again,
3035
+ unavailable
2398
3036
  };
2399
3037
  const want = new Set(sorted);
2400
3038
  for (const path of sorted) this.awaiting.add(path);
2401
3039
  let done = false;
2402
3040
  try {
2403
3041
  const have = /* @__PURE__ */ new Map();
2404
- for (const [path, text] of this.cloud) if (typeof text === "string" && !want.has(path) && this.syncs(path)) have.set(path, this.hashOf(path, text));
3042
+ const support = this.blobSupport === true;
3043
+ for (const [path, file] of this.cloud) {
3044
+ if (want.has(path) || !this.syncs(path)) continue;
3045
+ if (typeof file === "string") have.set(path, this.hashOf(path, file));
3046
+ else if (isBlob(file) && support) have.set(path, file.hash);
3047
+ }
2405
3048
  for (const [path, hash] of this.remote) if (!want.has(path)) have.set(path, hash);
2406
3049
  const listOnly = this.listOnly(false)?.filter((prefix) => !sorted.some((path) => path.startsWith(prefix)));
2407
3050
  const first = sorted[0];
@@ -2409,12 +3052,14 @@ var SyncClient = class extends EventEmitter {
2409
3052
  const parts = await this.api.snapshot(have, {
2410
3053
  listOnly,
2411
3054
  after,
2412
- through: sorted.at(-1)
3055
+ through: sorted.at(-1),
3056
+ blobs: support
2413
3057
  });
2414
3058
  if (this.closing) return {
2415
3059
  fetched: [],
2416
3060
  gone: [],
2417
- again
3061
+ again,
3062
+ unavailable
2418
3063
  };
2419
3064
  const mentioned = /* @__PURE__ */ new Set();
2420
3065
  const fetched = [];
@@ -2434,13 +3079,17 @@ var SyncClient = class extends EventEmitter {
2434
3079
  if (want.has(event.path)) {
2435
3080
  this.remote.delete(event.path);
2436
3081
  if ((this.fileSeq.get(event.path) ?? -1) <= part.seq) this.fileSeq.delete(event.path);
2437
- if (event.kind === "file") fetched.push(event.path);
3082
+ if (event.kind === "file" || event.kind === "blob") fetched.push(event.path);
2438
3083
  }
2439
3084
  this.receive(event, part.seq);
2440
3085
  }
2441
3086
  const gone = [];
2442
3087
  for (const path of sorted) {
2443
3088
  if (mentioned.has(path)) continue;
3089
+ if (!support) {
3090
+ unavailable.push(path);
3091
+ continue;
3092
+ }
2444
3093
  if (this.remote.delete(path)) gone.push(path);
2445
3094
  else this.receive({
2446
3095
  kind: "removed",
@@ -2453,7 +3102,8 @@ var SyncClient = class extends EventEmitter {
2453
3102
  return {
2454
3103
  fetched,
2455
3104
  gone,
2456
- again
3105
+ again,
3106
+ unavailable
2457
3107
  };
2458
3108
  } finally {
2459
3109
  for (const path of sorted) {
@@ -2474,28 +3124,27 @@ var SyncClient = class extends EventEmitter {
2474
3124
  if (targets.length === 0) return {
2475
3125
  fetched: [],
2476
3126
  gone: [],
2477
- none
3127
+ none,
3128
+ unavailable: []
2478
3129
  };
2479
- const { fetched, gone } = await this.fetchNow(targets);
3130
+ const { fetched, gone, unavailable } = await this.fetchNow(targets);
2480
3131
  return {
2481
3132
  fetched,
2482
3133
  gone,
2483
- none
3134
+ none,
3135
+ unavailable
2484
3136
  };
2485
3137
  });
2486
3138
  }
2487
3139
  /**
2488
3140
  * Download the files only on the server that `specs` name (fetchPaths), and resolve once each is written here: for
2489
3141
  * `mainflow sync fetch` and the PreToolUse hook. Offline, or when the server cannot be reached, nothing is
2490
- * downloaded and `error` says why.
3142
+ * downloaded and `error` says why. So it says why for each file that was not downloaded: a binary file whose download
3143
+ * failed, or one asked for while the client does not know that the server keeps binary files.
2491
3144
  */
2492
3145
  async fetch(specs) {
2493
3146
  const covers = (spec) => [...this.remote.keys(), ...this.awaiting].some((path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`));
2494
- if (!specs.some(covers)) return {
2495
- fetched: [],
2496
- gone: [],
2497
- none: specs
2498
- };
3147
+ if (!specs.some(covers)) return this.retryStuck(specs);
2499
3148
  if (!this.online) return {
2500
3149
  fetched: [],
2501
3150
  gone: [],
@@ -2514,10 +3163,44 @@ var SyncClient = class extends EventEmitter {
2514
3163
  };
2515
3164
  }
2516
3165
  await Promise.allSettled(result.fetched.map((path) => this.chains.get(path) ?? Promise.resolve()));
3166
+ const fetched = result.fetched.filter((path) => existsSync(join(this.paths.data, path)));
3167
+ const failed = [...result.fetched.flatMap((path) => fetched.includes(path) || !this.downloadErrors.has(path) ? [] : [`${path}: ${this.downloadErrors.get(path)}`]), ...result.unavailable.map((path) => `${path}: it may be a binary file, and ${this.blobSupport === false ? "the sync server keeps no binary files now" : "it is not known yet whether the sync server keeps binary files"}; it stays on the server, try again once mainflow sync is online`)];
2517
3168
  return {
2518
- fetched: result.fetched.filter((path) => existsSync(join(this.paths.data, path))),
3169
+ fetched,
2519
3170
  gone: result.gone,
2520
- none: result.none
3171
+ none: result.none,
3172
+ ...failed.length > 0 ? { error: `could not download ${failed.join("; ")}` } : {}
3173
+ };
3174
+ }
3175
+ /**
3176
+ * `fetch` for specs that name no file only on the server: a download that failed is no longer one, and is tried again
3177
+ * on its own. Try it now, and say why for each that still is not here.
3178
+ */
3179
+ async retryStuck(specs) {
3180
+ const covered = (spec, path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`);
3181
+ const missing = (path) => !existsSync(join(this.paths.data, path));
3182
+ const stuck = [...this.stuckDownloads().keys()].filter((path) => specs.some((spec) => covered(spec, path)));
3183
+ if (stuck.length === 0) return {
3184
+ fetched: [],
3185
+ gone: [],
3186
+ none: specs
3187
+ };
3188
+ const none = specs.filter((spec) => !stuck.some((path) => covered(spec, path)));
3189
+ if (!this.online) return {
3190
+ fetched: [],
3191
+ gone: [],
3192
+ none,
3193
+ error: "mainflow sync is offline: the sync server cannot be reached now"
3194
+ };
3195
+ for (const path of stuck) this.schedule(path);
3196
+ await Promise.allSettled(stuck.map((path) => this.chains.get(path) ?? Promise.resolve()));
3197
+ const failed = stuck.filter(missing).map((path) => `${path}: ${this.downloadErrors.get(path) ?? "it did not come"}`);
3198
+ const again = stuck.filter(missing).every((path) => this.stalledDownloads.has(path)) ? "at the next connection" : "later on its own";
3199
+ return {
3200
+ fetched: stuck.filter((path) => !missing(path)),
3201
+ gone: [],
3202
+ none,
3203
+ ...failed.length > 0 ? { error: `could not download ${failed.join("; ")}; mainflow sync tries again ${again}` } : {}
2521
3204
  };
2522
3205
  }
2523
3206
  /** Mainline `dir`'s file came: when it opens the mainline, its artifacts only on the server come here. */
@@ -2542,6 +3225,28 @@ var SyncClient = class extends EventEmitter {
2542
3225
  restoreRemote() {
2543
3226
  for (const [path, hash] of Object.entries(readRemote(this.root))) if (this.syncs(path) && artifactOf(path) !== null && this.expected.get(path) === void 0) this.remote.set(path, hash);
2544
3227
  }
3228
+ /**
3229
+ * What remote.json lists: the files only on the server, and the artifacts whose download failed and waits for a retry.
3230
+ * The interface lists both as files not here yet, so a failed download stays in sight; fetch tries it again and says
3231
+ * why it failed. A file leaves the list when its download succeeds.
3232
+ */
3233
+ listedRemote() {
3234
+ const listed = new Map(this.remote);
3235
+ for (const [path, hash] of this.stuckDownloads()) if (!listed.has(path)) listed.set(path, hash);
3236
+ return listed;
3237
+ }
3238
+ /**
3239
+ * The artifacts whose download failed and that are still not here, by the sha256 the server holds: the server still
3240
+ * has that binary version, and the client keeps the error for it. A file the server removed or changed since is not one.
3241
+ */
3242
+ stuckDownloads() {
3243
+ const stuck = /* @__PURE__ */ new Map();
3244
+ for (const path of this.downloadErrors.keys()) {
3245
+ const file = this.cloud.get(path);
3246
+ if (isBlob(file) && artifactOf(path) !== null && !existsSync(join(this.paths.data, path))) stuck.set(path, file.hash);
3247
+ }
3248
+ return stuck;
3249
+ }
2545
3250
  saveRemote() {
2546
3251
  if (this.remoteTimer) return;
2547
3252
  this.remoteTimer = setTimeout(() => this.saveRemoteNow(), 200);
@@ -2550,9 +3255,10 @@ var SyncClient = class extends EventEmitter {
2550
3255
  if (this.abandoned && !final) return;
2551
3256
  clearTimeout(this.remoteTimer);
2552
3257
  this.remoteTimer = void 0;
2553
- if (this.remote.size === 0 && !existsSync(this.paths.remote)) return;
3258
+ const listed = this.listedRemote();
3259
+ if (listed.size === 0 && !existsSync(this.paths.remote)) return;
2554
3260
  try {
2555
- writeRemote(this.root, Object.fromEntries([...this.remote].sort(([a], [b]) => comparePaths(a, b))));
3261
+ writeRemote(this.root, Object.fromEntries([...listed].sort(([a], [b]) => comparePaths(a, b))));
2556
3262
  } catch (error) {
2557
3263
  this.log(`could not save local/sync/remote.json: ${error.message}`);
2558
3264
  }
@@ -2566,15 +3272,18 @@ var SyncClient = class extends EventEmitter {
2566
3272
  * not lost; it is read again shortly after, for a program that still held the old file open. A new file is linked
2567
3273
  * into place, which fails rather than overwrite a file created meanwhile; a removed file is first moved aside and
2568
3274
  * checked. `unsynced` names where the text of a never synced file the server no longer has was saved: the writer is
2569
- * told of it once the file is removed, and it is deleted again when a write in the outbox keeps the file instead.
3275
+ * told of it once the file is removed, and it is deleted again when a write in the outbox keeps the file instead. A
3276
+ * binary file is downloaded into the prepared path first, checked against its sha256; one that cannot be downloaded
3277
+ * now is left as it is (download).
2570
3278
  */
2571
3279
  async replace(path, target, unsynced) {
2572
3280
  const file = join(this.paths.data, path);
2573
3281
  const temp = this.tempName();
2574
- this.trace(`replacing ${path} with ${short(target)}`);
3282
+ this.trace(`replacing ${path} with ${isBlob(target) ? `blob ${target.hash.slice(0, 7)}` : short(target)}`);
2575
3283
  if (target !== null) {
2576
3284
  await mkdir(dirname(file), { recursive: true });
2577
- await writeFile(temp, target);
3285
+ if (typeof target === "string") await writeFile(temp, target);
3286
+ else if (!await this.download(path, target, temp)) return;
2578
3287
  }
2579
3288
  let step;
2580
3289
  try {
@@ -2596,24 +3305,25 @@ var SyncClient = class extends EventEmitter {
2596
3305
  saved: unsynced,
2597
3306
  message: `${path} was never synced here and the sync server no longer has it, so it was removed here too; its text was saved to ${unsynced}`
2598
3307
  });
2599
- if (step.moved !== null && step.disk && contentHash(step.moved) !== step.disk.hash) {
2600
- const saved = await this.saveConflict(path, step.moved);
3308
+ if (step.moved !== null && step.disk && !this.matches(step.disk, step.moved)) {
3309
+ const saved = await this.saveConflict(path, step.moved.kind === "text" ? step.moved.text : step.moved.bytes);
2601
3310
  this.problem({
2602
3311
  kind: "conflict",
2603
3312
  path,
2604
3313
  saved,
2605
- message: `${path} was removed by another environment while it was being written here; the text written here was saved to ${saved}`
3314
+ message: `${path} was removed by another environment while it was being written here; the ${step.moved.kind === "text" ? "text" : "binary file"} written here was saved to ${saved}`
2606
3315
  });
2607
3316
  }
2608
3317
  return;
2609
3318
  }
2610
3319
  const { displaced, swapped, from } = step;
2611
- this.trace(`wrote ${path} ${short(target)}${swapped ? " (exchanged)" : ""}; displaced ${short(displaced)}`);
2612
- const changed = displaced !== null && contentHash(displaced) !== from.over.hash;
2613
- const later = swapped && changed && displaced === "";
3320
+ this.trace(`wrote ${path} ${isBlob(target) ? `blob ${target.hash.slice(0, 7)}` : short(target)}${swapped ? " (exchanged)" : ""}; displaced ${displaced?.kind === "binary" ? `blob ${displaced.hash.slice(0, 7)}` : short(displaced?.text ?? null)}`);
3321
+ const changed = displaced !== null && !this.matches(from.over, displaced);
3322
+ const later = swapped && changed && displaced.kind === "text" && displaced.text === "";
2614
3323
  if (changed && !later) {
2615
- this.log(`${path}: a local write landed while the server's version was written; sending it`);
2616
- await this.queueLocal(path, displaced, { displaced: from });
3324
+ this.log(`${path}: a local write landed while the server's version was written; ${displaced.kind === "text" ? "sending it" : "saving it"}`);
3325
+ if (displaced.kind === "text") await this.queueLocal(path, displaced.text, { displaced: from });
3326
+ else await this.displacedBinary(path, displaced);
2617
3327
  }
2618
3328
  if (swapped) {
2619
3329
  const recheck = this.recheck(path, temp, later ? null : displaced, from);
@@ -2674,11 +3384,16 @@ var SyncClient = class extends EventEmitter {
2674
3384
  const disk = this.expected.get(path);
2675
3385
  const read = (at) => {
2676
3386
  try {
2677
- return readFileSync(at, "utf8");
3387
+ return contentOf(readFileSync(at));
2678
3388
  } catch {
2679
3389
  return null;
2680
3390
  }
2681
3391
  };
3392
+ /** Whether the file at `at` holds the server's version. */
3393
+ const holdsTarget = (at) => {
3394
+ const content = read(at);
3395
+ return content !== null && target !== null && holdsServer(content, target);
3396
+ };
2682
3397
  if (target === null) {
2683
3398
  try {
2684
3399
  renameSync(file, temp);
@@ -2728,7 +3443,7 @@ var SyncClient = class extends EventEmitter {
2728
3443
  if (answer === "ok") this.swapFailures = 0;
2729
3444
  else {
2730
3445
  this.swapFailed(path, answer === "unsupported" ? "the file system refused it" : answer.message);
2731
- if (answer !== "unsupported" && read(file) === target && read(temp) !== target) answer = "ok";
3446
+ if (answer !== "unsupported" && holdsTarget(file) && !holdsTarget(temp)) answer = "ok";
2732
3447
  }
2733
3448
  if (answer === "ok") {
2734
3449
  swapped = true;
@@ -2761,34 +3476,119 @@ var SyncClient = class extends EventEmitter {
2761
3476
  */
2762
3477
  async recheck(path, temp, taken, from) {
2763
3478
  await sleep$1(taken === null ? EMPTY_RECHECK_MS : LATE_WRITE_MS);
2764
- const later = await readFile(temp, "utf8").catch(() => null);
3479
+ const later = await readFile(temp).then(contentOf, () => null);
2765
3480
  await rm(temp, { force: true });
2766
- if (later !== null && later !== taken && contentHash(later) !== from.over.hash) {
2767
- this.log(`${path}: a late write into the replaced file; sending it`);
2768
- const held = taken === null ? from : {
3481
+ if (later !== null && (taken === null || !sameLocal(later, taken)) && !this.matches(from.over, later)) {
3482
+ this.log(`${path}: a late write into the replaced file; ${later.kind === "text" ? "sending it" : "saving it"}`);
3483
+ if (later.kind === "binary") return this.withChain(path, () => this.displacedBinary(path, later));
3484
+ const held = taken?.kind !== "text" ? from : {
2769
3485
  ...from,
2770
- among: /* @__PURE__ */ new Set([...from.among, taken])
3486
+ among: /* @__PURE__ */ new Set([...from.among, taken.text])
2771
3487
  };
2772
- await this.withChain(path, () => this.queueLocal(path, later, { displaced: held }));
3488
+ await this.withChain(path, () => this.queueLocal(path, later.text, { displaced: held }));
3489
+ }
3490
+ }
3491
+ /**
3492
+ * A binary file a program wrote while the server's version took its place (layer 3), now off disk: it cannot be merged
3493
+ * with the server's version, so it is saved and the writer told, as a refused binary file is.
3494
+ */
3495
+ async displacedBinary(path, content) {
3496
+ const saved = await this.saveConflict(path, content.bytes);
3497
+ this.problem({
3498
+ kind: "conflict",
3499
+ path,
3500
+ saved,
3501
+ message: `${path}: a binary file was written here while the server's version took its place; the file written here was saved to ${saved} and the file now has the server's version`
3502
+ });
3503
+ }
3504
+ /**
3505
+ * Download the content of `blob`, the server's version of `path`, into `temp`: true once it is there and of its sha256.
3506
+ * Offline, or when the download fails, nothing is written and false comes back, and downloadErrors keeps why for
3507
+ * fetch. While online it is tried again after DOWNLOAD_BACKOFF_MS, doubling, with one timer for each path. A server
3508
+ * that could not be reached or failed is tried for as long as the connection holds, up to BLOB_BACKOFF_MAX_MS between
3509
+ * tries. Content of another sha256, or a file this machine could not write, as on a full disk, is tried DOWNLOAD_TRIES
3510
+ * times in a row, when the writer is told and the file waits for the next connection's snapshot. So does a file whose
3511
+ * download the connection dropped under: the snapshot after reconnecting names its version again, which receive takes
3512
+ * for one it already holds, so the file is reconciled again after that snapshot. Content the server no longer holds,
3513
+ * as when the file changed again meanwhile and the old content was cleaned up, reloads the snapshot, once per content
3514
+ * on a connection.
3515
+ */
3516
+ async download(path, blob, temp) {
3517
+ clearTimeout(this.downloadRetries.get(path));
3518
+ this.downloadRetries.delete(path);
3519
+ try {
3520
+ if (await this.api.downloadBlob(blob.hash, temp)) {
3521
+ this.downloadFailures.delete(blob.hash);
3522
+ if (this.downloadErrors.delete(path)) this.saveRemote();
3523
+ return true;
3524
+ }
3525
+ await rm(temp, { force: true });
3526
+ this.downloadErrors.set(path, `the sync server no longer holds its content ${blob.hash.slice(0, 7)}`);
3527
+ this.saveRemote();
3528
+ this.log(`${path}: the sync server no longer holds its content ${blob.hash.slice(0, 7)}; reloading the snapshot`);
3529
+ if (!this.missingDownloads.has(blob.hash) && this.online) {
3530
+ this.missingDownloads.add(blob.hash);
3531
+ this.track(this.resync().catch((error) => this.log(`could not reload the snapshot: ${error.message}`)));
3532
+ }
3533
+ return false;
3534
+ } catch (error) {
3535
+ await rm(temp, { force: true });
3536
+ const reason = error.message;
3537
+ this.downloadErrors.set(path, reason);
3538
+ this.saveRemote();
3539
+ this.log(`${path}: could not download its content ${blob.hash.slice(0, 7)}: ${reason}`);
3540
+ if (this.closing) return false;
3541
+ if (!this.online) {
3542
+ this.stalledDownloads.add(path);
3543
+ return false;
3544
+ }
3545
+ const failures = (this.downloadFailures.get(blob.hash) ?? 0) + 1;
3546
+ this.downloadFailures.set(blob.hash, failures);
3547
+ if (!!(error instanceof BlobMismatch || error instanceof LocalWriteError) && failures >= DOWNLOAD_TRIES) {
3548
+ this.stalledDownloads.add(path);
3549
+ this.tellOnce(`download:${path}:${blob.hash}`, {
3550
+ kind: "download-failed",
3551
+ path,
3552
+ message: `${path}: the sync server's version could not be downloaded ${failures} times in a row (${reason}); the file stays as it is here, and the next connection tries again`
3553
+ });
3554
+ return false;
3555
+ }
3556
+ const timer = setTimeout(() => {
3557
+ this.downloadRetries.delete(path);
3558
+ this.schedule(path);
3559
+ }, Math.min((this.opts.downloadBackoffMs ?? DOWNLOAD_BACKOFF_MS) * 2 ** Math.min(failures - 1, 30), BLOB_BACKOFF_MAX_MS));
3560
+ this.downloadRetries.set(path, timer);
3561
+ return false;
2773
3562
  }
2774
3563
  }
2775
3564
  withChain(path, fn) {
2776
3565
  const next = (this.chains.get(path) ?? Promise.resolve()).then(fn).catch((error) => this.log(`${path}: ${String(error)}`));
2777
3566
  this.chains.set(path, next);
2778
- return this.track(next);
2779
- }
2780
- wrote(path, text) {
3567
+ return this.track(next, path);
3568
+ }
3569
+ /** The server's version `target` was written at `path`: what the client leaves there, and confirmed. */
3570
+ wrote(path, target) {
3571
+ if (isBlob(target)) {
3572
+ const disk = {
3573
+ hash: target.hash,
3574
+ blob: true
3575
+ };
3576
+ this.expected.set(path, disk);
3577
+ this.history.delete(path);
3578
+ this.confirm(path, disk);
3579
+ return;
3580
+ }
2781
3581
  this.expected.set(path, {
2782
- hash: contentHash(text),
2783
- text
3582
+ hash: contentHash(target),
3583
+ text: target
2784
3584
  });
2785
3585
  this.remember(path, {
2786
- text,
2787
- server: text
3586
+ text: target,
3587
+ server: target
2788
3588
  });
2789
3589
  this.confirm(path, {
2790
- hash: contentHash(text),
2791
- text
3590
+ hash: contentHash(target),
3591
+ text: target
2792
3592
  });
2793
3593
  }
2794
3594
  /**
@@ -2831,7 +3631,8 @@ var SyncClient = class extends EventEmitter {
2831
3631
  * disk, is not on disk: it was written over the version the client had left there, and on a version the old file held
2832
3632
  * (layer 3); with none of those left, it is based on the text the client had left there, so layer 2 still tells the
2833
3633
  * writer's lines from the ones the server changed meanwhile. `start` overrides layer 1: the version the writer started
2834
- * from, or null for a new file.
3634
+ * from, or null for a new file. A text written over a binary file the client left there is based on that file's sha256,
3635
+ * with no base text: the server replaces the binary file with it when it still holds that one (PatchRequest).
2835
3636
  */
2836
3637
  async queueLocal(path, text, options = {}) {
2837
3638
  const { displaced } = options;
@@ -2843,7 +3644,7 @@ var SyncClient = class extends EventEmitter {
2843
3644
  hash: contentHash(text),
2844
3645
  text
2845
3646
  });
2846
- let start = options.start !== void 0 ? options.start ?? void 0 : before === null ? void 0 : this.startOf(path, text, before, {
3647
+ let start = options.start !== void 0 ? options.start ?? void 0 : before === null || before?.blob ? void 0 : this.startOf(path, text, before, {
2847
3648
  entries,
2848
3649
  among: displaced?.among
2849
3650
  });
@@ -2879,6 +3680,74 @@ var SyncClient = class extends EventEmitter {
2879
3680
  this.trace(`new ${path} changed before it was queued; reconciling it again`);
2880
3681
  this.schedule(path);
2881
3682
  }
3683
+ /**
3684
+ * Queue the binary file at `path`, which held `local` when it was read, as a write of its own after every write queued
3685
+ * already. In the hold of the write lock that queues it, the file is read again, and its content goes into blobs/
3686
+ * (stageHeld), so the entry uploads what the file held as it was queued (I2). The base is the hash of what the client
3687
+ * expected there before; `base: null` sends it as new. A file that changed since it was read, or that holds what the
3688
+ * client expects already, is reconciled again instead. With `unheld` the client expects that content there already, a
3689
+ * file the server never held (unheld), as after the client uploaded the hub; it is sent as new while the server still
3690
+ * has no file there.
3691
+ */
3692
+ async queueBlob(path, local, options = {}) {
3693
+ if (await this.enqueue(() => {
3694
+ const before = this.expected.get(path);
3695
+ const expected = before?.blob === true && before.hash === local.hash;
3696
+ if (options.unheld ? !expected || !this.unheld.has(path) || (this.cloud.get(path) ?? null) !== null : expected) return null;
3697
+ if (!this.stageHeld(path, local.hash)) return null;
3698
+ this.expected.set(path, {
3699
+ hash: local.hash,
3700
+ blob: true
3701
+ });
3702
+ const base = options.base === null || options.unheld ? null : before?.hash ?? null;
3703
+ this.trace(`local ${path} blob ${local.hash.slice(0, 7)} base ${base?.slice(0, 7) ?? "null"}`);
3704
+ return {
3705
+ kind: "blob",
3706
+ path,
3707
+ hash: local.hash,
3708
+ size: local.size,
3709
+ base
3710
+ };
3711
+ }) !== null) return;
3712
+ this.trace(`binary ${path} changed before it was queued; reconciling it again`);
3713
+ this.schedule(path);
3714
+ }
3715
+ /**
3716
+ * Keep the content of sha256 `hash` in blobs/ when the file at `path` holds it now: false when it holds anything else.
3717
+ * Called under the write lock, so the content leaves blobs/ only once no entry of the outbox names it (pruneStaged).
3718
+ * The file is copied first, a clone of it where the file system makes one (APFS), and the copy is hashed in chunks, so
3719
+ * a file of up to MAX_BLOB_BYTES is never held in memory whole, and what blobs/ keeps is what was hashed. With the
3720
+ * content kept already, the file is hashed in chunks alone.
3721
+ */
3722
+ stageHeld(path, hash) {
3723
+ const file = join(this.paths.data, path);
3724
+ const staged = join(this.paths.blobs, hash);
3725
+ if (existsSync(staged)) return sha256File(file) === hash;
3726
+ mkdirSync(this.paths.blobs, { recursive: true });
3727
+ const temp = `${staged}.tmp`;
3728
+ try {
3729
+ copyFileSync(file, temp);
3730
+ } catch {
3731
+ return false;
3732
+ }
3733
+ if (sha256File(temp) !== hash) {
3734
+ rmSync(temp, { force: true });
3735
+ return false;
3736
+ }
3737
+ renameSync(temp, staged);
3738
+ return true;
3739
+ }
3740
+ /** Remove from blobs/ the content no binary file's write in `entries`, the outbox, names. Called under the write lock. */
3741
+ pruneStaged(entries) {
3742
+ let names;
3743
+ try {
3744
+ names = readdirSync(this.paths.blobs);
3745
+ } catch {
3746
+ return;
3747
+ }
3748
+ const named = new Set(entries.flatMap((entry) => entry.kind === "blob" ? [entry.hash] : []));
3749
+ for (const name of names) if (!named.has(name)) rmSync(join(this.paths.blobs, name), { force: true });
3750
+ }
2882
3751
  /** What the response to queued edit `entry` means for the writer (layer 2), then wait for the broadcast it caused. */
2883
3752
  async settleEdit(entry, response, context, sent) {
2884
3753
  const { path } = entry;
@@ -2901,15 +3770,16 @@ var SyncClient = class extends EventEmitter {
2901
3770
  await this.refuse(entry, [{
2902
3771
  kind: conflict ? "conflict" : "refused",
2903
3772
  path,
2904
- text: context.writer,
3773
+ saves: true,
2905
3774
  message: (saved) => `${path}: the server refused the change (${response.message}); the text written here was saved to ${saved}${conflict ? " and the file now has the server's version" : ""}`
2906
3775
  }]);
2907
3776
  if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
2908
3777
  }
2909
3778
  /**
2910
3779
  * Layer 2. Report only when the writer's own content may be lost: lines it added that the result lacks, or lines
2911
- * it removed that the result still has. A writer that rewrote an older version unchanged, while the disk had
2912
- * other environments' additions, is taken as deleting those additions, which were kept. A result that only adds
3780
+ * it removed that the result still has. A writer that rewrote an older version unchanged, while the disk had lines
3781
+ * added since, here or in another environment, is taken as deleting those lines, which were kept: content alone does
3782
+ * not tell it from a program that read the older version and wrote it back. A result that only adds
2913
3783
  * other environments' changes to the writer's text is information.
2914
3784
  */
2915
3785
  classify(path, context, result) {
@@ -2950,7 +3820,7 @@ var SyncClient = class extends EventEmitter {
2950
3820
  if (reverted.length) return this.problem({
2951
3821
  kind: "deletion-undone",
2952
3822
  path,
2953
- message: `${path}: the write removed ${reverted.length} line(s) another environment added, such as "${reverted[0]}"; they were kept. Delete through a mainflow command or the Edit tool`
3823
+ message: `${path}: the write put back an earlier version of the file without ${reverted.length} line(s) added since, here or in another environment, such as "${reverted[0]}"; they were kept, as a program holding the earlier version writes it back the same way. Delete them through a mainflow command or the Edit tool`
2954
3824
  });
2955
3825
  }
2956
3826
  this.info(path, "merged with changes from other environments");
@@ -2978,7 +3848,7 @@ var SyncClient = class extends EventEmitter {
2978
3848
  if (!response.ok) return this.refuse(entry, [{
2979
3849
  kind: "refused",
2980
3850
  path,
2981
- text: null,
3851
+ saves: false,
2982
3852
  message: () => `${path}: the server refused the deletion (${response.message})`
2983
3853
  }]);
2984
3854
  if (response.files.some((f) => f.path === path && f.kind === "removed")) this.confirmWritten(path, null);
@@ -2990,12 +3860,45 @@ var SyncClient = class extends EventEmitter {
2990
3860
  if (response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
2991
3861
  }
2992
3862
  /**
3863
+ * What the server answered to binary file `entry`, its content uploaded first: confirmed when applied; refused, as a
3864
+ * conflict when the server holds another version than its base, its content saved and the writer told, as a refused
3865
+ * edit's text is, and the server's version takes its place.
3866
+ */
3867
+ async settleBlob(entry, response) {
3868
+ const { path } = entry;
3869
+ this.trace(`response ${path} ${response.ok ? `${response.value} blob ${entry.hash.slice(0, 7)}` : `${response.reason}: ${response.message}`}`);
3870
+ if (response.ok) {
3871
+ const disk = this.expected.get(path);
3872
+ if (disk?.blob && disk.hash === entry.hash) this.confirm(path, {
3873
+ hash: entry.hash,
3874
+ blob: true
3875
+ });
3876
+ if (response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
3877
+ return;
3878
+ }
3879
+ const conflict = response.reason === "conflict";
3880
+ await this.refuse(entry, [{
3881
+ kind: conflict ? "conflict" : "refused",
3882
+ path,
3883
+ saves: true,
3884
+ message: (saved) => `${path}: the server refused the binary file written here (${response.message}); it was saved to ${saved}${conflict ? " and the file now has the server's version" : ""}`
3885
+ }]);
3886
+ if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
3887
+ }
3888
+ /**
2993
3889
  * What an outbox entry left on disk is what the client expects there; a displaced text left nothing there, since the
2994
3890
  * server's version took it off. The versions it holds need nothing more: the outbox is where versions() reads them.
2995
3891
  */
2996
3892
  restoreEntry(entry) {
2997
3893
  if (entry.refused) {
2998
3894
  for (const path of entry.refused.paths) {
3895
+ if (entry.kind === "blob") {
3896
+ if (path === entry.path) this.expected.set(path, {
3897
+ hash: entry.hash,
3898
+ blob: true
3899
+ });
3900
+ continue;
3901
+ }
2999
3902
  const text = wroteAt(entry, path);
3000
3903
  if (text !== void 0) this.expected.set(path, text === null ? null : {
3001
3904
  hash: contentHash(text),
@@ -3013,7 +3916,11 @@ var SyncClient = class extends EventEmitter {
3013
3916
  hash: contentHash(entry.text),
3014
3917
  text: entry.text
3015
3918
  });
3016
- } else this.expected.set(entry.path, null);
3919
+ } else if (entry.kind === "blob") this.expected.set(entry.path, {
3920
+ hash: entry.hash,
3921
+ blob: true
3922
+ });
3923
+ else this.expected.set(entry.path, null);
3017
3924
  }
3018
3925
  /**
3019
3926
  * Hold the write lock for `fn`; work stop() abandoned gets an error ('abandoned after stop') instead, so it leaves
@@ -3060,7 +3967,10 @@ var SyncClient = class extends EventEmitter {
3060
3967
  changed = true;
3061
3968
  this.resolveBases(entries);
3062
3969
  }
3063
- if (changed) writeQueue(this.root, entries);
3970
+ if (changed) {
3971
+ writeQueue(this.root, entries);
3972
+ this.pruneStaged(entries);
3973
+ }
3064
3974
  this.queue = entries;
3065
3975
  return entries;
3066
3976
  }
@@ -3144,6 +4054,12 @@ var SyncClient = class extends EventEmitter {
3144
4054
  text: before,
3145
4055
  base: null
3146
4056
  };
4057
+ else if (seen.blob) write = {
4058
+ kind: "patch",
4059
+ path,
4060
+ text: before,
4061
+ base: seen.hash
4062
+ };
3147
4063
  else if (seen.hash !== contentHash(before)) write = editOn(path, before, this.startOf(path, before, seen, {
3148
4064
  entries,
3149
4065
  upto: at
@@ -3234,7 +4150,10 @@ var SyncClient = class extends EventEmitter {
3234
4150
  * outbox under the write lock; while another writer holds the lock longer than a writer waits for it, the entry
3235
4151
  * stays and is taken out on a later try, without being sent again. Before an entry is first sent, the request it
3236
4152
  * sends and its firstSentSeq are written to the outbox (`sent`), so it is sent again as it was even after a restart,
3237
- * and nothing is merged into it. An edit whose base cannot be moved onto the server's version is not sent.
4153
+ * and nothing is merged into it. An edit whose base cannot be moved onto the server's version is not sent. A binary
4154
+ * file's content is uploaded before its write is first sent, so the server never gets a `file.blob` of content it was
4155
+ * not given (uploadEntry); on a server that keeps no binary files it is not sent, and stays as it is here (dropBlob).
4156
+ * While its upload waits, the writes of other paths go ahead of it, each path's writes still in order (nextEntry).
3238
4157
  */
3239
4158
  flushQueue() {
3240
4159
  if (this.flushing || this.closing) return Promise.resolve();
@@ -3249,13 +4168,19 @@ var SyncClient = class extends EventEmitter {
3249
4168
  try {
3250
4169
  while (this.caughtUp) {
3251
4170
  let entry;
4171
+ let wake;
3252
4172
  let again = false;
3253
4173
  let unmergeable = false;
4174
+ let upload = false;
3254
4175
  this.editQueue((entries) => {
3255
- entry = entries.find((queued) => waiting(queued) || this.answered.has(queued.id));
4176
+ ({entry, wake} = this.nextEntry(entries));
3256
4177
  if (!entry || this.answered.has(entry.id)) return false;
3257
4178
  again = entry.firstSentSeq !== void 0;
3258
4179
  if (entry.sent) return false;
4180
+ if (entry.kind === "blob" && (this.blobSupport !== true || !this.uploadedBlobs.has(entry.hash))) {
4181
+ upload = true;
4182
+ return false;
4183
+ }
3259
4184
  const request = this.prepare(entry);
3260
4185
  if (!request) {
3261
4186
  unmergeable = true;
@@ -3264,7 +4189,14 @@ var SyncClient = class extends EventEmitter {
3264
4189
  entry.sent = request;
3265
4190
  entry.firstSentSeq ??= this.hubSeq;
3266
4191
  });
3267
- if (!entry || this.closing) break;
4192
+ if (!entry || this.closing) {
4193
+ if (!entry && wake !== void 0) this.retryQueue(Math.max(0, wake - Date.now()));
4194
+ break;
4195
+ }
4196
+ if (upload) {
4197
+ await this.uploadEntry(entry);
4198
+ continue;
4199
+ }
3268
4200
  if (unmergeable) await this.refuseUnmergeable(entry);
3269
4201
  else if (!this.answered.has(entry.id)) {
3270
4202
  this.sending = entry.id;
@@ -3279,9 +4211,199 @@ var SyncClient = class extends EventEmitter {
3279
4211
  this.flushing = false;
3280
4212
  }
3281
4213
  }
3282
- retryQueue() {
4214
+ /**
4215
+ * The entry the outbox sends next: the first one waiting, or answered but not taken out yet (a refused write is
4216
+ * answered and waits for nothing). A binary file's write whose content cannot be uploaded now (uploadHeld) waits, and
4217
+ * so does every write after it about one of the paths of a write that waits, so the writes of each path still go in
4218
+ * order; the writes of the other paths go ahead of it. An operation that waits, as one about such a path, holds every
4219
+ * write after it: it may give out ids the writes after it name (renumber). `wake` is when the first write that waits on
4220
+ * a time may go, undefined when none does.
4221
+ */
4222
+ nextEntry(entries) {
4223
+ const now = Date.now();
4224
+ const held = /* @__PURE__ */ new Set();
4225
+ let wake;
4226
+ for (const queued of entries) {
4227
+ if (this.answered.has(queued.id)) return {
4228
+ entry: queued,
4229
+ wake
4230
+ };
4231
+ if (!waiting(queued)) continue;
4232
+ const paths = entryPaths(queued);
4233
+ if (paths.some((path) => held.has(path))) {
4234
+ if (queued.kind === "op") return { wake };
4235
+ for (const path of paths) held.add(path);
4236
+ continue;
4237
+ }
4238
+ const until = queued.kind === "blob" ? this.uploadHeld(queued) : 0;
4239
+ if (until > now) {
4240
+ for (const path of paths) held.add(path);
4241
+ if (until !== Infinity) wake = Math.min(wake ?? until, until);
4242
+ continue;
4243
+ }
4244
+ return {
4245
+ entry: queued,
4246
+ wake
4247
+ };
4248
+ }
4249
+ return { wake };
4250
+ }
4251
+ /**
4252
+ * Until when binary file `entry`'s write waits for its content to be uploaded: 0 when it may go now, as one uploaded
4253
+ * already; Infinity while the client does not know whether the server keeps binary files, until the snapshot after
4254
+ * the probe sends the outbox again. On a server that keeps none it goes at once, to be dropped (uploadEntry).
4255
+ */
4256
+ uploadHeld(entry) {
4257
+ if (entry.sent !== void 0 || this.blobSupport === false) return 0;
4258
+ if (this.blobSupport === null) return Infinity;
4259
+ if (this.uploadedBlobs.has(entry.hash)) return 0;
4260
+ return Math.max(this.uploadHolds.get(entry.id) ?? 0, this.uploadAfter.get(entry.hash) ?? 0);
4261
+ }
4262
+ /** Send the outbox again in `after` ms, or shortly. */
4263
+ retryQueue(after) {
3283
4264
  clearTimeout(this.queueTimer);
3284
- if (!this.stopped && !this.closing) this.queueTimer = setTimeout(() => void this.track(this.flushQueue()), 1e3 + Math.random() * 1e3);
4265
+ if (!this.stopped && !this.closing) this.queueTimer = setTimeout(() => void this.track(this.flushQueue()), after ?? 1e3 + Math.random() * 1e3);
4266
+ }
4267
+ /**
4268
+ * Upload the content of binary file `entry` from blobs/, before its write is sent. Once uploaded, the write goes next.
4269
+ * One that cannot be uploaded now waits (uploadHeld) while the writes of other paths go ahead: the time the server
4270
+ * asked for while it deletes that content (Retry-After), or after a failure (uploadFailed). Content blobs/ no longer has
4271
+ * whole, or that the server found not of its sha256, is read again from the file each time, when the file still holds
4272
+ * it (stageHeld); when it does not, the write is dropped as superseded (supersede). On a server that keeps no binary
4273
+ * files, or that refuses the content as too large, the write is dropped and the file left as it is here (dropBlob).
4274
+ */
4275
+ async uploadEntry(entry) {
4276
+ if (this.blobSupport === false) return this.dropBlob(entry, () => this.skipBinary(entry.path, entry.hash));
4277
+ if (this.blobSupport === null) return;
4278
+ this.uploadAfter.delete(entry.hash);
4279
+ const staged = join(this.paths.blobs, entry.hash);
4280
+ if ((await stat(staged).catch(() => null))?.size !== entry.size) {
4281
+ rmSync(staged, { force: true });
4282
+ if (!await this.restage(entry)) return this.supersede(entry);
4283
+ }
4284
+ let result;
4285
+ try {
4286
+ result = await this.api.putBlob(entry.hash, staged, entry.size);
4287
+ } catch (error) {
4288
+ if (!(error instanceof Unreachable || error instanceof ServerError)) throw error;
4289
+ return this.uploadFailed(entry, error.message);
4290
+ }
4291
+ if (result.ok) {
4292
+ this.uploadedBlobs.add(entry.hash);
4293
+ this.uploadFailures.delete(entry.id);
4294
+ this.uploadHolds.delete(entry.id);
4295
+ this.trace(`uploaded ${entry.path} blob ${entry.hash.slice(0, 7)}`);
4296
+ return;
4297
+ }
4298
+ if ("retryAfter" in result) {
4299
+ this.log(`${entry.path}: the sync server is deleting the content ${entry.hash.slice(0, 7)}; uploading it again in ${result.retryAfter} s`);
4300
+ this.uploadAfter.set(entry.hash, Date.now() + result.retryAfter * 1e3);
4301
+ return;
4302
+ }
4303
+ this.log(`${entry.path}: the sync server refused its content ${entry.hash.slice(0, 7)}: ${result.message}`);
4304
+ switch (result.refused) {
4305
+ case "hash-mismatch":
4306
+ rmSync(staged, { force: true });
4307
+ if (!await this.restage(entry)) return this.supersede(entry);
4308
+ return this.uploadFailed(entry, result.message, true);
4309
+ case "too-large":
4310
+ this.oversized.add(`${entry.path}\0${entry.hash}`);
4311
+ return this.dropBlob(entry, () => this.skipLarge(entry.path, entry.hash, true));
4312
+ case "unsupported":
4313
+ this.blobSupport = false;
4314
+ return this.dropBlob(entry, () => this.skipBinary(entry.path, entry.hash));
4315
+ default: return this.uploadFailed(entry, result.message);
4316
+ }
4317
+ }
4318
+ /**
4319
+ * The upload of binary file `entry`'s content failed for `reason`: its write waits, while the writes of other paths go
4320
+ * ahead (nextEntry). The first BLOB_FAILURES - 1 failures in a row are tried again shortly, or `atOnce`, as content
4321
+ * read again after a hash-mismatch is; from the BLOB_FAILURES-th on, the wait doubles up to BLOB_BACKOFF_MAX_MS, and
4322
+ * the writer is told once for the write.
4323
+ */
4324
+ uploadFailed(entry, reason, atOnce = false) {
4325
+ const failures = (this.uploadFailures.get(entry.id) ?? 0) + 1;
4326
+ this.uploadFailures.set(entry.id, failures);
4327
+ if (failures < BLOB_FAILURES) {
4328
+ this.log(`${entry.path}: could not upload its content ${entry.hash.slice(0, 7)}: ${reason}; trying again`);
4329
+ if (!atOnce) this.uploadHolds.set(entry.id, Date.now() + BLOB_RETRY_MS * (1 + Math.random()));
4330
+ return;
4331
+ }
4332
+ const wait = Math.min(BLOB_BACKOFF_MS * 2 ** (failures - BLOB_FAILURES), BLOB_BACKOFF_MAX_MS);
4333
+ this.uploadHolds.set(entry.id, Date.now() + wait);
4334
+ this.log(`${entry.path}: could not upload its content ${entry.hash.slice(0, 7)} ${failures} times in a row: ${reason}; trying again in ${Math.round(wait / 1e3)} s`);
4335
+ this.tellOnce(`upload:${entry.id}`, {
4336
+ kind: "upload-failing",
4337
+ path: entry.path,
4338
+ message: `${entry.path}: the binary file written here could not be uploaded ${failures} times in a row (${reason}); it stays here, its write waits in the outbox and is tried again less and less often, while the writes of other files go ahead`
4339
+ });
4340
+ }
4341
+ /** Put the content of `entry` into blobs/ again from its file, when the file still holds it: false when it does not. */
4342
+ restage(entry) {
4343
+ return this.whenFree(() => this.locked(() => this.queue.some((queued) => queued.id === entry.id) && this.stageHeld(entry.path, entry.hash)));
4344
+ }
4345
+ /**
4346
+ * Take binary file `entry` out of the outbox without sending it, its file left as it is here: a server that keeps no
4347
+ * binary files, or not one this large. The client expects what the entry's base names there again (unqueueBlob), so
4348
+ * the file is a change still to send: the reconcile after it leaves it as it is (binarySkip), and sends it once the
4349
+ * server keeps binary files, as the snapshot of a later connection finds, or once it is small enough. `tell` tells the
4350
+ * writer, once for each content.
4351
+ */
4352
+ async dropBlob(entry, tell) {
4353
+ await this.unqueueBlob(entry);
4354
+ tell();
4355
+ this.schedule(entry.path);
4356
+ }
4357
+ /**
4358
+ * Take binary file `entry` out of the outbox once its content is nowhere: blobs/ lost it and the file holds another
4359
+ * since. The file's later content supersedes it, as a binary file keeps only its latest version: the reconcile after
4360
+ * it sends the file as it is now, on the entry's base (unqueueBlob). Recorded as information.
4361
+ */
4362
+ async supersede(entry) {
4363
+ await this.unqueueBlob(entry);
4364
+ this.info(entry.path, `the content ${entry.hash.slice(0, 7)} written here was no longer anywhere to upload, and the file holds another since; the file is sent as it is now`);
4365
+ this.schedule(entry.path);
4366
+ }
4367
+ /**
4368
+ * Take binary file `entry` out of the outbox unsent, as if it had never been queued: what the client expects at its
4369
+ * path goes back to the entry's base, and so do the bases of the writes after it on its content, in one hold of the
4370
+ * write lock.
4371
+ */
4372
+ async unqueueBlob(entry) {
4373
+ const back = entry.base === null ? null : BLOB_HASH.test(entry.base) ? {
4374
+ hash: entry.base,
4375
+ blob: true
4376
+ } : { hash: entry.base };
4377
+ await this.whenFree(() => this.editQueue((entries) => {
4378
+ const at = entries.findIndex((queued) => queued.id === entry.id);
4379
+ if (at < 0) return false;
4380
+ entries.splice(at, 1);
4381
+ for (const later of entries.slice(at)) {
4382
+ if (later.kind === "op" || later.path !== entry.path || later.sent !== void 0 || later.base !== entry.hash) continue;
4383
+ if (later.kind === "remove" && entry.base === null) continue;
4384
+ later.base = entry.base;
4385
+ }
4386
+ const disk = this.expected.get(entry.path);
4387
+ if (disk?.blob && disk.hash === entry.hash) this.expected.set(entry.path, back);
4388
+ }));
4389
+ this.uploadFailures.delete(entry.id);
4390
+ this.uploadHolds.delete(entry.id);
4391
+ }
4392
+ /**
4393
+ * The server refused binary file `entry` as `missing-blob`: it holds no such content, never given or cleaned up since.
4394
+ * The content is uploaded again and the write sent again as a new request, under a requestId of its own (requestIdOf),
4395
+ * since the server keeps the refusal under the one it was sent with.
4396
+ */
4397
+ async retryBlob(entry) {
4398
+ this.uploadedBlobs.delete(entry.hash);
4399
+ this.log(`${entry.path}: the sync server holds no content ${entry.hash.slice(0, 7)}; uploading it again`);
4400
+ await this.whenFree(() => this.editQueue((entries) => {
4401
+ const queued = entries.find((e) => e.id === entry.id);
4402
+ if (queued?.kind !== "blob") return false;
4403
+ queued.retry = (queued.retry ?? 0) + 1;
4404
+ delete queued.sent;
4405
+ delete queued.firstSentSeq;
4406
+ }));
3285
4407
  }
3286
4408
  /**
3287
4409
  * Take an answered entry out of the outbox, and give the ids the server renumbered to the files written here and
@@ -3352,15 +4474,16 @@ var SyncClient = class extends EventEmitter {
3352
4474
  */
3353
4475
  async request(entry, request, again, fresh = false) {
3354
4476
  const firstSentSeq = again ? entry.firstSentSeq : void 0;
4477
+ const requestId = requestIdOf(entry);
3355
4478
  this.trace(`send ${request.op} ${entry.id}${firstSentSeq === void 0 ? "" : ` again, first sent at seq ${firstSentSeq}`}`);
3356
4479
  let response;
3357
4480
  try {
3358
4481
  response = await this.api.send(firstSentSeq === void 0 ? {
3359
4482
  ...request,
3360
- requestId: entry.id
4483
+ requestId
3361
4484
  } : {
3362
4485
  ...request,
3363
- requestId: entry.id,
4486
+ requestId,
3364
4487
  firstSentSeq
3365
4488
  }, { fresh });
3366
4489
  } catch (error) {
@@ -3398,6 +4521,14 @@ var SyncClient = class extends EventEmitter {
3398
4521
  },
3399
4522
  now: requestTime()
3400
4523
  };
4524
+ if (entry.kind === "blob") return {
4525
+ protocol: 3,
4526
+ op: "file.blob",
4527
+ path: entry.path,
4528
+ hash: entry.hash,
4529
+ size: entry.size,
4530
+ base: entry.base
4531
+ };
3401
4532
  if (entry.replace) return {
3402
4533
  protocol: 3,
3403
4534
  op: "text.replace",
@@ -3414,7 +4545,7 @@ var SyncClient = class extends EventEmitter {
3414
4545
  await this.refuse(entry, [{
3415
4546
  kind: "conflict",
3416
4547
  path: entry.path,
3417
- text: entry.text,
4548
+ saves: true,
3418
4549
  message: (saved) => `${entry.path} was edited on top of a write the server did not take as it was, or the edit overlaps what the server merged into it; the text written here was saved to ${saved}`
3419
4550
  }]);
3420
4551
  }
@@ -3424,11 +4555,12 @@ var SyncClient = class extends EventEmitter {
3424
4555
  * the entry: what the writer is told, with the name each text is saved under, and the files that hold what the entry
3425
4556
  * wrote there as the client left it. Then each text is saved and the writer told. The entry is sent no more; it leaves
3426
4557
  * the outbox once no file holds what it wrote (releaseHeld), so a restart before the server's version is on disk puts
3427
- * that version back rather than send the refused text again. An operation's writer is answered with `outcome`.
4558
+ * that version back rather than send the refused text again. An operation's writer is answered with `outcome`. `saves`
4559
+ * says whether the entry wrote something at the path to save, a text or a binary file's content (tell).
3428
4560
  */
3429
4561
  async refuse(entry, told, outcome) {
3430
- const problems = told.map(({ kind, path, text, message }) => {
3431
- const saved = text === null ? void 0 : this.conflictName(path);
4562
+ const problems = told.map(({ kind, path, saves, message }) => {
4563
+ const saved = saves ? this.conflictName(path) : void 0;
3432
4564
  return {
3433
4565
  kind,
3434
4566
  path,
@@ -3466,7 +4598,9 @@ var SyncClient = class extends EventEmitter {
3466
4598
  for (const problem of refusal.told) {
3467
4599
  if (before(problem)) continue;
3468
4600
  const text = textAt(entry, problem.path);
3469
- if (problem.saved !== void 0 && typeof text === "string") await this.saveText(problem.saved, text).catch((error) => this.log(`${problem.path}: could not save ${problem.saved}: ${String(error)}`));
4601
+ const failed = (error) => this.log(`${problem.path}: could not save ${problem.saved}: ${String(error)}`);
4602
+ if (problem.saved !== void 0 && entry.kind === "blob") await this.saveStaged(problem.saved, entry.hash).catch(failed);
4603
+ else if (problem.saved !== void 0 && typeof text === "string") await this.saveText(problem.saved, text).catch(failed);
3470
4604
  this.problem(problem, entry.id);
3471
4605
  }
3472
4606
  if (refusal.outcome) this.settled(entry.id, {
@@ -3478,8 +4612,9 @@ var SyncClient = class extends EventEmitter {
3478
4612
  }
3479
4613
  /** Whether `path` holds what `entry` wrote there, as the client left it. */
3480
4614
  leftHere(entry, path) {
3481
- const text = wroteAt(entry, path);
3482
4615
  const disk = this.expected.get(path);
4616
+ if (entry.kind === "blob") return path === entry.path && disk?.blob === true && disk.hash === entry.hash;
4617
+ const text = wroteAt(entry, path);
3483
4618
  if (text === void 0 || disk === void 0) return false;
3484
4619
  return text === null ? disk === null : disk !== null && disk.hash === contentHash(text);
3485
4620
  }
@@ -3526,6 +4661,10 @@ var SyncClient = class extends EventEmitter {
3526
4661
  try {
3527
4662
  const response = await this.request(entry, request, again);
3528
4663
  if (response === null) return false;
4664
+ if (entry.kind === "blob" && !response.ok && response.reason === "missing-blob") {
4665
+ await this.retryBlob(entry);
4666
+ return false;
4667
+ }
3529
4668
  this.answer(entry);
3530
4669
  if (entry.kind === "op") await this.settleQueuedOp(entry, response);
3531
4670
  else if (entry.kind === "remove") await this.settleRemove(entry, response);
@@ -3535,6 +4674,7 @@ var SyncClient = class extends EventEmitter {
3535
4674
  base: entry.baseText,
3536
4675
  last: entry.replaced
3537
4676
  }, request.text);
4677
+ else if (entry.kind === "blob") await this.settleBlob(entry, response);
3538
4678
  return true;
3539
4679
  } finally {
3540
4680
  for (const path of paths) {
@@ -3550,7 +4690,7 @@ var SyncClient = class extends EventEmitter {
3550
4690
  const told = Object.entries(entry.written).flatMap(([path, text]) => text === null ? [] : [{
3551
4691
  kind: "refused",
3552
4692
  path,
3553
- text,
4693
+ saves: true,
3554
4694
  message: (saved) => `the server refused ${entry.request.op} (${response.message}); ${path} as it was written here was saved to ${saved} and the file has the server's version again`
3555
4695
  }]);
3556
4696
  return this.refuse(entry, told, {
@@ -3580,7 +4720,7 @@ var SyncClient = class extends EventEmitter {
3580
4720
  * The server gave some of the entry's provisional ids away: rewrite the files written offline, the queued requests
3581
4721
  * and edits after it, and what the client expects on disk (docs/sync-protocol.md). Called under the write lock with
3582
4722
  * the outbox as read, so a writer appending meanwhile sees either the old ids everywhere or the new. The outbox is
3583
- * sent in order, so every entry after it is still to be sent; one sent already (`sent`), as an older sync client left
4723
+ * sent in order for each path, so every entry after it about its files is still to be sent; one sent already (`sent`), as an older sync client left
3584
4724
  * behind an operation when it sent edits outside the outbox, stays as the server may have taken it, and so do the
3585
4725
  * files only it wrote; the server's version of them comes back once it is answered. An edit queued on top of it
3586
4726
  * keeps that text as its base, so its renumbering is what it changes, and it puts the server's id there when it
@@ -3590,7 +4730,7 @@ var SyncClient = class extends EventEmitter {
3590
4730
  * the sent write wrote it. A refused write is no write to send.
3591
4731
  */
3592
4732
  renumber(entries, entry, renumbered) {
3593
- const rest = entries.filter((queued) => queued.id !== entry.id && queued.sent === void 0 && waiting(queued));
4733
+ const rest = entries.slice(entries.findIndex((queued) => queued.id === entry.id) + 1).filter((queued) => queued.sent === void 0 && waiting(queued));
3594
4734
  const offlinePaths = [...entryPaths(entry), ...rest.flatMap(entryPaths)];
3595
4735
  this.renumberDisk(offlinePaths, renumbered);
3596
4736
  const files = (record) => Object.fromEntries(Object.entries(record).map(([path, text]) => {
@@ -3649,6 +4789,22 @@ var SyncClient = class extends EventEmitter {
3649
4789
  lock: (fn) => fn()
3650
4790
  };
3651
4791
  const seen = new Map([...new Set(paths)].filter((path) => this.syncs(path)).map((path) => [path, this.expected.get(path)]));
4792
+ const binary = [...seen].filter(([path]) => {
4793
+ try {
4794
+ return contentOf(readFileSync(join(this.paths.data, path))).kind === "binary";
4795
+ } catch {
4796
+ return false;
4797
+ }
4798
+ });
4799
+ for (const [path] of binary) seen.delete(path);
4800
+ for (const [path, disk] of binary) {
4801
+ const to = renumberPath(path, renumbered);
4802
+ if (to === path) continue;
4803
+ mkdirSync(dirname(join(this.paths.data, to)), { recursive: true });
4804
+ renameSync(join(this.paths.data, path), join(this.paths.data, to));
4805
+ this.expected.set(path, null);
4806
+ if (disk !== void 0) this.expected.set(to, disk);
4807
+ }
3652
4808
  const changes = renumberFiles(storage, seen.keys(), renumbered);
3653
4809
  const unknown = /* @__PURE__ */ new Set();
3654
4810
  for (const [path, disk] of seen) {
@@ -3684,30 +4840,204 @@ var SyncClient = class extends EventEmitter {
3684
4840
  this.reported = this.problems.length;
3685
4841
  return out;
3686
4842
  }
3687
- /** After a tool ran: find every change now, wait for the server, and return what went wrong. */
4843
+ /**
4844
+ * Send every change now, wait for the server, and return what went wrong: `mainflow sync flush`, and a hook of an
4845
+ * older CLI that names no tool. No change waits for an Edit's report (awaitEdit) any longer, nor starts to meanwhile.
4846
+ */
3688
4847
  async flush(timeoutMs = 2e4) {
3689
- await this.scan(true);
3690
- this.outboxChanged();
3691
- await this.idle(timeoutMs);
3692
- return {
3693
- problems: this.drain(),
3694
- queued: this.queued,
3695
- online: this.online
4848
+ this.sendingAll++;
4849
+ try {
4850
+ for (const wait of this.awaitingEdit.values()) wait.end(null);
4851
+ await this.scan(true);
4852
+ this.outboxChanged();
4853
+ await this.idle(timeoutMs);
4854
+ return {
4855
+ problems: this.drain(),
4856
+ queued: this.queued,
4857
+ online: this.online
4858
+ };
4859
+ } finally {
4860
+ this.sendingAll--;
4861
+ }
4862
+ }
4863
+ /**
4864
+ * After a tool other than an Edit ran, as its PostToolUse hook reports it: find every change now, wait for the server,
4865
+ * and return what went wrong. `path`, the file the tool wrote when it names one (Write), is no Edit: its change waiting
4866
+ * for an Edit's report is sent now, and one found meanwhile does not wait. `call`, the tool call when the hook names
4867
+ * it, ends the announcements of its session (settleCaller). A change of any other file waiting for an Edit's report,
4868
+ * which only an announced Edit's change does, as of another session sharing the sync client, goes on waiting for it:
4869
+ * the tool reported here did not write it, or may not have, as a Bash command, and the flush does not wait for it. Its
4870
+ * problems go to the caller after it ends (drain).
4871
+ */
4872
+ async flushTool(path, timeoutMs = 2e4, call) {
4873
+ this.settleCaller(call);
4874
+ if (path !== void 0) {
4875
+ this.sendingPaths.set(path, (this.sendingPaths.get(path) ?? 0) + 1);
4876
+ this.awaitingEdit.get(path)?.end(null);
4877
+ }
4878
+ try {
4879
+ await this.scan(true, false, false);
4880
+ this.outboxChanged();
4881
+ await this.idle(timeoutMs, true);
4882
+ return {
4883
+ problems: this.drain(),
4884
+ queued: this.queued,
4885
+ online: this.online
4886
+ };
4887
+ } finally {
4888
+ if (path !== void 0) {
4889
+ const left = (this.sendingPaths.get(path) ?? 1) - 1;
4890
+ if (left > 0) this.sendingPaths.set(path, left);
4891
+ else this.sendingPaths.delete(path);
4892
+ }
4893
+ }
4894
+ }
4895
+ /**
4896
+ * Before an Edit runs, as its PreToolUse hook announces it, with the tool call when the hook names it: the change it
4897
+ * makes, its one replacement of the version on disk, may wait for its report (awaitEdit). A change no Edit announced,
4898
+ * as a Bash command's, an editor's, or an Edit's whose hook does not announce it (an older CLI or forwarding script),
4899
+ * is sent at once. Several Edits of one file may be announced at a time, as by sessions sharing the sync client.
4900
+ * Announcements past announceTtlMs go here.
4901
+ */
4902
+ announceEdit(edit, call) {
4903
+ const now = Date.now();
4904
+ for (const [path, list] of this.announced) {
4905
+ const live = list.filter((a) => a.expires > now);
4906
+ if (live.length) this.announced.set(path, live);
4907
+ else this.announced.delete(path);
4908
+ }
4909
+ if (this.closing || !this.syncs(edit.path) || edit.old === "") return;
4910
+ const announcement = {
4911
+ edit: {
4912
+ path: edit.path,
4913
+ old: edit.old,
4914
+ new: edit.new
4915
+ },
4916
+ ...call ? { call } : {},
4917
+ left: this.opts.editReportMs ?? 3e3,
4918
+ expires: now + (this.opts.announceTtlMs ?? 6e5)
3696
4919
  };
4920
+ this.announced.set(edit.path, [...this.announced.get(edit.path) ?? [], announcement]);
4921
+ this.trace(`announce edit ${edit.path}`);
4922
+ }
4923
+ /** The announcement of `path`, live and with time left to wait, of an Edit that makes `to` of `from`. */
4924
+ announcementFor(path, from, to) {
4925
+ const now = Date.now();
4926
+ return this.announced.get(path)?.find((a) => a.expires > now && a.left > 0 && replaces(a.edit, from, to));
4927
+ }
4928
+ dropAnnouncement(path, announcement) {
4929
+ const list = this.announced.get(path)?.filter((a) => a !== announcement) ?? [];
4930
+ if (list.length) this.announced.set(path, list);
4931
+ else this.announced.delete(path);
4932
+ }
4933
+ /**
4934
+ * A tool call of `call`'s session, or of the same subagent in it, was reported: the Edits that session announced
4935
+ * before have run, and were reported, or never will, as one refused or failed, which runs no PostToolUse hook. A
4936
+ * session runs one such tool at a time, so their announcements are dropped, and a change waiting on one is sent now,
4937
+ * before the flush of this report waits for what it finds: a Bash command that made the change a refused Edit
4938
+ * announced hears of it at its own hook. Another session's announcements, or the session's subagents', stay.
4939
+ */
4940
+ settleCaller(call) {
4941
+ if (call === void 0) return;
4942
+ const dropped = /* @__PURE__ */ new Set();
4943
+ for (const [path, list] of this.announced) for (const announcement of list.filter((a) => sameCaller(a.call, call))) {
4944
+ dropped.add(announcement);
4945
+ this.dropAnnouncement(path, announcement);
4946
+ }
4947
+ for (const wait of this.awaitingEdit.values()) if (dropped.has(wait.announcement)) wait.end(null);
4948
+ if (dropped.size) this.trace(`dropped ${dropped.size} announcement(s) of ${call.session}${call.agent ? `/${call.agent}` : ""}`);
3697
4949
  }
3698
4950
  /**
3699
4951
  * After an Edit: replay the replacement on the server's latest version, so an Edit made while a broadcast was on
3700
- * its way is not mistaken for a rewrite of the older file. Anything else is flushed as usual.
4952
+ * its way is not mistaken for a rewrite of the older file, nor one of lines added a moment ago for a write of the
4953
+ * version before them. The report takes the Edit's announcement: the one of its tool call when the hook names it,
4954
+ * else the oldest of the same replacement. The rest of the session's announcements go (settleCaller). The change the
4955
+ * Edit made, waiting for this report (awaitEdit), is replayed where it waits. One not waiting yet, as when the
4956
+ * watcher's reconcile is still on its way to awaitEdit, takes the report from reportedEdits when it gets there, ahead
4957
+ * of the replay queued here on the path's chain; the report is dropped once that replay ran. Then the files are
4958
+ * flushed as for any tool that wrote this one.
3701
4959
  */
3702
- async flushEdit(edit, timeoutMs = 2e4) {
4960
+ async flushEdit(edit, timeoutMs = 2e4, call) {
3703
4961
  const { path } = edit;
3704
- if (this.caughtUp && this.syncs(path) && edit.old !== "") await this.withChain(path, () => this.replayEdit(edit));
3705
- return this.flush(timeoutMs);
4962
+ const list = this.announced.get(path);
4963
+ const announcement = call?.toolUseId !== void 0 ? list?.find((a) => sameCaller(a.call, call) && a.call?.toolUseId === call.toolUseId) : list?.find((a) => a.edit.old === edit.old && a.edit.new === edit.new);
4964
+ if (announcement) this.dropAnnouncement(path, announcement);
4965
+ const replays = this.caughtUp && this.syncs(path) && edit.old !== "";
4966
+ const waiting = replays ? this.awaitingEdit.get(path) : void 0;
4967
+ const handed = waiting !== void 0 && replaces(edit, waiting.from, waiting.to);
4968
+ if (handed) waiting.end(edit);
4969
+ this.settleCaller(call);
4970
+ if (replays && !handed) {
4971
+ this.reportedEdits.set(path, edit);
4972
+ await this.withChain(path, () => this.replayEdit(edit));
4973
+ if (this.reportedEdits.get(path) === edit) this.reportedEdits.delete(path);
4974
+ }
4975
+ return this.flushTool(path, timeoutMs, call);
4976
+ }
4977
+ /**
4978
+ * The Edit reported for the local `text` of `path`, which the client left holding `disk`, or null. An Edit starts
4979
+ * from the version on disk, and the watcher sees its write before the PostToolUse hook reports it. Layer 1 bases a
4980
+ * change on the recent version nearest to it, which for an Edit that removes or rewrites lines added a moment ago is
4981
+ * the version before them, as for a program that read that version and wrote it back: the lines would be kept, or
4982
+ * the change refused as overlapping them.
4983
+ * So a change that is exactly the replacement of an Edit announced before it ran (announceEdit), when layer 1 would
4984
+ * base it on another version than the one on disk, waits for the report: the Edit's (flushEdit) is replayed in its
4985
+ * place; a report of the tool that wrote this file (flushTool), flush(), the announcement's time running out or stop()
4986
+ * sends it as layer 1 bases it, as does a later report of the session that announced it (settleCaller). The waits on
4987
+ * one announcement add up to editReportMs at most: once used up, it is dropped. Any other change does not wait: no
4988
+ * Edit named it before it ran, so no report is due for it.
4989
+ * A report that came first (reportedEdits) is taken at once when the Edit it names made this text; while there is one,
4990
+ * no change of the path starts to wait, as its report came already.
4991
+ * Nor does a change wait unless it may be an Edit's replay: the client caught up, as flushEdit replays only then, and
4992
+ * no write about the file waiting for the server (pending). An answer to one changes the versions layer 1 chooses
4993
+ * from, so a change that waited past it would be based on another version than the one it was read against. Nor does
4994
+ * a change found while every change is being sent (flush), or one of a file a tool reported as written meanwhile.
4995
+ */
4996
+ async awaitEdit(path, text, disk) {
4997
+ if (disk.text === void 0 || this.closing) return null;
4998
+ const from = disk.text;
4999
+ const reported = this.reportedEdits.get(path);
5000
+ if (reported) {
5001
+ if (!replaces(reported, from, text)) return null;
5002
+ this.reportedEdits.delete(path);
5003
+ this.trace(`await edit ${path} ${short(text)} reported before it waited`);
5004
+ return reported;
5005
+ }
5006
+ const announcement = this.announcementFor(path, from, text);
5007
+ if (!announcement || !this.caughtUp || this.pending(path) || this.sendingAll > 0 || this.sendingPaths.has(path)) return null;
5008
+ const start = this.startOf(path, text, disk);
5009
+ if (start === void 0 || start.text === from) return null;
5010
+ this.trace(`await edit ${path} ${short(text)} on disk ${short(from)} nearest ${short(start.text)} up to ${announcement.left} ms`);
5011
+ const started = Date.now();
5012
+ let timedOut = false;
5013
+ const edit = await new Promise((resolve) => {
5014
+ const wait = {
5015
+ from,
5016
+ to: text,
5017
+ announcement,
5018
+ end: (edit) => {
5019
+ clearTimeout(timer);
5020
+ if (this.awaitingEdit.get(path) === wait) this.awaitingEdit.delete(path);
5021
+ resolve(edit);
5022
+ }
5023
+ };
5024
+ const timer = setTimeout(() => {
5025
+ timedOut = true;
5026
+ wait.end(null);
5027
+ }, announcement.left);
5028
+ this.awaitingEdit.get(path)?.end(null);
5029
+ this.awaitingEdit.set(path, wait);
5030
+ });
5031
+ announcement.left -= Date.now() - started;
5032
+ if (timedOut || announcement.left <= 0) this.dropAnnouncement(path, announcement);
5033
+ this.trace(`await edit ${path} ${short(text)} ${edit ? "reported as an Edit" : "over"}`);
5034
+ return edit;
3706
5035
  }
3707
5036
  /**
3708
5037
  * Queue an Edit as its replacement, which the server makes again on its latest version, based as layer 1 knows the
3709
- * version the Edit changed, the one on disk. One the watcher queued as a whole text already, or that is not the plain
3710
- * replacement of the version on disk, is left to the flush. The Edit's result is adopted in the hold of the write lock
5038
+ * version the Edit changed, the one on disk. The replacement sent is the text the Edit replaced there (editTarget),
5039
+ * with the line break after a deleted line when the Edit took it. One the watcher queued as a whole text already, or
5040
+ * that is not that replacement of the version on disk, is left to the flush. The Edit's result is adopted in the hold of the write lock
3711
5041
  * that queues it (I2); one changed there since it was read, as by an operation written on it, which then has the Edit
3712
5042
  * queued ahead of it as a direct edit, is reconciled again.
3713
5043
  */
@@ -3715,7 +5045,11 @@ var SyncClient = class extends EventEmitter {
3715
5045
  const disk = this.expected.get(path);
3716
5046
  const local = await this.readLocal(path);
3717
5047
  if (local.kind !== "text" || disk?.text === void 0 || contentHash(local.text) === disk.hash) return;
3718
- if (disk.text.split(old).length !== 2 || disk.text.replace(old, () => replacement) !== local.text) return;
5048
+ const target = editTarget(disk.text, {
5049
+ old,
5050
+ new: replacement
5051
+ });
5052
+ if (target === null || disk.text.replace(target, () => replacement) !== local.text) return;
3719
5053
  const edited = local.text;
3720
5054
  const { hash, text } = disk;
3721
5055
  const id = await this.enqueue((entries) => {
@@ -3731,7 +5065,7 @@ var SyncClient = class extends EventEmitter {
3731
5065
  return {
3732
5066
  ...editOn(path, edited, start, hash),
3733
5067
  replace: {
3734
- old,
5068
+ old: target,
3735
5069
  new: replacement
3736
5070
  }
3737
5071
  };
@@ -3759,7 +5093,7 @@ var SyncClient = class extends EventEmitter {
3759
5093
  await this.refuse(entry, [{
3760
5094
  kind: "edit-rejected",
3761
5095
  path,
3762
- text: writer,
5096
+ saves: true,
3763
5097
  message: (saved) => `${path}: the Edit could not be replayed on the latest version (${response.message}); the edited text was saved to ${saved} and the file now has the latest version. Read it and edit again`
3764
5098
  }]);
3765
5099
  if ("files" in response && response.files.some((f) => f.path === path)) await this.delivery(path, response.seq);
@@ -3826,9 +5160,10 @@ var SyncClient = class extends EventEmitter {
3826
5160
  /**
3827
5161
  * Compare every synced file's size, modification time and inode with the last scan and reconcile what changed,
3828
5162
  * plus every file expected on disk that is gone. A file modified in the last moment may still be being written,
3829
- * so it waits for the next scan unless `immediate`. `baseline` only records what is there.
5163
+ * so it waits for the next scan unless `immediate`. `baseline` only records what is there. An immediate scan waits for
5164
+ * the reconciles it started unless `settle` is false, for a caller that waits for them itself (flushTool).
3830
5165
  */
3831
- async scan(immediate, baseline = false) {
5166
+ async scan(immediate, baseline = false, settle = immediate) {
3832
5167
  const seen = /* @__PURE__ */ new Set();
3833
5168
  for (const path of await this.walk()) {
3834
5169
  seen.add(path);
@@ -3845,7 +5180,7 @@ var SyncClient = class extends EventEmitter {
3845
5180
  this.signatures.delete(path);
3846
5181
  this.schedule(path);
3847
5182
  }
3848
- if (immediate) await Promise.allSettled(this.chains.values());
5183
+ if (settle) await Promise.allSettled(this.chains.values());
3849
5184
  }
3850
5185
  /**
3851
5186
  * Watch the directories that may hold synced files, and local/sync/ for the outbox. The client's own files in the
@@ -3930,7 +5265,7 @@ var SyncClient = class extends EventEmitter {
3930
5265
  const name = `${path.split("/").join("__")}.${(/* @__PURE__ */ new Date()).toISOString().replace(/[:.]/g, "-")}.${Math.random().toString(36).slice(2, 6)}`;
3931
5266
  return relative(this.paths.data, join(this.paths.conflicts, name)).split(sep).join("/");
3932
5267
  }
3933
- /** Save `text` as `saved`, relative to .mainflow/; a text saved there already is kept. */
5268
+ /** Save `text`, or a binary file's bytes, as `saved`, relative to .mainflow/; a text saved there already is kept. */
3934
5269
  async saveText(saved, text) {
3935
5270
  await mkdir(this.paths.conflicts, { recursive: true });
3936
5271
  await writeFile(join(this.paths.data, saved), text, { flag: "wx" }).catch((error) => {
@@ -3942,16 +5277,30 @@ var SyncClient = class extends EventEmitter {
3942
5277
  await this.saveText(saved, text);
3943
5278
  return saved;
3944
5279
  }
5280
+ /** Save the content of sha256 `hash` that blobs/ keeps as `saved`, relative to .mainflow/; one saved there already is kept. */
5281
+ async saveStaged(saved, hash) {
5282
+ await mkdir(this.paths.conflicts, { recursive: true });
5283
+ try {
5284
+ copyFileSync(join(this.paths.blobs, hash), join(this.paths.data, saved), COPYFILE_EXCL);
5285
+ } catch (error) {
5286
+ if (error.code !== "EEXIST") throw error;
5287
+ }
5288
+ }
3945
5289
  saveKnown() {
3946
5290
  if (this.knownTimer) return;
3947
5291
  this.knownTimer = setTimeout(() => this.saveKnownNow(), 200);
3948
5292
  }
5293
+ /** What known.json names: each confirmed version, a binary file's sha256 with BLOB_PREFIX. */
5294
+ knownFiles() {
5295
+ const files = {};
5296
+ for (const [path, disk] of this.confirmed) if (disk) files[path] = disk.blob ? { hash: `${BLOB_PREFIX}${disk.hash}` } : disk;
5297
+ return files;
5298
+ }
3949
5299
  saveKnownNow(final = false) {
3950
5300
  if (this.abandoned && !final) return;
3951
5301
  clearTimeout(this.knownTimer);
3952
5302
  this.knownTimer = void 0;
3953
- const files = {};
3954
- for (const [path, disk] of this.confirmed) if (disk) files[path] = disk;
5303
+ const files = this.knownFiles();
3955
5304
  try {
3956
5305
  underLock(this.root, () => writeKnown(this.root, files));
3957
5306
  } catch (error) {
@@ -4049,6 +5398,26 @@ function forgetDaemon(root, gone) {
4049
5398
  rmSync(syncPaths(root).daemon, { force: true });
4050
5399
  return true;
4051
5400
  }
5401
+ /** The Edit a request's body names, its file and its one replacement, or null. */
5402
+ function editIn(body) {
5403
+ const edit = body.edit;
5404
+ if (typeof edit?.path !== "string" || typeof edit.old !== "string" || typeof edit.new !== "string") return null;
5405
+ return {
5406
+ path: edit.path,
5407
+ old: edit.old,
5408
+ new: edit.new
5409
+ };
5410
+ }
5411
+ /** The tool call a request's body names, its session at least, or undefined. */
5412
+ function callIn(body) {
5413
+ const call = body.call;
5414
+ if (typeof call?.session !== "string" || call.session === "") return void 0;
5415
+ return {
5416
+ session: call.session,
5417
+ ...typeof call.agent === "string" ? { agent: call.agent } : {},
5418
+ ...typeof call.toolUseId === "string" ? { toolUseId: call.toolUseId } : {}
5419
+ };
5420
+ }
4052
5421
  /** Serve `client` to this machine's mainflow commands until closed. */
4053
5422
  async function serveIpc(root, client) {
4054
5423
  const secret = randomBytes(16).toString("hex");
@@ -4070,8 +5439,14 @@ async function serveIpc(root, client) {
4070
5439
  return client.outboxChanged();
4071
5440
  }
4072
5441
  if (req.url === "/flush") {
4073
- const edit = body.edit;
4074
- return reply(200, edit ? await client.flushEdit(edit) : await client.flush());
5442
+ const edit = editIn(body);
5443
+ const call = callIn(body);
5444
+ return reply(200, edit ? await client.flushEdit(edit, void 0, call) : body.tool === true ? await client.flushTool(typeof body.path === "string" ? body.path : void 0, void 0, call) : await client.flush());
5445
+ }
5446
+ if (req.url === "/announce") {
5447
+ const edit = editIn(body);
5448
+ if (edit) client.announceEdit(edit, callIn(body));
5449
+ return reply(200, {});
4075
5450
  }
4076
5451
  if (req.url === "/status") return reply(200, {
4077
5452
  online: client.online,
@@ -4142,6 +5517,18 @@ async function startDaemon(options) {
4142
5517
  }
4143
5518
  //#endregion
4144
5519
  //#region ../sync/dist/hook.js
5520
+ /**
5521
+ * The tool call `payload` is of, as the sync client tells sessions apart: the session, the subagent in it for a
5522
+ * subagent's call (agent_id), and the call's id (tool_use_id), which both hooks of one call name. None without a session.
5523
+ */
5524
+ function toolCallOf(payload) {
5525
+ if (typeof payload.session_id !== "string" || payload.session_id === "") return void 0;
5526
+ return {
5527
+ session: payload.session_id,
5528
+ ...typeof payload.agent_id === "string" && payload.agent_id !== "" ? { agent: payload.agent_id } : {},
5529
+ ...typeof payload.tool_use_id === "string" && payload.tool_use_id !== "" ? { toolUseId: payload.tool_use_id } : {}
5530
+ };
5531
+ }
4145
5532
  /** Tools that write the one file named in file_path. */
4146
5533
  const FILE_TOOLS = /* @__PURE__ */ new Set([
4147
5534
  "Edit",
@@ -4166,26 +5553,37 @@ function hubPath(root, payload) {
4166
5553
  return path;
4167
5554
  }
4168
5555
  /**
5556
+ * The one replacement an Edit of `path` makes, as both its hooks name it, the PreToolUse one announcing it and the
5557
+ * PostToolUse one reporting it; null for an Edit of every occurrence, which the sync client does not replay.
5558
+ */
5559
+ function editOf(path, payload) {
5560
+ const input = payload.tool_input ?? {};
5561
+ if (payload.tool_name !== "Edit" || input.replace_all === true || typeof input.old_string !== "string" || typeof input.new_string !== "string") return null;
5562
+ return {
5563
+ path,
5564
+ old: input.old_string,
5565
+ new: input.new_string
5566
+ };
5567
+ }
5568
+ /**
4169
5569
  * What to flush after the tool in `payload` ran in the hub at `root`. A file tool outside the hub's synced files
4170
- * needs nothing; an Edit of one replacement is replayed on the latest version; anything else, such as a Bash
4171
- * command that may have written any file, rescans every file.
5570
+ * needs nothing; an Edit of one replacement is replayed on the latest version; another file tool names the file it
5571
+ * wrote; anything else, such as a Bash command that may have written any file, rescans every file and names none.
4172
5572
  */
4173
5573
  function planFlush(root, payload) {
4174
5574
  const tool = payload.tool_name ?? "";
4175
5575
  if (!FILE_TOOLS.has(tool)) return { skip: false };
4176
- const input = payload.tool_input ?? {};
4177
- if (typeof input.file_path !== "string") return { skip: false };
5576
+ if (typeof (payload.tool_input ?? {}).file_path !== "string") return { skip: false };
4178
5577
  const path = hubPath(root, payload);
4179
5578
  if (path === null || !isSynced(path)) return { skip: true };
4180
- if (tool === "Edit" && input.replace_all !== true && typeof input.old_string === "string" && typeof input.new_string === "string") return {
5579
+ const edit = editOf(path, payload);
5580
+ return edit ? {
4181
5581
  skip: false,
4182
- edit: {
4183
- path,
4184
- old: input.old_string,
4185
- new: input.new_string
4186
- }
5582
+ edit
5583
+ } : {
5584
+ skip: false,
5585
+ path
4187
5586
  };
4188
- return { skip: false };
4189
5587
  }
4190
5588
  /** The PostToolUse answer: block with the problems as the reason, or nothing when the change went through as written. */
4191
5589
  function hookAnswer(result) {
@@ -4214,6 +5612,17 @@ function planFetch(root, payload) {
4214
5612
  write: tool !== "Read"
4215
5613
  };
4216
5614
  }
5615
+ /**
5616
+ * The Edit to announce to the sync client before it runs in the hub at `root`: one replacement of a synced file, whose
5617
+ * change then waits for the Edit's PostToolUse report (docs/sync.md "等 Edit 的回報"). Null for anything else, which
5618
+ * announces nothing: a change no Edit announced is sent at once.
5619
+ */
5620
+ function planAnnounce(root, payload) {
5621
+ const path = hubPath(root, payload);
5622
+ if (path === null || !isSynced(path)) return null;
5623
+ const edit = editOf(path, payload);
5624
+ return edit && edit.old !== "" ? edit : null;
5625
+ }
4217
5626
  /** The PreToolUse answer that stops the tool, with `reason` for the agent. */
4218
5627
  const fetchDenial = (reason) => JSON.stringify({ hookSpecificOutput: {
4219
5628
  hookEventName: "PreToolUse",
@@ -4493,4 +5902,4 @@ function hubWriter(root, env = process.env, options = {}) {
4493
5902
  };
4494
5903
  }
4495
5904
  //#endregion
4496
- export { hubName as C, isSynced as D, SYNC_DIR as E, syncPaths as O, earlierToken as S, SETTINGS_FILE as T, readRemote as _, remoteMode as a, ServerError as b, planFetch as c, DaemonRunning as d, DaemonUnavailable as f, silentHint as g, readDaemon as h, queuedCount as i, planFlush as l, probeDaemon as m, hubWriter as n, fetchDenial as o, callDaemon as p, writtenLines as r, hookAnswer as s, CONFIRM_MS as t, startDaemon as u, oversizedFiles as v, syncSettings as w, ENV as x, ServerApi as y };
5905
+ export { syncPaths as A, ENV as C, SETTINGS_FILE as D, syncSettings as E, SYNC_DIR as O, ServerError as S, hubName as T, readDaemon as _, remoteMode as a, oversizedFiles as b, planAnnounce as c, toolCallOf as d, startDaemon as f, probeDaemon as g, callDaemon as h, queuedCount as i, isSynced as k, planFetch as l, DaemonUnavailable as m, hubWriter as n, fetchDenial as o, DaemonRunning as p, writtenLines as r, hookAnswer as s, CONFIRM_MS as t, planFlush as u, silentHint as v, earlierToken as w, ServerApi as x, readRemote as y };