mainflow 0.30.0-alpha.1 → 0.31.0-alpha.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,4 +1,4 @@
1
- import { $ as AUTHORIZATION, F as renumberFiles, I as renumberPath, L as renumberText, N as applyOffline, P as renumberRequest, Q as nodeStorage, R as scopeOfPath, X as nowMinute, at as ConflictError, ct as contentHash, et as ROUTES, ot as DATA_DIR, st as WriteError, tt as requestTime, z as applyOperation } from "./host-DmVNYRql.js";
1
+ import { $ as ACCESS_HEADERS, F as renumberFiles, I as renumberPath, L as renumberText, N as applyOffline, P as renumberRequest, Q as nodeStorage, R as scopeOfPath, X as nowMinute, ct as DATA_DIR, dt as splitFrontMatter, et as ROUTES, lt as WriteError, nt as comparePaths, rt as requestTime, st as ConflictError, tt as SOCKET_MAX_AGE_CLOSE, ut as contentHash, z as applyOperation } from "./host-CdB_vDY_.js";
2
2
  import { createHash, randomBytes } from "node:crypto";
3
3
  import { accessSync, appendFileSync, chmodSync, closeSync, constants, copyFileSync, existsSync, linkSync, lstatSync, mkdirSync, openSync, readFileSync, readdirSync, renameSync, rmSync, statSync, watch, writeFileSync } from "node:fs";
4
4
  import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
@@ -6,11 +6,139 @@ import { fileURLToPath } from "node:url";
6
6
  import { execFileSync, spawn, spawnSync } from "node:child_process";
7
7
  import { parse as parse$1 } from "yaml";
8
8
  import { homedir } from "node:os";
9
+ import { createBrotliDecompress, createGunzip, createInflate, createZstdDecompress } from "node:zlib";
9
10
  import { Agent, createServer, request } from "node:http";
10
11
  import { Agent as Agent$1, request as request$1 } from "node:https";
11
12
  import { EventEmitter } from "node:events";
12
13
  import { appendFile, mkdir, readFile, readdir, rm, stat, writeFile } from "node:fs/promises";
13
14
  import { diff3Merge } from "node-diff3";
15
+ //#region ../sync/dist/paths.js
16
+ /** The sync client's own directory, inside local/ so it is neither synced nor committed. */
17
+ const SYNC_DIR = "local/sync";
18
+ /** The settings a machine keeps instead of the environment variables: { url, clientId, clientSecret }. Never synced or committed. */
19
+ const SETTINGS_FILE = "local/sync.json";
20
+ const syncPaths = (root) => {
21
+ const data = join(root, DATA_DIR);
22
+ const own = join(data, SYNC_DIR);
23
+ return {
24
+ data,
25
+ own,
26
+ tmp: join(own, "tmp"),
27
+ conflicts: join(own, "conflicts"),
28
+ known: join(own, "known.json"),
29
+ /** The files only on the server, the artifacts of closed mainlines not downloaded here, by their content hash */
30
+ remote: join(own, "remote.json"),
31
+ /** The text of each version known.json names, in a file named by its content hash */
32
+ knownTexts: join(own, "known"),
33
+ queue: join(own, "queue.json"),
34
+ notices: join(own, "notices.jsonl"),
35
+ daemon: join(own, "daemon.json"),
36
+ /** The sync client a mainflow write last started in the background: { pid, at } */
37
+ started: join(own, "started.json"),
38
+ /** What a sync client started in the background prints */
39
+ log: join(own, "sync.log")
40
+ };
41
+ };
42
+ /** Names programs leave behind while writing: editor swap and backup files, temporary files, Finder's. */
43
+ const TRANSIENT = [
44
+ /\.tmp$/,
45
+ /\.sw[a-p]$/,
46
+ /\.swx$/,
47
+ /~$/,
48
+ /^\.#/,
49
+ /^4913$/,
50
+ /\.crswap$/,
51
+ /^\.DS_Store$/
52
+ ];
53
+ /** Whether a file at `path` (relative to .mainflow/) is synced. */
54
+ function isSynced(path) {
55
+ const parts = path.split("/");
56
+ if (parts.some((part) => part === "" || part === "." || part === "..")) return false;
57
+ if (parts[0] === "local" && !(parts[1] === "sessions" && parts.length === 3)) return false;
58
+ const name = parts.at(-1);
59
+ return !TRANSIENT.some((pattern) => pattern.test(name));
60
+ }
61
+ /** Whether the directory at `path` may hold synced files, so a scan walks into it. */
62
+ const mayHoldSynced = (path) => path !== "local" ? !path.startsWith("local/") || path === "local/sessions" : true;
63
+ /**
64
+ * Whether the client's file watcher watches inside the directory at `path`: one that may hold synced files, and
65
+ * local/sync/ itself for the outbox; the directories under local/sync/ hold only the client's own files.
66
+ */
67
+ const watchedDir = (path) => path === "local/sync" || mayHoldSynced(path);
68
+ /** An artifact of a mainline or of one of its tasks: `mainlines/<mainline>/artifacts/...` or `.../tasks/<task>/artifacts/...`. */
69
+ const ARTIFACT = /^mainlines\/([^/]+)\/(?:tasks\/[^/]+\/)?artifacts\/./;
70
+ /**
71
+ * The directory of the mainline whose artifact `path` is, such as `m001-sync`; null for any other path. Only such a file
72
+ * may stay on the server until it is used, when its mainline is closed (docs/sync.md "按需下載").
73
+ */
74
+ const artifactOf = (path) => ARTIFACT.exec(path)?.[1] ?? null;
75
+ //#endregion
76
+ //#region ../sync/dist/settings.js
77
+ const ENV = {
78
+ url: "MAINFLOW_SYNC_URL",
79
+ clientId: "MAINFLOW_SYNC_CLIENT_ID",
80
+ clientSecret: "MAINFLOW_SYNC_CLIENT_SECRET",
81
+ onDemand: "MAINFLOW_SYNC_ON_DEMAND"
82
+ };
83
+ /** The values of MAINFLOW_SYNC_ON_DEMAND that turn it on; any other value that is not empty turns it off. */
84
+ const ON = /* @__PURE__ */ new Set([
85
+ "1",
86
+ "true",
87
+ "on"
88
+ ]);
89
+ /** The hub's name as config.yaml gives it. */
90
+ function hubName(root) {
91
+ const config = parse$1(readFileSync(join(root, DATA_DIR, "config.yaml"), "utf8"));
92
+ if (typeof config?.hub !== "string") throw new Error(`${join(root, DATA_DIR, "config.yaml")} names no hub`);
93
+ return config.hub;
94
+ }
95
+ const text = (value) => typeof value === "string" ? value : "";
96
+ /** What .mainflow/local/sync.json holds, nothing when there is no such file. */
97
+ function stored(root) {
98
+ const file = join(root, DATA_DIR, SETTINGS_FILE);
99
+ if (!existsSync(file)) return {};
100
+ try {
101
+ return JSON.parse(readFileSync(file, "utf8")) ?? {};
102
+ } catch (error) {
103
+ throw new Error(`${file} is not JSON: ${error.message}`);
104
+ }
105
+ }
106
+ /**
107
+ * The sync settings of the hub at `root`, or null when the server or either half of the token is missing. Each value
108
+ * comes from its environment variable, or from sync.json when the variable is unset or empty.
109
+ */
110
+ function syncSettings(root, env = process.env) {
111
+ const saved = stored(root);
112
+ const url = env[ENV.url] || text(saved.url);
113
+ const clientId = env[ENV.clientId] || text(saved.clientId);
114
+ const clientSecret = env[ENV.clientSecret] || text(saved.clientSecret);
115
+ if (!url || !clientId || !clientSecret) return null;
116
+ const onDemand = env[ENV.onDemand] ? ON.has(env[ENV.onDemand].trim().toLowerCase()) : saved.onDemand === true;
117
+ return {
118
+ url: url.replace(/\/+$/, ""),
119
+ clientId,
120
+ clientSecret,
121
+ hub: hubName(root),
122
+ onDemand
123
+ };
124
+ }
125
+ /** The environment variable and the sync.json key that held the token before Cloudflare Access let requests through. */
126
+ const EARLIER_TOKEN = {
127
+ env: "MAINFLOW_SYNC_TOKEN",
128
+ key: "token"
129
+ };
130
+ /**
131
+ * What to tell the user when the hub at `root` has no sync settings but keeps the token of an earlier version, which
132
+ * nothing reads now: where that token is and which two values to set instead, never the token itself. Null when the
133
+ * settings are whole or no earlier token is kept, so a hub that never synced writes to its local files without a word.
134
+ */
135
+ function earlierToken(root, env = process.env) {
136
+ if (syncSettings(root, env)) return null;
137
+ const where = [...env[EARLIER_TOKEN.env] ? [EARLIER_TOKEN.env] : [], ...text(stored(root)[EARLIER_TOKEN.key]) ? [`the ${EARLIER_TOKEN.key} key of .mainflow/${SETTINGS_FILE}`] : []];
138
+ if (where.length === 0) return null;
139
+ return `${where.join(" and ")} ${where.length > 1 ? "are" : "is"} no longer read, so mainflow does not write through the sync server; set ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}, to a Cloudflare Access service token`;
140
+ }
141
+ //#endregion
14
142
  //#region ../sync/dist/api.js
15
143
  /**
16
144
  * The server could not be reached, or the connection broke. `sent` says whether the request may have reached the
@@ -27,7 +155,7 @@ var Unreachable = class extends Error {
27
155
  this.broken = broken;
28
156
  }
29
157
  };
30
- /** The server refused before the hub (401, 400, 404) or failed (500). */
158
+ /** The server refused before the hub (400, 404) or failed (500), or Cloudflare Access in front of it refused (AccessRefused). */
31
159
  var ServerError = class extends Error {
32
160
  status;
33
161
  constructor(message, status) {
@@ -35,6 +163,8 @@ var ServerError = class extends Error {
35
163
  this.status = status;
36
164
  }
37
165
  };
166
+ /** Cloudflare Access refused the request: no service token, a wrong or revoked one, or a policy that does not admit it. */
167
+ var AccessRefused = class extends ServerError {};
38
168
  /** Error codes of a connection that never carried the request. */
39
169
  const NOT_SENT = /* @__PURE__ */ new Set([
40
170
  "ECONNREFUSED",
@@ -55,6 +185,16 @@ const STALE = /* @__PURE__ */ new Set([
55
185
  const WARM_MS = 1e4;
56
186
  /** How long keeping the connection open waits for its answer. */
57
187
  const WARM_TIMEOUT_MS = 5e3;
188
+ /** Snapshots in a row whose batches went back in seq, as when the server's storage was replaced midway, before giving up */
189
+ const SNAPSHOT_ATTEMPTS = 3;
190
+ /** The encodings an answer may come in, by Content-Encoding, best first; Cloudflare's free plan sends Zstandard. */
191
+ const DECODERS = {
192
+ zstd: createZstdDecompress,
193
+ br: createBrotliDecompress,
194
+ gzip: createGunzip,
195
+ deflate: createInflate
196
+ };
197
+ const ACCEPT_ENCODING = Object.keys(DECODERS).join(", ");
58
198
  const causeCode = (error) => {
59
199
  for (let cause = error; cause; cause = cause.cause) if (typeof cause.code === "string") return cause.code;
60
200
  };
@@ -66,35 +206,56 @@ var ServerApi = class {
66
206
  * connection at a time, since the client sends one write at a time; a request waits for the one before it.
67
207
  */
68
208
  agent;
209
+ /** The connections snapshots go on, apart from the writes', so a snapshot never waits behind a write or holds one up */
210
+ reads;
69
211
  /** Requests on `agent` not answered yet */
70
212
  busy = 0;
213
+ transfer = {
214
+ batches: 0,
215
+ files: 0,
216
+ wire: 0,
217
+ body: 0,
218
+ sent: 0,
219
+ listed: 0
220
+ };
71
221
  constructor(settings, timeoutMs = 3e4) {
72
222
  this.settings = settings;
73
223
  this.timeoutMs = timeoutMs;
74
- const options = {
224
+ const Agent$2 = settings.url.startsWith("https:") ? Agent$1 : Agent;
225
+ this.agent = new Agent$2({
75
226
  keepAlive: true,
76
227
  maxSockets: 1,
77
228
  maxFreeSockets: 1
78
- };
79
- this.agent = settings.url.startsWith("https:") ? new Agent$1(options) : new Agent(options);
229
+ });
230
+ this.reads = new Agent$2({
231
+ keepAlive: true,
232
+ maxSockets: 2,
233
+ maxFreeSockets: 1
234
+ });
80
235
  }
81
- get headers() {
236
+ /** The service token's headers, which every request and the WebSocket's upgrade carry. Never logged. */
237
+ get access() {
82
238
  return {
83
- [AUTHORIZATION]: `Bearer ${this.settings.token}`,
84
- "content-type": "application/json"
239
+ [ACCESS_HEADERS.clientId]: this.settings.clientId,
240
+ [ACCESS_HEADERS.clientSecret]: this.settings.clientSecret
85
241
  };
86
242
  }
87
- async call(method, route, body, fresh = false) {
243
+ async call(method, route, body, connection) {
88
244
  const url = `${this.settings.url}${route}`;
245
+ const text = JSON.stringify(body);
89
246
  let answer;
90
247
  try {
91
- answer = method === "GET" ? await this.fetchGet(url) : await this.write(method, url, JSON.stringify(body), fresh);
248
+ answer = connection === "read" ? await this.read(method, url, text) : await this.write(method, url, text, connection === "fresh");
92
249
  } catch (error) {
93
250
  const code = causeCode(error);
94
251
  const detail = [code, error.cause?.message ?? error.message].filter(Boolean).join(": ");
95
252
  const sent = !(code && NOT_SENT.has(code));
96
253
  throw new Unreachable(`cannot reach the sync server at ${this.settings.url}${detail ? ` (${detail})` : ""}`, sent, sent && STALE.has(code ?? ""));
97
254
  }
255
+ if (answer.status === 401 || answer.status === 403 || answer.status >= 300 && answer.status < 400) {
256
+ const to = answer.location ? ` to ${new URL(answer.location, url).origin}` : "";
257
+ throw new AccessRefused(`Cloudflare Access in front of the sync server refused the request (HTTP ${answer.status}${to}); check the service token in ${ENV.clientId} and ${ENV.clientSecret}, or clientId and clientSecret in .mainflow/${SETTINGS_FILE}`, answer.status);
258
+ }
98
259
  let parsed;
99
260
  try {
100
261
  parsed = JSON.parse(answer.text);
@@ -103,23 +264,22 @@ var ServerApi = class {
103
264
  }
104
265
  if (answer.status >= 500) throw new Unreachable(`the sync server failed (${answer.status})`, true);
105
266
  const refused = parsed;
106
- if (answer.status === 401 || answer.status === 404 || refused.reason === "malformed") throw new ServerError(`the sync server refused: ${refused.message ?? answer.status}`, answer.status);
267
+ if (answer.status === 404 || refused.reason === "malformed") throw new ServerError(`the sync server refused: ${refused.message ?? answer.status}`, answer.status);
107
268
  return {
108
269
  status: answer.status,
109
- body: parsed
270
+ body: parsed,
271
+ wire: answer.wire,
272
+ size: Buffer.byteLength(answer.text),
273
+ sent: Buffer.byteLength(text)
110
274
  };
111
275
  }
112
- /** A read may go on a pooled connection: one the server closed meanwhile fails at once, and the read is repeated. */
113
- async fetchGet(url) {
276
+ /**
277
+ * A read goes on a connection of the reads: one the server closed meanwhile fails at once, and the read is repeated,
278
+ * since asking for a snapshot again changes nothing.
279
+ */
280
+ async read(method, url, body) {
114
281
  for (let attempt = 0;; attempt++) try {
115
- const response = await fetch(url, {
116
- headers: this.headers,
117
- signal: AbortSignal.timeout(this.timeoutMs)
118
- });
119
- return {
120
- status: response.status,
121
- text: await response.text()
122
- };
282
+ return await this.exchange(method, url, body, this.reads, this.timeoutMs);
123
283
  } catch (error) {
124
284
  if (attempt > 0 || !STALE.has(causeCode(error) ?? "")) throw error;
125
285
  }
@@ -131,14 +291,19 @@ var ServerApi = class {
131
291
  * the response it kept if it took the request after all. Keeping the connection busy (warm) makes that rare.
132
292
  */
133
293
  write(method, url, body, fresh) {
134
- return this.exchange(method, url, {
135
- ...this.headers,
136
- "content-length": String(Buffer.byteLength(body))
137
- }, body, fresh ? false : this.agent, this.timeoutMs);
294
+ return this.exchange(method, url, body, fresh ? false : this.agent, this.timeoutMs);
138
295
  }
139
- exchange(method, url, headers, body, agent, timeoutMs) {
296
+ exchange(method, url, body, agent, timeoutMs) {
140
297
  const send = url.startsWith("https:") ? request$1 : request;
141
- if (agent) this.busy++;
298
+ const headers = {
299
+ ...this.access,
300
+ "accept-encoding": ACCEPT_ENCODING
301
+ };
302
+ if (body !== void 0) Object.assign(headers, {
303
+ "content-type": "application/json",
304
+ "content-length": String(Buffer.byteLength(body))
305
+ });
306
+ if (agent === this.agent) this.busy++;
142
307
  return new Promise((resolve, reject) => {
143
308
  const req = send(url, {
144
309
  method,
@@ -146,19 +311,31 @@ var ServerApi = class {
146
311
  headers,
147
312
  timeout: timeoutMs
148
313
  }, (res) => {
314
+ let wire = 0;
315
+ res.on("data", (chunk) => wire += chunk.length);
316
+ const encoding = String(res.headers["content-encoding"] ?? "identity").toLowerCase();
317
+ const decode = DECODERS[encoding];
318
+ if (!decode && encoding !== "identity") {
319
+ res.resume();
320
+ return reject(/* @__PURE__ */ new Error(`the answer came in content-encoding ${encoding}, which this client cannot read`));
321
+ }
322
+ const stream = decode ? res.pipe(decode()) : res;
149
323
  const chunks = [];
150
- res.on("data", (chunk) => chunks.push(chunk));
151
- res.on("end", () => resolve({
324
+ stream.on("data", (chunk) => chunks.push(chunk));
325
+ stream.on("end", () => resolve({
152
326
  status: res.statusCode ?? 0,
153
- text: Buffer.concat(chunks).toString("utf8")
327
+ text: Buffer.concat(chunks).toString("utf8"),
328
+ location: res.headers.location,
329
+ wire
154
330
  }));
331
+ stream.on("error", reject);
155
332
  res.on("error", reject);
156
333
  });
157
334
  req.on("timeout", () => req.destroy(Object.assign(/* @__PURE__ */ new Error("no answer in time"), { code: req.socket?.connecting === false ? "ETIMEDOUT" : "ECONNECT_TIMEOUT" })));
158
335
  req.on("error", reject);
159
336
  req.end(body);
160
337
  }).finally(() => {
161
- if (agent) this.busy--;
338
+ if (agent === this.agent) this.busy--;
162
339
  });
163
340
  }
164
341
  /**
@@ -168,25 +345,76 @@ var ServerApi = class {
168
345
  async warm() {
169
346
  if (this.busy > 0) return;
170
347
  try {
171
- await this.exchange("GET", `${this.settings.url}${ROUTES.version}`, {}, void 0, this.agent, WARM_TIMEOUT_MS);
348
+ await this.exchange("GET", `${this.settings.url}${ROUTES.version}`, void 0, this.agent, WARM_TIMEOUT_MS);
172
349
  } catch {}
173
350
  }
174
- /** Close the kept connection; a later request opens another. */
351
+ /** Close the kept connections; a later request opens others. */
175
352
  close() {
176
353
  this.agent.destroy();
354
+ this.reads.destroy();
177
355
  }
178
356
  /** Send one request; the response is the hub's answer, applied or refused. */
179
357
  async send(request, options = {}) {
180
- return (await this.call("POST", ROUTES.request(this.settings.hub), request, options.fresh === true)).body;
358
+ return (await this.call("POST", ROUTES.request(this.settings.hub), request, options.fresh === true ? "fresh" : "write")).body;
181
359
  }
182
- /** Every file of the hub, as of the hub's `seq`. */
183
- async snapshot() {
184
- const { body } = await this.call("GET", `${ROUTES.snapshot(this.settings.hub)}?protocol=2`);
360
+ /** One batch of the snapshot, refused with a ServerError. */
361
+ async batch(request) {
362
+ const { body, wire, size, sent } = await this.call("POST", ROUTES.snapshot(this.settings.hub), request, "read");
185
363
  if (!body.ok) throw new ServerError(`the sync server refused the snapshot: ${body.message}`, body.status);
186
- return {
187
- seq: body.seq,
188
- files: body.files
189
- };
364
+ this.transfer.batches++;
365
+ this.transfer.files += body.files.filter((entry) => entry.kind === "file").length;
366
+ this.transfer.listed += body.files.filter((entry) => entry.kind === "listed").length;
367
+ this.transfer.wire += wire;
368
+ this.transfer.body += size;
369
+ this.transfer.sent += sent;
370
+ return body;
371
+ }
372
+ /**
373
+ * The hub's snapshot, batch by batch (docs/sync-protocol.md "快照與整份上傳"). `have` gives the content hash of the
374
+ * text the client holds of each file, so the server sends only the files that differ, and names the ones of `have` it
375
+ * does not hold as removed; each batch is sent only the part of `have` in its own range. A path of `have` that no
376
+ * batch mentions is a file the server holds with that text. The batches are in order, each with its seq; one whose seq
377
+ * is below the batch before, as when the server's storage was replaced meanwhile, starts the snapshot over.
378
+ * `listOnly` asks for the files under those prefixes as `listed`, without their text. A part of the hub alone is taken
379
+ * from `after` on, and ends with the batch that covers `through`; only the paths of that range are told of.
380
+ */
381
+ async snapshot(have = /* @__PURE__ */ new Map(), options = {}) {
382
+ const paths = [...have.keys()].sort(comparePaths);
383
+ const { through } = options;
384
+ for (let attempt = 1;; attempt++) {
385
+ const parts = [];
386
+ let after = options.after;
387
+ let start = 0;
388
+ for (;;) {
389
+ while (after !== void 0 && start < paths.length && comparePaths(paths[start], after) <= 0) start++;
390
+ const range = Object.fromEntries(paths.slice(start).map((path) => [path, have.get(path)]));
391
+ const { seq, files, next } = await this.batch({
392
+ protocol: 3,
393
+ have: range,
394
+ ...after === void 0 ? {} : { after },
395
+ ...options.listOnly?.length ? { listOnly: options.listOnly } : {}
396
+ });
397
+ if (parts.length > 0 && seq < parts.at(-1).seq) break;
398
+ parts.push({
399
+ seq,
400
+ files
401
+ });
402
+ if (next === null || through !== void 0 && comparePaths(next, through) >= 0) return parts;
403
+ if (after !== void 0 && comparePaths(next, after) <= 0) throw new Unreachable(`the sync server's snapshot did not move past ${after}`, true);
404
+ after = next;
405
+ }
406
+ if (attempt >= SNAPSHOT_ATTEMPTS) throw new Unreachable(`the sync server's seq went back while the snapshot was taken, ${SNAPSHOT_ATTEMPTS} times in a row`, true);
407
+ }
408
+ }
409
+ /**
410
+ * Whether the server takes this client's requests for the hub: the service token, the protocol version and the hub's
411
+ * name. One batch of at most one file, so the hub's size does not matter; a refusal is a ServerError.
412
+ */
413
+ async check() {
414
+ await this.batch({
415
+ protocol: 3,
416
+ limit: 1
417
+ });
190
418
  }
191
419
  /**
192
420
  * Upload a whole hub; refused with 409 when the server already holds one and `replace` is not set. It goes on a
@@ -194,17 +422,17 @@ var ServerApi = class {
194
422
  */
195
423
  async upload(files, dirs, replace = false) {
196
424
  const { body } = await this.call("PUT", ROUTES.snapshot(this.settings.hub), {
197
- protocol: 2,
425
+ protocol: 3,
198
426
  files,
199
427
  dirs,
200
428
  replace
201
- }, true);
429
+ }, "fresh");
202
430
  return body;
203
431
  }
204
432
  /** Open the hub's broadcast connection. */
205
433
  socket() {
206
- const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=2`;
207
- return new WebSocket(url, { headers: { [AUTHORIZATION]: `Bearer ${this.settings.token}` } });
434
+ const url = `${this.settings.url.replace(/^http/, "ws")}${ROUTES.socket(this.settings.hub)}?protocol=3`;
435
+ return new WebSocket(url, { headers: this.access });
208
436
  }
209
437
  };
210
438
  //#endregion
@@ -334,58 +562,6 @@ function mergeThreeWay(current, base, incoming) {
334
562
  };
335
563
  }
336
564
  //#endregion
337
- //#region ../sync/dist/paths.js
338
- /** The sync client's own directory, inside local/ so it is neither synced nor committed. */
339
- const SYNC_DIR = "local/sync";
340
- /** The settings a machine keeps instead of the environment variables: { url, token }. Never synced or committed. */
341
- const SETTINGS_FILE = "local/sync.json";
342
- const syncPaths = (root) => {
343
- const data = join(root, DATA_DIR);
344
- const own = join(data, SYNC_DIR);
345
- return {
346
- data,
347
- own,
348
- tmp: join(own, "tmp"),
349
- conflicts: join(own, "conflicts"),
350
- known: join(own, "known.json"),
351
- /** The text of each version known.json names, in a file named by its content hash */
352
- knownTexts: join(own, "known"),
353
- queue: join(own, "queue.json"),
354
- notices: join(own, "notices.jsonl"),
355
- daemon: join(own, "daemon.json"),
356
- /** The sync client a mainflow write last started in the background: { pid, at } */
357
- started: join(own, "started.json"),
358
- /** What a sync client started in the background prints */
359
- log: join(own, "sync.log")
360
- };
361
- };
362
- /** Names programs leave behind while writing: editor swap and backup files, temporary files, Finder's. */
363
- const TRANSIENT = [
364
- /\.tmp$/,
365
- /\.sw[a-p]$/,
366
- /\.swx$/,
367
- /~$/,
368
- /^\.#/,
369
- /^4913$/,
370
- /\.crswap$/,
371
- /^\.DS_Store$/
372
- ];
373
- /** Whether a file at `path` (relative to .mainflow/) is synced. */
374
- function isSynced(path) {
375
- const parts = path.split("/");
376
- if (parts.some((part) => part === "" || part === "." || part === "..")) return false;
377
- if (parts[0] === "local" && !(parts[1] === "sessions" && parts.length === 3)) return false;
378
- const name = parts.at(-1);
379
- return !TRANSIENT.some((pattern) => pattern.test(name));
380
- }
381
- /** Whether the directory at `path` may hold synced files, so a scan walks into it. */
382
- const mayHoldSynced = (path) => path !== "local" ? !path.startsWith("local/") || path === "local/sessions" : true;
383
- /**
384
- * Whether the client's file watcher watches inside the directory at `path`: one that may hold synced files, and
385
- * local/sync/ itself for the outbox; the directories under local/sync/ hold only the client's own files.
386
- */
387
- const watchedDir = (path) => path === "local/sync" || mayHoldSynced(path);
388
- //#endregion
389
565
  //#region ../sync/dist/state.js
390
566
  /** Whether `entry` waits for the server: not one it refused. */
391
567
  const waiting = (entry) => entry.refused === void 0;
@@ -394,11 +570,14 @@ const entryId = () => randomBytes(8).toString("hex");
394
570
  const entryPaths = (entry) => entry.kind === "op" ? Object.keys(entry.written) : [entry.path];
395
571
  /** A local minute without an offset, the `now` of protocol 1 */
396
572
  const LOCAL_MINUTE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})$/;
573
+ /** Protocols whose write requests read the same as the current one's: protocol 3 changed only how a snapshot is taken. */
574
+ const SAME_WRITES = /* @__PURE__ */ new Set([2]);
397
575
  /**
398
576
  * A queued operation on the current protocol. One a command of mainflow before the upgrade queued is sent on the
399
- * current one: protocol 1 differs only in `now`, a minute in this machine's time zone without its offset, which gets
400
- * the offset this machine has at that minute. The outbox is this machine's, so the time zone is the one it was
401
- * written in. Any other protocol is sent as it is, and a server on another version refuses it with both versions.
577
+ * current one: protocol 2 writes the same requests, and protocol 1 differs only in `now`, a minute in this machine's
578
+ * time zone without its offset, which gets the offset this machine has at that minute. The outbox is this machine's,
579
+ * so the time zone is the one it was written in. Any other protocol is sent as it is, and a server on another version
580
+ * refuses it with both versions.
402
581
  *
403
582
  * Around a daylight saving change the minute is read as Date reads a local time. A minute that occurs twice, in the
404
583
  * hour an autumn change repeats, always gets the earlier offset, the one before the change; the minute itself is
@@ -407,15 +586,26 @@ const LOCAL_MINUTE = /^(\d{4})-(\d{2})-(\d{2})T(\d{2}):(\d{2})$/;
407
586
  * hour a spring change skips would move an hour forward, but a local clock never shows one, so none is queued.
408
587
  */
409
588
  function currentRequest(request) {
410
- if (request.protocol !== 1) return request;
589
+ if (request.protocol !== 1) return onCurrentProtocol(request);
411
590
  const local = LOCAL_MINUTE.exec(request.now);
412
591
  const now = local ? requestTime(new Date(Number(local[1]), Number(local[2]) - 1, Number(local[3]), Number(local[4]), Number(local[5]))) : request.now;
413
592
  return {
414
593
  ...request,
415
- protocol: 2,
594
+ protocol: 3,
416
595
  now
417
596
  };
418
597
  }
598
+ /**
599
+ * A request an earlier version sent, or kept in `sent`, as it goes out now: on the current protocol when that protocol's
600
+ * write requests read the same. The server leaves `protocol` out when it compares a request sent again with the first
601
+ * (docs/sync-protocol.md "請求編號"), so it still answers it with the response it kept.
602
+ */
603
+ function onCurrentProtocol(request) {
604
+ return SAME_WRITES.has(request.protocol) ? {
605
+ ...request,
606
+ protocol: 3
607
+ } : request;
608
+ }
419
609
  function readJson(file, fallback) {
420
610
  if (!existsSync(file)) return fallback;
421
611
  try {
@@ -462,6 +652,8 @@ function writeKnown(root, files) {
462
652
  for (const name of held) if (!named.has(name)) rmSync(join(dir, name), { force: true });
463
653
  }
464
654
  const writeQueue = (root, entries) => writeJson(syncPaths(root).queue, { entries });
655
+ const readRemote = (root) => readJson(syncPaths(root).remote, {}).files ?? {};
656
+ const writeRemote = (root, files) => writeJson(syncPaths(root).remote, { files });
465
657
  /** The queue as queue.json has it; an entry from before entries had ids gets one, written back. Call under the write lock. */
466
658
  function readQueue(root) {
467
659
  const entries = readJson(syncPaths(root).queue, {}).entries ?? [];
@@ -786,6 +978,10 @@ function exists(file) {
786
978
  }
787
979
  }
788
980
  const HISTORY = 20;
981
+ /** The statuses of a closed mainline, whose artifacts stay on the server until they are used */
982
+ const CLOSED = /* @__PURE__ */ new Set(["done", "dropped"]);
983
+ /** A mainline's own file, whose status says whether its artifacts stay on the server */
984
+ const MAINLINE_FILE = /^mainlines\/([^/]+)\/mainline\.md$/;
789
985
  /** A text's short content hash, for traces. */
790
986
  const short = (text) => text == null ? String(text) : contentHash(text).slice(0, 7);
791
987
  /** How long after the file watcher's last event for a path the path is looked at, so a write in progress is not sent half done */
@@ -892,13 +1088,22 @@ function patchRequest(entry) {
892
1088
  text = merged.text;
893
1089
  }
894
1090
  return {
895
- protocol: 2,
1091
+ protocol: 3,
896
1092
  op: "file.patch",
897
1093
  path: entry.path,
898
1094
  base: entry.base,
899
1095
  text
900
1096
  };
901
1097
  }
1098
+ /** Whether the server holds any file, by a snapshot taken with `held` named: one it sent, or one of `held` it did not name removed. */
1099
+ function holdsFiles(parts, held) {
1100
+ const removed = /* @__PURE__ */ new Set();
1101
+ for (const part of parts) for (const event of part.files) {
1102
+ if (event.kind === "file" || event.kind === "listed") return true;
1103
+ if (event.kind === "removed") removed.add(event.path);
1104
+ }
1105
+ return [...held.keys()].some((path) => !removed.has(path));
1106
+ }
902
1107
  var SyncClient = class extends EventEmitter {
903
1108
  opts;
904
1109
  root;
@@ -921,6 +1126,22 @@ var SyncClient = class extends EventEmitter {
921
1126
  /** The confirmed versions of each path, oldest first, at most HISTORY; versions() adds what the outbox holds */
922
1127
  history = /* @__PURE__ */ new Map();
923
1128
  cloud = /* @__PURE__ */ new Map();
1129
+ /** The content hash last worked out for a text the client named in a snapshot request, by path */
1130
+ hashes = /* @__PURE__ */ new Map();
1131
+ /**
1132
+ * The text the last snapshot request named for each path, with what the server made of it: left out, as the same text
1133
+ * it holds, or named removed. A file this client never synced, as one in a hub taken with git, that still holds that
1134
+ * text holds the server's version, or one the server removed since; not a text written here (reconcile).
1135
+ */
1136
+ snapshotAnswers = /* @__PURE__ */ new Map();
1137
+ /**
1138
+ * Whether this run started without known.json, as in a hub taken with git that never synced here. Only then does the
1139
+ * first snapshot's `removed` for a file the client named tell of a file the server removed since the copy: the server
1140
+ * names removed every path it does not have, so later, or with known.json, such a file is one created here.
1141
+ */
1142
+ copied = false;
1143
+ /** The server has no copy of the hub, which this environment, downloading on demand, does not upload (snapshotNow) */
1144
+ uploadWanted = false;
924
1145
  /** The hub's seq of each path's version in `cloud`: a version from the server with a seq no higher is not newer */
925
1146
  fileSeq = /* @__PURE__ */ new Map();
926
1147
  /** The hub's seq up to which the client has taken in every change: the last snapshot and the broadcasts after it */
@@ -932,6 +1153,21 @@ var SyncClient = class extends EventEmitter {
932
1153
  chains = /* @__PURE__ */ new Map();
933
1154
  work = /* @__PURE__ */ new Set();
934
1155
  binaries = /* @__PURE__ */ new Set();
1156
+ /**
1157
+ * The files only on the server, by the hash of the server's text: artifacts of closed mainlines this client never
1158
+ * had on disk and has not downloaded (onDemand). remote.json keeps them.
1159
+ */
1160
+ remote = /* @__PURE__ */ new Map();
1161
+ /**
1162
+ * Files whose server text is on its way: listed by a snapshot although this client holds them, or being downloaded.
1163
+ * No reconcile runs on them until the text is in, since the server has them and the client does not know what.
1164
+ */
1165
+ awaiting = /* @__PURE__ */ new Set();
1166
+ /** The status each mainline's file gives, by the file's text, read once per text */
1167
+ statuses = /* @__PURE__ */ new Map();
1168
+ /** The snapshot or download under way: one at a time, so neither takes in what the other holds a stale view of */
1169
+ reading = Promise.resolve();
1170
+ remoteTimer;
935
1171
  signatures = /* @__PURE__ */ new Map();
936
1172
  /** The outbox as last read from queue.json, which every writer of this machine appends to */
937
1173
  queue = [];
@@ -945,6 +1181,8 @@ var SyncClient = class extends EventEmitter {
945
1181
  sending = null;
946
1182
  ws = null;
947
1183
  connected = false;
1184
+ /** The server closed the connection at its maximum age and the next one is opening (rejoin); not offline yet */
1185
+ rejoining = false;
948
1186
  /** Whether a snapshot was taken on the current connection; cleared when a connection opens or is lost, and when one fails */
949
1187
  current = false;
950
1188
  /** No reconnecting: stop() or the test hook disconnect() */
@@ -1021,11 +1259,24 @@ var SyncClient = class extends EventEmitter {
1021
1259
  }
1022
1260
  /**
1023
1261
  * The file at `path` as the server last had it to this client's knowledge: its text, null when the server has no such
1024
- * file, or undefined when the client has not heard of it, such as before the first snapshot.
1262
+ * file, or undefined when the client does not know it, such as before the first snapshot, or for a file only on the
1263
+ * server, whose text is not here.
1025
1264
  */
1026
1265
  serverText(path) {
1266
+ if (this.remote.has(path) || this.awaiting.has(path)) return void 0;
1027
1267
  return this.cloud.has(path) ? this.cloud.get(path) : this.synced ? null : void 0;
1028
1268
  }
1269
+ /** How many files are only on the server, not downloaded here (onDemand). */
1270
+ get remoteCount() {
1271
+ return this.remote.size;
1272
+ }
1273
+ /**
1274
+ * Whether the sync server has no copy of the hub and this environment, downloading on demand, waits for one that keeps
1275
+ * every file to upload it.
1276
+ */
1277
+ get waitingForUpload() {
1278
+ return this.uploadWanted;
1279
+ }
1029
1280
  /** Start: restore what the last run left, watch the files, connect, and resolve once synced or found offline. */
1030
1281
  async start() {
1031
1282
  await mkdir(this.paths.tmp, { recursive: true });
@@ -1040,6 +1291,7 @@ var SyncClient = class extends EventEmitter {
1040
1291
  }
1041
1292
  }
1042
1293
  await this.restoreKnown();
1294
+ this.restoreRemote();
1043
1295
  await this.whenFree(() => this.syncQueue());
1044
1296
  await this.resumeRefusals();
1045
1297
  if (this.opts.watch !== false) this.startWatcher();
@@ -1058,6 +1310,7 @@ var SyncClient = class extends EventEmitter {
1058
1310
  * a file that still holds it gives it. What the outbox wrote, refused writes included, is read after it (restoreEntry).
1059
1311
  */
1060
1312
  async restoreKnown() {
1313
+ this.copied = !existsSync(this.paths.known);
1061
1314
  for (const [path, hash] of Object.entries(readKnown(this.root))) {
1062
1315
  const local = await this.readLocal(path);
1063
1316
  const text = local.kind === "text" && contentHash(local.text) === hash ? local.text : readKnownText(this.root, hash);
@@ -1129,6 +1382,7 @@ var SyncClient = class extends EventEmitter {
1129
1382
  clearTimeout(timer);
1130
1383
  this.api.close();
1131
1384
  this.saveKnownNow(true);
1385
+ this.saveRemoteNow(true);
1132
1386
  }
1133
1387
  /** Test hook: drop the connection as a lost network would, and stay offline until reconnect(). */
1134
1388
  disconnect() {
@@ -1162,13 +1416,16 @@ var SyncClient = class extends EventEmitter {
1162
1416
  this.ws = ws;
1163
1417
  this.ahead.clear();
1164
1418
  let opened = false;
1419
+ let openedAt = 0;
1165
1420
  ws.addEventListener("open", () => {
1166
1421
  if (this.closing) return ws.close();
1167
1422
  opened = true;
1423
+ openedAt = Date.now();
1168
1424
  this.diagnosed = false;
1169
1425
  this.connected = true;
1426
+ this.rejoining = false;
1170
1427
  this.current = false;
1171
- this.backoff = 500;
1428
+ if (!this.uploadWanted) this.backoff = 500;
1172
1429
  this.lastPong = Date.now();
1173
1430
  clearInterval(this.pingTimer);
1174
1431
  this.pingTimer = setInterval(() => this.ping(ws), 15e3);
@@ -1179,8 +1436,16 @@ var SyncClient = class extends EventEmitter {
1179
1436
  this.track(this.resync().then(() => this.flushQueue()).then(() => this.emit("synced")).catch((error) => this.log(`could not sync: ${error.message}`)));
1180
1437
  });
1181
1438
  ws.addEventListener("message", (event) => {
1182
- if (this.ws === ws) this.onMessage(String(event.data));
1439
+ if (this.ws === ws) this.onMessage(String(event.data), () => atMaxAge());
1183
1440
  });
1441
+ const atMaxAge = () => {
1442
+ if (this.ws !== ws) return;
1443
+ const age = Date.now() - openedAt;
1444
+ if (age >= (this.opts.rejoinMinAgeMs ?? 1e4)) return this.rejoin(ws);
1445
+ this.log(`the sync server closed a connection ${age} ms old at its maximum age; reconnecting after a pause`);
1446
+ ws.close();
1447
+ lost();
1448
+ };
1184
1449
  const lost = () => {
1185
1450
  if (this.ws !== ws) return;
1186
1451
  this.ws = null;
@@ -1191,23 +1456,46 @@ var SyncClient = class extends EventEmitter {
1191
1456
  this.reconnectTimer = setTimeout(() => this.connect(), this.backoff * (.5 + Math.random()));
1192
1457
  this.backoff = Math.min(this.backoff * 2, 1e4);
1193
1458
  };
1194
- ws.addEventListener("close", lost);
1459
+ ws.addEventListener("close", (event) => opened && event.code === SOCKET_MAX_AGE_CLOSE.code ? atMaxAge() : lost());
1195
1460
  ws.addEventListener("error", lost);
1196
1461
  }
1462
+ /**
1463
+ * The server closed the connection at its maximum age (SOCKET_MAX_AGE_CLOSE), told by its SocketClosing message or by
1464
+ * the close, whichever comes first, on `ws`, the client's connection (connect's atMaxAge); the close itself may come
1465
+ * seconds later, and does nothing then, as `ws` is no longer the client's connection. The next connection opens at once,
1466
+ * through Cloudflare Access again, and its snapshot brings what changed meanwhile, as after any reconnect. The server
1467
+ * was not lost: nothing is reported, and writers waiting for the server's answer keep waiting. Only when the next
1468
+ * connection fails, as when Access refuses a revoked service token, is the client offline (markOffline).
1469
+ */
1470
+ rejoin(ws) {
1471
+ this.ws = null;
1472
+ this.log("the sync server closed the connection at its maximum age; reconnecting");
1473
+ ws.close();
1474
+ this.markOffline(!this.stopped);
1475
+ if (this.stopped) return;
1476
+ clearTimeout(this.reconnectTimer);
1477
+ this.connect();
1478
+ }
1197
1479
  diagnosed = false;
1198
- /** A connection refused before it opened says nothing about why; one plain request tells a wrong token apart. */
1480
+ /** A connection refused before it opened says nothing about why; one plain request tells a refused service token apart. */
1199
1481
  diagnose() {
1200
1482
  this.diagnosed = true;
1201
- this.api.snapshot().then(() => this.log("the WebSocket was refused although the server answers; retrying"), (error) => this.log(`cannot connect: ${error.message}; retrying`));
1483
+ this.api.check().then(() => this.log("the WebSocket was refused although the server answers; retrying"), (error) => this.log(`cannot connect: ${error.message}; retrying`));
1202
1484
  }
1203
- markOffline() {
1204
- const was = this.connected;
1485
+ /**
1486
+ * The connection is gone, so nothing is sent before the next one has its snapshot. With `rejoining` the next one is
1487
+ * opening at once (rejoin): the client is not reported offline until that one fails too.
1488
+ */
1489
+ markOffline(rejoining = false) {
1490
+ const was = this.connected || this.rejoining;
1205
1491
  this.connected = false;
1492
+ this.rejoining = rejoining;
1206
1493
  this.current = false;
1207
1494
  clearInterval(this.pingTimer);
1208
1495
  clearInterval(this.warmTimer);
1209
1496
  for (const waiter of this.waiters) waiter.done();
1210
1497
  this.waiters.clear();
1498
+ if (rejoining) return;
1211
1499
  if (was) this.log("offline");
1212
1500
  this.emit("offline");
1213
1501
  }
@@ -1227,7 +1515,8 @@ var SyncClient = class extends EventEmitter {
1227
1515
  ws.send("ping");
1228
1516
  } catch {}
1229
1517
  }
1230
- onMessage(data) {
1518
+ /** A message on the current connection; `atMaxAge` runs when it says the server is closing it at its maximum age. */
1519
+ onMessage(data, atMaxAge) {
1231
1520
  if (this.closing) return;
1232
1521
  if (data === "pong") {
1233
1522
  this.lastPong = Date.now();
@@ -1240,6 +1529,11 @@ var SyncClient = class extends EventEmitter {
1240
1529
  } catch {
1241
1530
  return;
1242
1531
  }
1532
+ if (typeof message !== "object" || message === null) return;
1533
+ if ("closing" in message) {
1534
+ if (message.closing?.code === SOCKET_MAX_AGE_CLOSE.code) atMaxAge();
1535
+ return;
1536
+ }
1243
1537
  if (!Number.isInteger(message.seq)) return;
1244
1538
  const seq = message.seq;
1245
1539
  if (seq === this.hubSeq + 1) this.takenUpTo(seq);
@@ -1248,11 +1542,11 @@ var SyncClient = class extends EventEmitter {
1248
1542
  this.track(this.resync());
1249
1543
  return;
1250
1544
  }
1545
+ for (const event of message.files) this.receive(event, seq);
1251
1546
  if (this.ahead.size > 0 && this.synced && this.resyncing === null) {
1252
1547
  this.log(`missed the hub's changes ${this.hubSeq + 1} to ${Math.min(...this.ahead) - 1}; reloading the snapshot`);
1253
1548
  this.track(this.resync());
1254
1549
  }
1255
- for (const event of message.files) this.receive(event, seq);
1256
1550
  this.wake();
1257
1551
  }
1258
1552
  /** The client has every change up to `seq`: hubSeq moves there, and on over the broadcasts that came after it in a row. */
@@ -1263,7 +1557,9 @@ var SyncClient = class extends EventEmitter {
1263
1557
  }
1264
1558
  /**
1265
1559
  * One file as the server has it as of the hub's `seq`, from a broadcast or a snapshot. A version no newer than the
1266
- * one the client holds, such as a broadcast that arrives after a later snapshot, is left out.
1560
+ * one the client holds, such as a broadcast that arrives after a later snapshot, is left out. A file only on the
1561
+ * server whose mainline is still closed gets its new hash, or leaves `remote` when removed; nothing is written here.
1562
+ * A mainline's file that opens it again brings its artifacts here.
1267
1563
  */
1268
1564
  receive(event, seq) {
1269
1565
  if (event.kind === "dir") {
@@ -1277,10 +1573,21 @@ var SyncClient = class extends EventEmitter {
1277
1573
  this.trace(`older ${event.path} ${short(text)} seq ${seq}, holding seq ${held}`);
1278
1574
  return;
1279
1575
  }
1280
- this.trace(`receive ${event.path} ${short(text)} seq ${seq}`);
1281
1576
  this.fileSeq.set(event.path, seq);
1577
+ if (this.remote.has(event.path) && !this.awaiting.has(event.path) && (text === null || this.staysRemote(event.path))) {
1578
+ this.trace(`remote ${event.path} ${short(text)} seq ${seq}`);
1579
+ if (text === null) this.remote.delete(event.path);
1580
+ else this.remote.set(event.path, contentHash(text));
1581
+ this.saveRemote();
1582
+ return;
1583
+ }
1584
+ if (this.remote.delete(event.path)) this.saveRemote();
1585
+ this.trace(`receive ${event.path} ${short(text)} seq ${seq}`);
1586
+ this.awaiting.delete(event.path);
1282
1587
  this.cloud.set(event.path, text);
1283
1588
  this.schedule(event.path);
1589
+ const mainline = MAINLINE_FILE.exec(event.path)?.[1];
1590
+ if (mainline !== void 0 && this.remote.size > 0) this.fetchOpened(mainline);
1284
1591
  }
1285
1592
  /** Whether the client holds `path` as of the hub's `seq` or later. */
1286
1593
  reached(path, seq) {
@@ -1369,41 +1676,145 @@ var SyncClient = class extends EventEmitter {
1369
1676
  return this.resyncing;
1370
1677
  }
1371
1678
  /**
1372
- * Take the hub as of the snapshot's seq. A broadcast arriving meanwhile may be older or newer than the snapshot; each
1373
- * file keeps whichever version has the higher seq. A server without the hub first gets the local one.
1679
+ * Take the hub as of the snapshot's seq, asking only for what differs from what the client holds (holdings). The
1680
+ * snapshot comes in batches, each as of its own seq; the WebSocket was open before the first, so every change after
1681
+ * it comes as a broadcast, and the first batch's seq is the one the whole snapshot is as of. A file the server sent,
1682
+ * or named removed, is taken at its batch's seq; a file it left out holds what the client named, as of the first
1683
+ * batch's seq; a file neither named nor sent is not on the server. A broadcast arriving meanwhile may be older or
1684
+ * newer than either; each file keeps whichever version has the higher seq. A server without the hub first gets the
1685
+ * local one.
1686
+ *
1687
+ * With onDemand, the artifacts of mainlines not known to be open come `listed` (listOnly), and the files only on the
1688
+ * server are named by their hash, so one that has not changed is not mentioned. A listed file this client has no
1689
+ * text of and never left on disk stays on the server (`remote`); any other listed file, as one held here, is
1690
+ * downloaded before the snapshot ends, with the files only on the server whose mainline is open now. The first
1691
+ * snapshot of a hub without known.json lists all of mainlines/, since the client may not know a mainline or its
1692
+ * status yet: the mainlines' own files are downloaded after, then the artifacts of the open ones.
1374
1693
  */
1375
- async takeSnapshot() {
1376
- let snapshot;
1694
+ takeSnapshot() {
1695
+ return this.exclusive(() => this.snapshotNow());
1696
+ }
1697
+ async snapshotNow() {
1698
+ let parts;
1699
+ let held;
1700
+ let have;
1377
1701
  let before;
1378
1702
  for (let upload = 0;; upload++) {
1379
1703
  before = this.hubSeq;
1380
- snapshot = await this.api.snapshot();
1381
- if (upload > 0 || snapshot.files.some((event) => event.kind === "file") || !await this.uploadIfEmpty()) break;
1704
+ held = await this.holdings();
1705
+ for (const path of this.remote.keys()) if (held.has(path) || this.expected.get(path) !== void 0) this.remote.delete(path);
1706
+ have = new Map([...held].map(([path, text]) => [path, this.hashOf(path, text)]));
1707
+ for (const [path, hash] of this.remote) have.set(path, hash);
1708
+ parts = await this.api.snapshot(have, { listOnly: this.listOnly(this.copied && !this.synced) });
1709
+ if (upload > 0 || holdsFiles(parts, have)) break;
1710
+ if (this.opts.settings.onDemand === true) {
1711
+ if (!this.uploadWanted) {
1712
+ this.uploadWanted = true;
1713
+ this.problem({
1714
+ kind: "upload-needed",
1715
+ path: "",
1716
+ message: `the sync server has no copy of hub ${this.opts.settings.hub}; this environment downloads on demand and may lack files, so it does not upload its copy. Run mainflow sync, with on-demand downloads off, in an environment that keeps every file of the hub, such as the one that commits it with git, to upload it; this one waits and syncs once the hub is there`
1717
+ });
1718
+ }
1719
+ throw new Error("the sync server has no copy of the hub, and this environment downloads on demand, so it waits for one that keeps every file to upload it");
1720
+ }
1721
+ if (!await this.uploadIfEmpty()) break;
1722
+ }
1723
+ if (this.uploadWanted) {
1724
+ this.uploadWanted = false;
1725
+ this.backoff = 500;
1382
1726
  }
1383
1727
  if (this.closing) return;
1384
- const { seq, files } = snapshot;
1728
+ const seq = parts[0].seq;
1385
1729
  if (seq < before) {
1386
1730
  this.log(`the sync server's seq went back from ${before} to ${seq}; taking its snapshot as it is`);
1387
1731
  this.fileSeq.clear();
1388
1732
  this.hubSeq = -1;
1389
1733
  this.ahead.clear();
1390
1734
  }
1735
+ const named = /* @__PURE__ */ new Set();
1391
1736
  const present = /* @__PURE__ */ new Set();
1392
- for (const event of files) {
1393
- if (event.kind === "file") present.add(event.path);
1394
- this.receive(event, seq);
1737
+ const wanted = /* @__PURE__ */ new Set();
1738
+ let sent = 0;
1739
+ this.snapshotAnswers.clear();
1740
+ const fromCopy = this.copied && !this.synced;
1741
+ for (const part of parts) for (const event of part.files) {
1742
+ if (event.kind === "listed") {
1743
+ named.add(event.path);
1744
+ if (this.mayStay(event.path, held)) this.keepRemote(event.path, event.hash, part.seq);
1745
+ else {
1746
+ wanted.add(event.path);
1747
+ this.awaiting.add(event.path);
1748
+ if (part.seq > (this.fileSeq.get(event.path) ?? -1)) this.fileSeq.set(event.path, part.seq);
1749
+ }
1750
+ continue;
1751
+ }
1752
+ named.add(event.path);
1753
+ if (event.kind === "file") {
1754
+ present.add(event.path);
1755
+ sent++;
1756
+ }
1757
+ if (event.kind === "removed" && fromCopy && held.has(event.path)) this.snapshotAnswers.set(event.path, {
1758
+ text: held.get(event.path),
1759
+ removed: true
1760
+ });
1761
+ this.receive(event, part.seq);
1762
+ }
1763
+ for (const [path, text] of held) {
1764
+ if (named.has(path)) continue;
1765
+ present.add(path);
1766
+ this.snapshotAnswers.set(path, {
1767
+ text,
1768
+ removed: false
1769
+ });
1770
+ this.receive({
1771
+ kind: "file",
1772
+ path,
1773
+ text
1774
+ }, seq);
1395
1775
  }
1396
- for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) if (!present.has(path) && isSynced(path)) this.receive({
1776
+ for (const path of /* @__PURE__ */ new Set([...this.cloud.keys(), ...this.expected.keys()])) if (!present.has(path) && !named.has(path) && isSynced(path)) this.receive({
1397
1777
  kind: "removed",
1398
1778
  path
1399
1779
  }, seq);
1780
+ this.log(`took the snapshot at seq ${seq} in ${parts.length} batch${parts.length === 1 ? "" : "es"}: ${sent} files sent, ${present.size - sent} already held here${this.remote.size > 0 ? `, ${this.remote.size} only on the server` : ""}`);
1400
1781
  this.takenUpTo(seq);
1401
1782
  this.synced = true;
1783
+ this.saveRemote();
1784
+ await this.fetchWanted(wanted);
1402
1785
  await this.settleUnsynced();
1403
1786
  this.wake();
1404
1787
  for (const path of await this.walk()) this.schedule(path);
1405
1788
  }
1406
1789
  /**
1790
+ * The text the client holds of each synced file, which a snapshot request names by hash so the server leaves out what
1791
+ * is the same (SnapshotRequest.have): the server's version this run last took in, else the version known.json names,
1792
+ * else the file on disk, as in a hub cloned with git that never synced here. Only a text the client has is named,
1793
+ * since a file the server leaves out holds the text named for it.
1794
+ */
1795
+ async holdings() {
1796
+ const held = /* @__PURE__ */ new Map();
1797
+ for (const [path, text] of this.cloud) if (typeof text === "string" && isSynced(path)) held.set(path, text);
1798
+ for (const [path, disk] of this.confirmed) if (!held.has(path) && disk?.text !== void 0 && isSynced(path)) held.set(path, disk.text);
1799
+ for (const path of await this.walk()) {
1800
+ if (held.has(path)) continue;
1801
+ const local = await this.readLocal(path);
1802
+ if (local.kind === "text") held.set(path, local.text);
1803
+ }
1804
+ return held;
1805
+ }
1806
+ /** The content hash of `text`, the one `path` was named with, worked out once per text. */
1807
+ hashOf(path, text) {
1808
+ const known = this.hashes.get(path);
1809
+ if (known?.text === text) return known.hash;
1810
+ const hash = contentHash(text);
1811
+ this.hashes.set(path, {
1812
+ text,
1813
+ hash
1814
+ });
1815
+ return hash;
1816
+ }
1817
+ /**
1407
1818
  * A server without the hub gets the local one; true when it was uploaded and a reload follows. The local files
1408
1819
  * already hold what the outbox's operations did, which the server does when they are sent, so a file an operation
1409
1820
  * changed is uploaded as it was before the first one that did (d017).
@@ -1483,6 +1894,8 @@ var SyncClient = class extends EventEmitter {
1483
1894
  * each write an entry of its own; the server's version waits while the outbox has a write about the file (I4).
1484
1895
  */
1485
1896
  async reconcile(path) {
1897
+ if (this.awaiting.has(path)) return;
1898
+ if (this.remote.has(path)) return this.reconcileRemote(path);
1486
1899
  const local = await this.readLocal(path);
1487
1900
  if (local.kind === "binary") {
1488
1901
  await sleep$1(BINARY_RECHECK_MS);
@@ -1506,7 +1919,21 @@ var SyncClient = class extends EventEmitter {
1506
1919
  return;
1507
1920
  }
1508
1921
  if (target === local.text) return this.adopt(path, local.text);
1509
- if (target === void 0) return;
1922
+ if (!this.synced) return;
1923
+ const answer = this.snapshotAnswers.get(path);
1924
+ if (answer?.text === local.text && !this.pending(path)) {
1925
+ this.snapshotAnswers.delete(path);
1926
+ if (!answer.removed) {
1927
+ this.adopt(path, local.text);
1928
+ return this.schedule(path);
1929
+ }
1930
+ const saved = await this.saveConflict(path, local.text);
1931
+ this.expected.set(path, {
1932
+ hash: contentHash(local.text),
1933
+ text: local.text
1934
+ });
1935
+ return this.replace(path, null, saved);
1936
+ }
1510
1937
  return this.queueLocal(path, local.text, { start: target === null ? null : {
1511
1938
  text: local.text,
1512
1939
  server: local.text
@@ -1622,16 +2049,294 @@ var SyncClient = class extends EventEmitter {
1622
2049
  message: `${path} is not UTF-8 text, so it is not synced and not overwritten; other environments do not see it`
1623
2050
  });
1624
2051
  }
2052
+ /**
2053
+ * Whether mainline `dir` is open, closed (done or dropped) or of a status the client does not know, by its
2054
+ * mainline.md: the server's text the client took in, else the file here.
2055
+ */
2056
+ mainlineState(dir) {
2057
+ const path = `mainlines/${dir}/mainline.md`;
2058
+ const taken = this.cloud.get(path);
2059
+ let text;
2060
+ if (typeof taken === "string") text = taken;
2061
+ else if (taken === void 0) try {
2062
+ text = readFileSync(join(this.paths.data, path), "utf8");
2063
+ } catch {
2064
+ text = void 0;
2065
+ }
2066
+ if (text === void 0) return "unknown";
2067
+ let known = this.statuses.get(dir);
2068
+ if (known?.text !== text) {
2069
+ let status = null;
2070
+ try {
2071
+ const data = splitFrontMatter(text).data;
2072
+ if (typeof data?.status === "string") status = data.status;
2073
+ } catch {}
2074
+ known = {
2075
+ text,
2076
+ status
2077
+ };
2078
+ this.statuses.set(dir, known);
2079
+ }
2080
+ return known.status === null ? "unknown" : CLOSED.has(known.status) ? "closed" : "open";
2081
+ }
2082
+ /**
2083
+ * Whether `path`, a file only on the server, stays there: with onDemand on, an artifact of a mainline not known to be
2084
+ * open. One whose mainline.md tells no status, or is not there, stays too; sync fetch downloads it.
2085
+ */
2086
+ staysRemote(path) {
2087
+ const mainline = artifactOf(path);
2088
+ return this.opts.settings.onDemand === true && mainline !== null && this.mainlineState(mainline) !== "open";
2089
+ }
2090
+ /**
2091
+ * Whether a file a snapshot listed may stay on the server: an artifact of a mainline not known to be open, of which
2092
+ * the client holds no text (`held`, the server's text taken in), never left a file on disk, and has no write about.
2093
+ */
2094
+ mayStay(path, held) {
2095
+ if (!this.staysRemote(path)) return false;
2096
+ return !(held?.has(path) ?? false) && typeof this.cloud.get(path) !== "string" && this.expected.get(path) === void 0 && !this.pending(path);
2097
+ }
2098
+ /** Note `path` as only on the server, with the hash of the server's text as of `seq`, unless a newer version came. */
2099
+ keepRemote(path, hash, seq) {
2100
+ if (seq <= (this.fileSeq.get(path) ?? -1)) return;
2101
+ this.fileSeq.set(path, seq);
2102
+ this.remote.set(path, hash);
2103
+ this.cloud.delete(path);
2104
+ this.awaiting.delete(path);
2105
+ this.trace(`remote ${path} ${hash.slice(0, 7)} seq ${seq}`);
2106
+ }
2107
+ /**
2108
+ * The prefixes a snapshot asks to list (SnapshotRequest.listOnly): the artifacts directories of every mainline not
2109
+ * known to be open, its own and each task's the client knows of, or all of mainlines/ when `broad`. None without
2110
+ * onDemand.
2111
+ */
2112
+ listOnly(broad) {
2113
+ if (this.opts.settings.onDemand !== true) return void 0;
2114
+ if (broad) return ["mainlines/"];
2115
+ const prefixes = /* @__PURE__ */ new Set();
2116
+ const states = /* @__PURE__ */ new Map();
2117
+ for (const path of /* @__PURE__ */ new Set([
2118
+ ...this.cloud.keys(),
2119
+ ...this.remote.keys(),
2120
+ ...this.expected.keys()
2121
+ ])) {
2122
+ const match = /^mainlines\/([^/]+)\/(tasks\/[^/]+\/)?/.exec(path);
2123
+ if (!match) continue;
2124
+ const [, mainline, task] = match;
2125
+ if (!states.has(mainline)) states.set(mainline, this.mainlineState(mainline));
2126
+ if (states.get(mainline) === "open") continue;
2127
+ prefixes.add(`mainlines/${mainline}/artifacts/`);
2128
+ if (task) prefixes.add(`mainlines/${mainline}/${task}artifacts/`);
2129
+ }
2130
+ return [...prefixes].sort(comparePaths);
2131
+ }
2132
+ /** Run `fn`, a snapshot or a download, after the one before it, so neither takes in a range the other changed under it. */
2133
+ exclusive(fn) {
2134
+ const run = this.reading.then(fn, fn);
2135
+ this.reading = run.catch(() => {});
2136
+ return run;
2137
+ }
2138
+ /**
2139
+ * Download what a snapshot listed although it may not stay on the server (`listed`), with the files only on the
2140
+ * server that do not stay there: their mainline is known to be open now, or onDemand is off. A download may tell the status of
2141
+ * more mainlines, as the mainlines' own files a first snapshot only listed do, so it is repeated while it finds more,
2142
+ * three times at most; what is left waits for the next snapshot.
2143
+ */
2144
+ async fetchWanted(listed) {
2145
+ let wanted = listed;
2146
+ for (let round = 0; round < 3 && !this.closing; round++) {
2147
+ for (const path of this.remote.keys()) if (!this.staysRemote(path)) wanted.add(path);
2148
+ if (wanted.size === 0) return;
2149
+ this.log(`downloading ${wanted.size} file${wanted.size === 1 ? "" : "s"} the snapshot only listed or left on the server`);
2150
+ wanted = (await this.fetchNow([...wanted])).again;
2151
+ }
2152
+ }
2153
+ /**
2154
+ * Download `targets` in one part of the snapshot, from the directory of the first to the last, naming every other
2155
+ * file the client knows by its hash, so the server sends the targets and only what changed of the rest; the artifacts
2156
+ * of closed mainlines other than the targets' stay listed. What comes is taken in as a broadcast of its batch's seq
2157
+ * is: a target's text is written here, and a target named nowhere is not on the server, since the range covers it
2158
+ * and `have` does not name it. Returns, besides, the files of other paths listed that may not stay on the server.
2159
+ */
2160
+ async fetchNow(targets) {
2161
+ const sorted = [...new Set(targets)].filter(isSynced).sort(comparePaths);
2162
+ const again = /* @__PURE__ */ new Set();
2163
+ if (sorted.length === 0 || this.closing) return {
2164
+ fetched: [],
2165
+ gone: [],
2166
+ again
2167
+ };
2168
+ const want = new Set(sorted);
2169
+ for (const path of sorted) this.awaiting.add(path);
2170
+ let done = false;
2171
+ try {
2172
+ const have = /* @__PURE__ */ new Map();
2173
+ for (const [path, text] of this.cloud) if (typeof text === "string" && !want.has(path) && isSynced(path)) have.set(path, this.hashOf(path, text));
2174
+ for (const [path, hash] of this.remote) if (!want.has(path)) have.set(path, hash);
2175
+ const listOnly = this.listOnly(false)?.filter((prefix) => !sorted.some((path) => path.startsWith(prefix)));
2176
+ const first = sorted[0];
2177
+ const after = first.includes("/") ? first.slice(0, first.lastIndexOf("/")) : void 0;
2178
+ const parts = await this.api.snapshot(have, {
2179
+ listOnly,
2180
+ after,
2181
+ through: sorted.at(-1)
2182
+ });
2183
+ if (this.closing) return {
2184
+ fetched: [],
2185
+ gone: [],
2186
+ again
2187
+ };
2188
+ const mentioned = /* @__PURE__ */ new Set();
2189
+ const fetched = [];
2190
+ for (const part of parts) for (const event of part.files) {
2191
+ if (event.kind === "dir") {
2192
+ this.receive(event, part.seq);
2193
+ continue;
2194
+ }
2195
+ mentioned.add(event.path);
2196
+ if (event.kind === "listed") {
2197
+ if (want.has(event.path)) continue;
2198
+ if (this.mayStay(event.path)) this.keepRemote(event.path, event.hash, part.seq);
2199
+ else again.add(event.path);
2200
+ continue;
2201
+ }
2202
+ if (want.has(event.path)) {
2203
+ this.remote.delete(event.path);
2204
+ if ((this.fileSeq.get(event.path) ?? -1) <= part.seq) this.fileSeq.delete(event.path);
2205
+ if (event.kind === "file") fetched.push(event.path);
2206
+ }
2207
+ this.receive(event, part.seq);
2208
+ }
2209
+ const gone = [];
2210
+ for (const path of sorted) {
2211
+ if (mentioned.has(path)) continue;
2212
+ if (this.remote.delete(path)) gone.push(path);
2213
+ else this.receive({
2214
+ kind: "removed",
2215
+ path
2216
+ }, parts[0].seq);
2217
+ }
2218
+ this.saveRemote();
2219
+ done = true;
2220
+ if (fetched.length + gone.length > 0) this.log(`downloaded ${fetched.length} file${fetched.length === 1 ? "" : "s"}${gone.length > 0 ? `; ${gone.length} no longer on the server` : ""}`);
2221
+ return {
2222
+ fetched,
2223
+ gone,
2224
+ again
2225
+ };
2226
+ } finally {
2227
+ for (const path of sorted) {
2228
+ if (done || this.remote.has(path)) this.awaiting.delete(path);
2229
+ this.schedule(path);
2230
+ }
2231
+ }
2232
+ }
2233
+ /**
2234
+ * Download the files only on the server that `specs` name: a path, or a directory with every such file under it.
2235
+ * `opened` downloads only the ones that no longer stay on the server, as the artifacts of a mainline opened again.
2236
+ */
2237
+ fetchPaths(specs, opened = false) {
2238
+ return this.exclusive(async () => {
2239
+ const covers = (spec, path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`);
2240
+ const targets = [...this.remote.keys()].filter((path) => specs.some((spec) => covers(spec, path)) && !(opened && this.staysRemote(path)));
2241
+ const none = specs.filter((spec) => !targets.some((path) => covers(spec, path)));
2242
+ if (targets.length === 0) return {
2243
+ fetched: [],
2244
+ gone: [],
2245
+ none
2246
+ };
2247
+ const { fetched, gone } = await this.fetchNow(targets);
2248
+ return {
2249
+ fetched,
2250
+ gone,
2251
+ none
2252
+ };
2253
+ });
2254
+ }
2255
+ /**
2256
+ * Download the files only on the server that `specs` name (fetchPaths), and resolve once each is written here: for
2257
+ * `mainflow sync fetch` and the PreToolUse hook. Offline, or when the server cannot be reached, nothing is
2258
+ * downloaded and `error` says why.
2259
+ */
2260
+ async fetch(specs) {
2261
+ const covers = (spec) => [...this.remote.keys(), ...this.awaiting].some((path) => path === spec || path.startsWith(spec.endsWith("/") ? spec : `${spec}/`));
2262
+ if (!specs.some(covers)) return {
2263
+ fetched: [],
2264
+ gone: [],
2265
+ none: specs
2266
+ };
2267
+ if (!this.online) return {
2268
+ fetched: [],
2269
+ gone: [],
2270
+ none: [],
2271
+ error: "mainflow sync is offline: the sync server cannot be reached now"
2272
+ };
2273
+ let result;
2274
+ try {
2275
+ result = await this.fetchPaths(specs);
2276
+ } catch (error) {
2277
+ return {
2278
+ fetched: [],
2279
+ gone: [],
2280
+ none: [],
2281
+ error: error.message
2282
+ };
2283
+ }
2284
+ await Promise.allSettled(result.fetched.map((path) => this.chains.get(path) ?? Promise.resolve()));
2285
+ return {
2286
+ fetched: result.fetched.filter((path) => existsSync(join(this.paths.data, path))),
2287
+ gone: result.gone,
2288
+ none: result.none
2289
+ };
2290
+ }
2291
+ /** Mainline `dir`'s file came: when it opens the mainline, its artifacts only on the server come here. */
2292
+ fetchOpened(dir) {
2293
+ if (this.closing || this.opts.settings.onDemand === true && this.mainlineState(dir) !== "open") return;
2294
+ const prefix = `mainlines/${dir}/`;
2295
+ if (![...this.remote.keys()].some((path) => path.startsWith(prefix))) return;
2296
+ this.track(this.fetchPaths([prefix], true).catch((error) => this.log(`could not download the artifacts of ${dir}: ${error.message}; the next snapshot does`)));
2297
+ }
2298
+ /**
2299
+ * A file only on the server that a program wrote here, without downloading it first. Its text is downloaded, so it is
2300
+ * reconciled as a file never synced here is: kept when the server holds that text among its recent ones, and
2301
+ * otherwise refused, saved and reported, never sent over the server's text unseen. Offline, it waits: the next
2302
+ * snapshot names the file here by its own hash and downloads the server's text.
2303
+ */
2304
+ async reconcileRemote(path) {
2305
+ if ((await this.readLocal(path)).kind === "absent") return;
2306
+ if (!this.caughtUp) return;
2307
+ this.log(`${path}: written here while only on the server; downloading the server's text first`);
2308
+ await this.fetchPaths([path]).catch((error) => this.log(`${path}: could not download it: ${error.message}`));
2309
+ }
2310
+ restoreRemote() {
2311
+ for (const [path, hash] of Object.entries(readRemote(this.root))) if (isSynced(path) && artifactOf(path) !== null && this.expected.get(path) === void 0) this.remote.set(path, hash);
2312
+ }
2313
+ saveRemote() {
2314
+ if (this.remoteTimer) return;
2315
+ this.remoteTimer = setTimeout(() => this.saveRemoteNow(), 200);
2316
+ }
2317
+ saveRemoteNow(final = false) {
2318
+ if (this.abandoned && !final) return;
2319
+ clearTimeout(this.remoteTimer);
2320
+ this.remoteTimer = void 0;
2321
+ if (this.remote.size === 0 && !existsSync(this.paths.remote)) return;
2322
+ try {
2323
+ writeRemote(this.root, Object.fromEntries([...this.remote].sort(([a], [b]) => comparePaths(a, b))));
2324
+ } catch (error) {
2325
+ this.log(`could not save local/sync/remote.json: ${error.message}`);
2326
+ }
2327
+ }
1625
2328
  tempName() {
1626
2329
  return join(this.paths.tmp, `${Date.now()}-${process.pid}-${Math.random().toString(36).slice(2)}`);
1627
2330
  }
1628
2331
  /**
1629
- * Put the server's version of `path` on disk. An existing file is exchanged with the prepared one in one step,
1630
- * so whatever a local program wrote just before ends up in the prepared path and is sent, not lost; it is read
1631
- * again shortly after, for a program that still held the old file open. A new file is linked into place, which
1632
- * fails rather than overwrite a file created meanwhile; a removed file is first moved aside and checked.
2332
+ * Put the server's version of `path` on disk, or remove the file for null. An existing file is exchanged with the
2333
+ * prepared one in one step, so whatever a local program wrote just before ends up in the prepared path and is sent,
2334
+ * not lost; it is read again shortly after, for a program that still held the old file open. A new file is linked
2335
+ * into place, which fails rather than overwrite a file created meanwhile; a removed file is first moved aside and
2336
+ * checked. `unsynced` names where the text of a never synced file the server no longer has was saved: the writer is
2337
+ * told of it once the file is removed, and it is deleted again when a write in the outbox keeps the file instead.
1633
2338
  */
1634
- async replace(path, target) {
2339
+ async replace(path, target, unsynced) {
1635
2340
  const file = join(this.paths.data, path);
1636
2341
  const temp = this.tempName();
1637
2342
  this.trace(`replacing ${path} with ${short(target)}`);
@@ -1647,8 +2352,18 @@ var SyncClient = class extends EventEmitter {
1647
2352
  throw error;
1648
2353
  }
1649
2354
  if (step.kind === "exists" || step.kind === "missing") return this.schedule(path);
1650
- if (step.kind === "linked" || step.kind === "kept") return;
2355
+ if (step.kind === "kept") {
2356
+ if (unsynced !== void 0) await rm(join(this.paths.data, unsynced), { force: true });
2357
+ return;
2358
+ }
2359
+ if (step.kind === "linked") return;
1651
2360
  if (step.kind === "removed") {
2361
+ if (unsynced !== void 0) this.problem({
2362
+ kind: "removed-unsynced",
2363
+ path,
2364
+ saved: unsynced,
2365
+ message: `${path} was never synced here and the sync server no longer has it, so it was removed here too; its text was saved to ${unsynced}`
2366
+ });
1652
2367
  if (step.moved !== null && step.disk && contentHash(step.moved) !== step.disk.hash) {
1653
2368
  const saved = await this.saveConflict(path, step.moved);
1654
2369
  this.problem({
@@ -2351,7 +3066,7 @@ var SyncClient = class extends EventEmitter {
2351
3066
  } catch {}
2352
3067
  if (!this.queue.some((entry) => entry.id === id)) return { state: "unknown" };
2353
3068
  await Promise.race([this.firstContact, sleep$1(timeoutMs)]);
2354
- if (!this.online || Date.now() >= deadline) return answered() ?? { state: "waiting" };
3069
+ if (!(this.online || this.rejoining) || Date.now() >= deadline) return answered() ?? { state: "waiting" };
2355
3070
  this.track(this.flushQueue());
2356
3071
  return new Promise((resolve) => {
2357
3072
  const finish = (outcome) => {
@@ -2425,7 +3140,7 @@ var SyncClient = class extends EventEmitter {
2425
3140
  prepare(entry) {
2426
3141
  if (entry.kind === "op") return currentRequest(entry.request);
2427
3142
  if (entry.kind === "remove") return {
2428
- protocol: 2,
3143
+ protocol: 3,
2429
3144
  op: "file.remove",
2430
3145
  input: {
2431
3146
  path: entry.path,
@@ -2434,7 +3149,7 @@ var SyncClient = class extends EventEmitter {
2434
3149
  now: requestTime()
2435
3150
  };
2436
3151
  if (entry.replace) return {
2437
- protocol: 2,
3152
+ protocol: 3,
2438
3153
  op: "text.replace",
2439
3154
  path: entry.path,
2440
3155
  old: entry.replace.old,
@@ -2554,7 +3269,7 @@ var SyncClient = class extends EventEmitter {
2554
3269
  }
2555
3270
  /** Send one queued entry as `sent`; false when the server cannot be reached, so it stays queued. `again`: see request. */
2556
3271
  async sendEntry(entry, again) {
2557
- const request = entry.sent;
3272
+ const request = onCurrentProtocol(entry.sent);
2558
3273
  const paths = entryPaths(entry);
2559
3274
  if (!again && entry.kind === "patch" && request.op === "file.patch") this.trace(`patch ${entry.path} ${entry.id} base ${request.base?.slice(0, 7) ?? "null"} text ${short(request.text)} writer ${short(entry.text)} baseText ${short(entry.baseText ?? null)}`);
2560
3275
  for (const path of paths) this.inflight.set(path, (this.inflight.get(path) ?? 0) + 1);
@@ -3079,8 +3794,11 @@ async function serveIpc(root, client) {
3079
3794
  caughtUp: client.caughtUp,
3080
3795
  queued: client.queued,
3081
3796
  swapping: client.swapping,
3082
- problems: client.problems.length
3797
+ problems: client.problems.length,
3798
+ remote: client.remoteCount,
3799
+ waitingForUpload: client.waitingForUpload
3083
3800
  });
3801
+ if (req.url === "/fetch") return reply(200, await client.fetch(Array.isArray(body.paths) ? body.paths.map(String) : []));
3084
3802
  if (req.url === "/file") return reply(200, { text: client.serverText(String(body.path)) });
3085
3803
  return reply(404, { error: `no route ${req.url}` });
3086
3804
  } catch (error) {
@@ -3147,6 +3865,22 @@ const FILE_TOOLS = /* @__PURE__ */ new Set([
3147
3865
  "MultiEdit",
3148
3866
  "NotebookEdit"
3149
3867
  ]);
3868
+ /** Tools that read or write the one file named in file_path, which must be here before they run. */
3869
+ const FETCH_TOOLS = /* @__PURE__ */ new Set([
3870
+ "Read",
3871
+ "Edit",
3872
+ "Write",
3873
+ "MultiEdit"
3874
+ ]);
3875
+ /** The file of the hub at `root` that `payload` names in file_path, relative to .mainflow/; null outside it. */
3876
+ function hubPath(root, payload) {
3877
+ const input = payload.tool_input ?? {};
3878
+ if (typeof input.file_path !== "string") return null;
3879
+ const file = isAbsolute(input.file_path) ? input.file_path : resolve(payload.cwd ?? process.cwd(), input.file_path);
3880
+ const path = relative(join(root, DATA_DIR), file).split(sep).join("/");
3881
+ if (path.startsWith("../") || path === ".." || isAbsolute(path)) return null;
3882
+ return path;
3883
+ }
3150
3884
  /**
3151
3885
  * What to flush after the tool in `payload` ran in the hub at `root`. A file tool outside the hub's synced files
3152
3886
  * needs nothing; an Edit of one replacement is replayed on the latest version; anything else, such as a Bash
@@ -3157,9 +3891,8 @@ function planFlush(root, payload) {
3157
3891
  if (!FILE_TOOLS.has(tool)) return { skip: false };
3158
3892
  const input = payload.tool_input ?? {};
3159
3893
  if (typeof input.file_path !== "string") return { skip: false };
3160
- const file = isAbsolute(input.file_path) ? input.file_path : resolve(payload.cwd ?? process.cwd(), input.file_path);
3161
- const path = relative(join(root, DATA_DIR), file).split(sep).join("/");
3162
- if (path.startsWith("../") || path === ".." || isAbsolute(path) || !isSynced(path)) return { skip: true };
3894
+ const path = hubPath(root, payload);
3895
+ if (path === null || !isSynced(path)) return { skip: true };
3163
3896
  if (tool === "Edit" && input.replace_all !== true && typeof input.old_string === "string" && typeof input.new_string === "string") return {
3164
3897
  skip: false,
3165
3898
  edit: {
@@ -3183,36 +3916,26 @@ function hookAnswer(result) {
3183
3916
  reason
3184
3917
  });
3185
3918
  }
3186
- //#endregion
3187
- //#region ../sync/dist/settings.js
3188
- const ENV = {
3189
- url: "MAINFLOW_SYNC_URL",
3190
- token: "MAINFLOW_SYNC_TOKEN"
3191
- };
3192
- /** The hub's name as config.yaml gives it. */
3193
- function hubName(root) {
3194
- const config = parse$1(readFileSync(join(root, DATA_DIR, "config.yaml"), "utf8"));
3195
- if (typeof config?.hub !== "string") throw new Error(`${join(root, DATA_DIR, "config.yaml")} names no hub`);
3196
- return config.hub;
3197
- }
3198
- /** The sync settings of the hub at `root`, or null when either the server or the token is missing. */
3199
- function syncSettings(root, env = process.env) {
3200
- const file = join(root, DATA_DIR, SETTINGS_FILE);
3201
- let stored = {};
3202
- if (existsSync(file)) try {
3203
- stored = JSON.parse(readFileSync(file, "utf8")) ?? {};
3204
- } catch (error) {
3205
- throw new Error(`${file} is not JSON: ${error.message}`);
3206
- }
3207
- const url = env[ENV.url] || (typeof stored.url === "string" ? stored.url : "");
3208
- const token = env[ENV.token] || (typeof stored.token === "string" ? stored.token : "");
3209
- if (!url || !token) return null;
3919
+ /**
3920
+ * What to download before the tool in `payload` runs in the hub at `root`: the artifact it reads or writes, when the file
3921
+ * is not here, so it may be only on the sync server. Null for anything else, which needs nothing.
3922
+ */
3923
+ function planFetch(root, payload) {
3924
+ const tool = payload.tool_name ?? "";
3925
+ if (!FETCH_TOOLS.has(tool)) return null;
3926
+ const path = hubPath(root, payload);
3927
+ if (path === null || !isSynced(path) || artifactOf(path) === null || existsSync(join(root, ".mainflow", path))) return null;
3210
3928
  return {
3211
- url: url.replace(/\/+$/, ""),
3212
- token,
3213
- hub: hubName(root)
3929
+ path,
3930
+ write: tool !== "Read"
3214
3931
  };
3215
3932
  }
3933
+ /** The PreToolUse answer that stops the tool, with `reason` for the agent. */
3934
+ const fetchDenial = (reason) => JSON.stringify({ hookSpecificOutput: {
3935
+ hookEventName: "PreToolUse",
3936
+ permissionDecision: "deny",
3937
+ permissionDecisionReason: `mainflow sync: ${reason}`
3938
+ } });
3216
3939
  //#endregion
3217
3940
  //#region ../sync/dist/outbox.js
3218
3941
  /** Whether the hub at `root` writes through the sync server. */
@@ -3326,7 +4049,7 @@ const alive = (pid) => {
3326
4049
  };
3327
4050
  /**
3328
4051
  * Make sure the hub at `root` has a running sync client, starting `mainflow sync` in the background when it has
3329
- * none. `env` gives the sync server and token as the environment variables do; for a hub other than the one this
4052
+ * none. `env` gives the sync server and service token as the environment variables do; for a hub other than the one this
3330
4053
  * process was started in it is `{}`, so the process started reads that hub's own .mainflow/local/sync.json.
3331
4054
  */
3332
4055
  async function startSyncInBackground(root, env = process.env) {
@@ -3368,8 +4091,7 @@ async function startSyncInBackground(root, env = process.env) {
3368
4091
  ],
3369
4092
  env: {
3370
4093
  ...process.env,
3371
- [ENV.url]: env[ENV.url] ?? "",
3372
- [ENV.token]: env[ENV.token] ?? ""
4094
+ ...Object.fromEntries(Object.values(ENV).map((name) => [name, env[name] ?? ""]))
3373
4095
  }
3374
4096
  });
3375
4097
  child.on("error", () => {});
@@ -3425,7 +4147,7 @@ function writtenLines(written) {
3425
4147
  ];
3426
4148
  }
3427
4149
  /**
3428
- * The writer for the hub at `root`. `env` gives the sync server and token as the environment variables do; a hub
4150
+ * The writer for the hub at `root`. `env` gives the sync server and service token as the environment variables do; a hub
3429
4151
  * other than the one this process was started in, such as another hub the interface shows, passes `{}`, so only its
3430
4152
  * own .mainflow/local/sync.json and sync client count.
3431
4153
  */
@@ -3441,7 +4163,7 @@ function hubWriter(root, env = process.env, options = {}) {
3441
4163
  let queuedOp;
3442
4164
  try {
3443
4165
  queuedOp = queueOperation(root, {
3444
- protocol: 2,
4166
+ protocol: 3,
3445
4167
  op,
3446
4168
  input,
3447
4169
  now: requestTime()
@@ -3487,4 +4209,4 @@ function hubWriter(root, env = process.env, options = {}) {
3487
4209
  };
3488
4210
  }
3489
4211
  //#endregion
3490
- export { ServerError as C, ServerApi as S, silentHint as _, remoteMode as a, isSynced as b, syncSettings as c, startDaemon as d, DaemonRunning as f, readDaemon as g, probeDaemon as h, queuedCount as i, hookAnswer as l, callDaemon as m, hubWriter as n, ENV as o, DaemonUnavailable as p, writtenLines as r, hubName as s, CONFIRM_MS as t, planFlush as u, SETTINGS_FILE as v, syncPaths as x, SYNC_DIR as y };
4212
+ export { syncSettings as C, syncPaths as D, isSynced as E, hubName as S, SYNC_DIR as T, readRemote as _, remoteMode as a, ENV as b, planFetch as c, DaemonRunning as d, DaemonUnavailable as f, silentHint as g, readDaemon as h, queuedCount as i, planFlush as l, probeDaemon as m, hubWriter as n, fetchDenial as o, callDaemon as p, writtenLines as r, hookAnswer as s, CONFIRM_MS as t, startDaemon as u, ServerApi as v, SETTINGS_FILE as w, earlierToken as x, ServerError as y };