dsh-usage-badge 0.1.5 → 0.1.7

package/lib/sync.js

@@ -104,20 +104,27 @@ export const safeShardKey = (key) => {
 /**
  * One device's local rows grouped into month shards.
  *
- * Only `current` rows this device folded are exported (`owns` selects them; it is a parameter so a
- * test can exercise the filter without pretending to be another device). Each row is copied out with
- * its `sessionId` attached, because a shard has to be able to rebuild a session record on the other
- * side.
+ * Only this installation's own `current` rows are exported (`owns` selects them; it is a parameter so a
+ * test can exercise the filter without pretending to be another device). Each row is copied out with its
+ * `sessionId` attached, because a shard has to be able to rebuild a session record on the other side.
+ *
+ * `owns` is usually `'local'`, but it may be a **set** of names: after a rename, a device pulls its own
+ * previous directory and imports those rows under the old name, and they are still this installation's
+ * history. Leaving them out means the device counts them (they resolve like any other row) while never
+ * publishing them, so a fresh machine sees less than its owner does — 18766 against 10000 in the case this
+ * was found with. A name only gets here after being checked against this installation's `instanceId`, so
+ * another machine's rows are never claimed.
  *
  * @param {object[]} records - records as `store.scan()` returns them, each with a `rows` array.
- * @param {{owns?:string}} [options]
+ * @param {{owns?:string|string[]}} [options]
  * @returns {Map<string, object[]>} shard key → rows, sorted by date then session.
  */
 export function chunkRows(records, { owns = LOCAL_SOURCE } = {}) {
+  const owners = new Set(Array.isArray(owns) ? owns : [owns])
   const shards = new Map()
   for (const record of records ?? []) {
     for (const row of record?.rows ?? []) {
-      if (rowSource(row) !== owns) continue
+      if (!owners.has(rowSource(row))) continue
       if (rowKind(row) === FALLBACK) continue
       const key = monthOf(row.date)
       if (!/^\d{4}-\d{2}$/.test(key)) continue
@@ -160,14 +167,18 @@ export function chunkRows(records, { owns = LOCAL_SOURCE } = {}) {
  *     mean "device `<id>` published it". Carrying `local` across would make the receiver treat
  *     another machine's history as **its own** — it would then publish it onward under its own name,
  *     and a later fold here could delete rows it does not own.
+ *
+ *     For the same reason every **other** name of this installation is rewritten too: a device that has
+ *     been renamed re-publishes rows it once imported under its old name, and they are published *now* by
+ *     this name. Leaving the old name in them would put a source in the receiver's store that no directory
+ *     on the server answers for.
  */
 export function encodeShard(rows, { device }) {
   const id = safeDeviceId(device)
   const header = JSON.stringify({ v: SHARD_VERSION, device: id })
-  const body = [
-    header,
-    ...rows.map((row) => JSON.stringify({ ...row, source: rowSource(row) === LOCAL_SOURCE ? id : rowSource(row) })),
-  ].join('\n')
+  // Everything in a shard was published by the device whose name is on it: `chunkRows` only ever hands over
+  // this installation's own rows, so the source is stamped rather than inherited.
+  const body = [header, ...rows.map((row) => JSON.stringify({ ...row, source: id }))].join('\n')
   const compressed = zstdCompressSync(Buffer.from(body, 'utf8'))
   return { body: compressed, sha256: sha256(compressed), rows: rows.length }
 }
@@ -245,12 +256,17 @@ export function chunkEntry({ device, key, body, rows, at }) {
   }
 }
 
-/** A device's manifest: what this device has published. */
-export function buildManifest({ device, label = '', chunks = {}, at = Date.now(), instanceId = '' }) {
+/** A device's manifest: what this device has published, and what it has withdrawn. */
+export function buildManifest({ device, label = '', chunks = {}, at = Date.now(), instanceId = '', retired = {} }) {
   // `instanceId` is written so that the *other* installations can tell whether the name in this
   // directory is theirs. Two machines that both call themselves `office` would otherwise each overwrite
   // the other's month — the shard path is the same — and the rows would be gone with no warning at all.
-  return { v: MANIFEST_VERSION, device, label, generatedAt: at, instanceId, chunks }
+  //
+  // `retired` is the **tombstone**: months this device published and no longer holds any of. A month that
+  // simply disappears from `chunks` is not enough, because a reader that already imported it has no way to
+  // learn that it is gone — it would keep those rows forever and the two machines would disagree for good.
+  // An entry is dropped as soon as the month comes back, so a restored log un-retires it naturally.
+  return { v: MANIFEST_VERSION, device, label, generatedAt: at, instanceId, chunks, retired }
 }
 
 /** Parse a manifest, refusing a version this build does not write and a device id it cannot use. */
@@ -279,7 +295,14 @@ export function parseManifest(text) {
       at: Number(entry.at) || 0,
     }
   }
-  return { version: raw.v, device, label: String(raw.label ?? ''), generatedAt: Number(raw.generatedAt) || 0, instanceId: String(raw.instanceId ?? ''), chunks }
+  // The tombstones travel with the manifest. Only the key and when it was withdrawn are needed; a reader
+  // compares them against what it holds, so an entry for a month it never imported costs nothing.
+  const retired = {}
+  for (const [key, entry] of Object.entries(raw.retired ?? {})) {
+    if (!/^\d{4}-\d{2}$/.test(key)) continue
+    retired[key] = { at: Number(entry?.at) || 0, sha256: String(entry?.sha256 ?? '') }
+  }
+  return { version: raw.v, device, label: String(raw.label ?? ''), generatedAt: Number(raw.generatedAt) || 0, instanceId: String(raw.instanceId ?? ''), chunks, retired }
 }
 
 /**
@@ -311,7 +334,11 @@ export function planUpload(shards, previous = {}, { device, label = '', at = Dat
     chunks[key] = entry
     pending.push({ key, body, entry, rows: rows.length })
   }
-  return { pending, unchanged, manifest: buildManifest({ device: safeDevice, label, chunks, at, instanceId }) }
+  // A month that is published again is no longer withdrawn, so its tombstone goes: otherwise a reader that
+  // imported the month before it was withdrawn would keep dropping it on every run.
+  const retired = { ...(previous?.retired ?? {}) }
+  for (const key of Object.keys(chunks)) delete retired[key]
+  return { pending, unchanged, manifest: buildManifest({ device: safeDevice, label, chunks, at, instanceId, retired }) }
 }
 
 /**
@@ -323,8 +350,9 @@ export function planUpload(shards, previous = {}, { device, label = '', at = Dat
  * manifest still lists everything it has.
  *
  * @param {{manifests:object[], held?:object, now?:number, windowYears?:number, except?:string}} options
- *   `held` maps `<device>/<key>` to the sha256 this reader already applied. `except` is the reader's
- *   own device id: its own manifest lists what it published, which is already in its store.
+ *   `held` maps `<device>/<key>` to what this reader already applied — either the sha256 (what older
+ *   builds wrote) or `{sha, at}`. `except` is the reader's own device id: its own manifest lists what it
+ *   published, which is already in its store.
  */
 export function planDownload({ manifests, held = {}, now = Date.now(), windowYears = 5, except = '' }) {
   const cutoff = `${new Date(now).getFullYear() - Math.max(0, windowYears) + 1}-01`
@@ -334,7 +362,7 @@ export function planDownload({ manifests, held = {}, now = Date.now(), windowYea
     for (const [key, entry] of Object.entries(manifest?.chunks ?? {})) {
       if (!entry?.path || !entry?.sha256) continue
       if (windowYears > 0 && key < cutoff) continue
-      if (held[`${manifest.device}/${key}`] === entry.sha256) continue
+      if (heldSha(held[`${manifest.device}/${key}`]) === entry.sha256) continue
       wanted.push({ device: manifest.device, key, entry })
     }
   }
@@ -342,6 +370,21 @@ export function planDownload({ manifests, held = {}, now = Date.now(), windowYea
   return wanted
 }
 
+/**
+ * What a `held` entry says was applied, whichever shape it is in.
+ *
+ * Older builds stored the bare sha; this one stores `{sha, at}` so that a shard **written earlier** than
+ * what is already held can be recognised as a rollback. A reader that cannot tell the difference would
+ * apply a restored old shard over newer rows — and with a replacing import that means deleting them.
+ */
+export function heldEntry(value) {
+  if (typeof value === 'string') return value ? { sha: value, at: 0 } : null
+  if (value && typeof value === 'object' && typeof value.sha === 'string') return { sha: value.sha, at: Number(value.at) || 0 }
+  return null
+}
+
+const heldSha = (value) => heldEntry(value)?.sha ?? null
+
 /**
  * Turn decoded shard rows into the records the store's import takes.
  *
@@ -506,6 +549,28 @@ export function acquireLock(dir, { instanceId, log = () => {} } = {}) {
     return { acquired: false, owner: null, reason: `the data directory cannot be created: ${error?.message ?? error}` }
   }
   const path = join(dir, LOCK_FILE)
+  const body = { pid: process.pid, startedAt: PROCESS_STARTED_AT, instanceId, hostname: hostname(), at: Date.now() }
+
+  /**
+   * Create the file **only if it is not there**.
+   *
+   * `wx` is the whole point: the earlier read-then-write had a window in which two processes could both
+   * see no lock (or both see a stale one) and both believe they had taken it. Now the kernel decides — one
+   * `open` wins, and the loser gets `EEXIST`.
+   */
+  const claim = () => {
+    try {
+      writeFileSync(path, JSON.stringify(body), { flag: 'wx' })
+      return { acquired: true, owner: body }
+    } catch (error) {
+      if (error?.code !== 'EEXIST') return { acquired: false, owner: null, reason: `could not write ${LOCK_FILE}: ${error?.message ?? error}` }
+      return null
+    }
+  }
+
+  const first = claim()
+  if (first) return first
+
   const existing = readLock(dir)
   if (existing && !lockIsStale(existing)) {
     return {
@@ -514,14 +579,23 @@ export function acquireLock(dir, { instanceId, log = () => {} } = {}) {
       reason: `this data directory is open by another DSH instance (pid ${existing.pid} on ${existing.hostname || 'this host'})`,
     }
   }
+  // Stale: remove it and try once more. If another process gets there first, its `wx` wins and ours sees
+  // `EEXIST` again — which is exactly the answer we want, and the reason this is safe without a lock file
+  // to guard the lock file.
   if (existing) log(`taking over a stale ${LOCK_FILE} left by pid ${existing.pid}`)
-  const body = { pid: process.pid, startedAt: PROCESS_STARTED_AT, instanceId, hostname: hostname(), at: Date.now() }
   try {
-    writeFileSync(path, JSON.stringify(body))
-  } catch (error) {
-    return { acquired: false, owner: null, reason: `could not write ${LOCK_FILE}: ${error?.message ?? error}` }
+    unlinkSync(path)
+  } catch {
+    // Someone else removed it, or it was never there. Either way the next claim decides.
+  }
+  const second = claim()
+  if (second) return second
+  const now = readLock(dir)
+  return {
+    acquired: false,
+    owner: now,
+    reason: now ? `this data directory is open by another DSH instance (pid ${now.pid} on ${now.hostname || 'this host'})` : `could not take ${LOCK_FILE}: another process took it first`,
   }
-  return { acquired: true, owner: body }
 }
 
 /** Release the lock, but only if this process is the one holding it. */
@@ -553,6 +627,10 @@ export const defaultSyncConfig = () => ({
   deviceLabel: hostname(),
   instanceId: '',
   windowYears: 5,
+  // The two automatic triggers, both on by default: a backup nobody remembers to run is not a backup. Both
+  // are switchable, because a sync that holds up quitting is worse than one that happens a minute later.
+  onStart: true,
+  onExit: true,
 })
 
 /** Read `sync.json`, or a default document when it is missing or unreadable. */
@@ -646,6 +724,17 @@ export class SyncEngine {
     return released
   }
 
+  /**
+   * Ask a run in progress to stop at its next step boundary.
+   *
+   * Requests already on the wire cannot be recalled, but the remaining phases can be skipped. That matters
+   * when the host is unloading: without it a run keeps going after `aggregator.close()`, writing state for
+   * a process that is going away and touching a store whose statements have been finalized.
+   */
+  abort() {
+    this.aborted = true
+  }
+
   get deviceId() {
     return this.config.deviceId
   }
@@ -718,7 +807,13 @@ export class SyncEngine {
     }
   }
 
-  /** Everything the remote knows, as manifests. */
+  /**
+   * Everything the remote knows: the directories that exist, and the manifests that could be read.
+   *
+   * Both are returned because they answer different questions. A directory whose manifest is missing or
+   * unreadable is not "nothing there" — it is a directory whose owner cannot be established, which is
+   * exactly when writing into it is most dangerous.
+   */
   async readManifests(client) {
     const devices = await this.discover(client)
     const manifests = []
@@ -726,7 +821,7 @@ export class SyncEngine {
       const manifest = await this.readManifest(client, device)
       if (manifest) manifests.push(manifest)
     }
-    return manifests
+    return { manifests, devices }
   }
 
   /**
@@ -758,6 +853,9 @@ export class SyncEngine {
     let fetched = 0
     const failed = []
     for (const item of wanted) {
+      // One shard at a time, and a shard is the natural boundary: stopping here leaves the store consistent
+      // and the untouched months to be fetched next run.
+      if (this.aborted) break
       let body
       try {
         body = await client.get(item.entry.path)
@@ -782,16 +880,64 @@ export class SyncEngine {
         failed.push({ path: item.entry.path, error: 'the shard does not match the hash its manifest advertises' })
         continue
       }
+      // A shard that does not decode into every row its manifest advertises is **not** remembered as
+      // applied. The good rows are still imported (that is idempotent), but advancing `held` would mark the
+      // month done forever while some of it was silently dropped — the row count and the manifest disagree,
+      // and the honest answer is to keep asking. This build never writes such a shard; an older one or a
+      // hand-made file can.
+      if (decoded.malformed > 0 || (item.entry.rows > 0 && decoded.rows.length !== item.entry.rows)) {
+        this.store?.intakeRows(decoded.rows)
+        failed.push({
+          path: item.entry.path,
+          error: `the shard holds ${decoded.rows.length} of the ${item.entry.rows || decoded.rows.length} rows its manifest names` + (decoded.malformed ? ` (${decoded.malformed} unreadable line(s))` : ''),
+        })
+        continue
+      }
+      // A shard written **earlier** than the one already applied is a rollback, not news: a restored
+      // backup or an older manifest would otherwise delete rows this reader has that the publisher no
+      // longer knows about. Skipped rather than applied, and said out loud.
+      const previous = heldEntry(this.state.held?.[`${item.device}/${item.key}`])
+      if (previous && item.entry.at && previous.at && previous.at > item.entry.at) {
+        this.log(`keeping ${item.device}/${item.key}: the server's copy is older than the one already applied`)
+        continue
+      }
       if (this.store) {
-        this.store.intakeRows(decoded.rows)
+        // `replace`: the month is this device's own statement about it, so rows it no longer lists go.
+        // Without that, a publisher that dropped a row keeps every reader counting it forever.
+        this.store.intakeRows(decoded.rows, { replace: { source: item.device, month: item.key } })
         // `held` is only advanced once the rows are in the store: if the import throws, the shard
         // must be fetched again rather than remembered as applied.
-        this.state.held = { ...(this.state.held ?? {}), [`${item.device}/${item.key}`]: item.entry.sha256 }
+        this.state.held = { ...(this.state.held ?? {}), [`${item.device}/${item.key}`]: { sha: item.entry.sha256, at: Number(item.entry.at) || 0 } }
       }
       rows += decoded.rows.length
       fetched++
     }
-    return { fetched, rows, failed }
+
+    // ── what the server says is gone ─────────────────────────────────────────
+    //
+    // A month that vanished from a manifest used to be invisible to a reader that had already imported
+    // it: `intakeRows` only adds, so the reader kept rows its publisher had dropped and the two machines
+    // disagreed for good. A device that withdraws a month now leaves a tombstone, and this is the other
+    // half — drop what came from that device for that month.
+    let dropped = 0
+    const droppedKeys = []
+    for (const manifest of manifests ?? []) {
+      const device = manifest?.device
+      if (!device || (this.pullOwnDirectory ? false : device === this.deviceId)) continue
+      for (const [key, entry] of Object.entries(manifest?.retired ?? {})) {
+        const context = `${device}/${key}`
+        // Only what this reader actually imported: a tombstone for a month it never held is nothing.
+        if (!this.state.held?.[context]) continue
+        const removed = this.store?.dropSourceMonth(device, key) ?? 0
+        const next = { ...(this.state.held ?? {}) }
+        delete next[context]
+        this.state.held = next
+        dropped += removed
+        droppedKeys.push(context)
+        this.log(`dropped ${removed} row(s) from ${context}: the device that published it no longer holds that month (withdrawn ${entry.at ? new Date(entry.at).toISOString().slice(0, 10) : 'at some point'})`)
+      }
+    }
+    return { fetched, rows, failed, dropped, droppedKeys }
   }
 
   /**
@@ -810,7 +956,38 @@ export class SyncEngine {
     // The owner is recorded in the directory's own manifest. A directory with no recorded owner is one
     // published by a build that did not write the field (or the first publish of anything), and adopting
     // it is what makes an upgrade work; an owner that is a *different* installation is a real clash.
-    const remote = (await this.readManifests(client)).find((manifest) => manifest.device === this.deviceId)
+    const { manifests: discovered, devices: directories } = await this.readManifests(client)
+    const remote = discovered.find((manifest) => manifest.device === this.deviceId)
+    // A directory that exists but whose manifest cannot be read is not "nothing there": nobody knows whose
+    // it is. Writing into it because the owner check had nothing to look at is the one shape of this bug
+    // the instanceId guard cannot see, so an **unreadable** manifest counts as occupied — unless this
+    // device has published here before, in which case it is this device's own torn manifest and repairing
+    // it is the whole point of the next publish.
+    const unknownHere = directories.includes(this.deviceId) && !remote && Object.keys(this.state.published ?? {}).length === 0
+    if (unknownHere) {
+      this.state.collision = { device: this.deviceId, owner: null, kind: 'unreadable', at: this.now() }
+      this.log(
+        `not publishing: usage-badge/${this.deviceId}/ exists but its manifest could not be read, so this device cannot tell whose it is. ` +
+          `Pick a device name that is not taken, or remove that directory on the server if it is yours.`,
+      )
+      return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'unreadable' }
+    }
+    // Two directories whose names differ only in case land in **one** directory on a case-insensitive
+    // server (Windows/SMB-backed DAV), where the owner check by exact name would never fire. Refused rather
+    // than merged: on the target server (Linux Apache) they are genuinely two, and a name that differs only
+    // in case is never what someone means.
+    const caseTwin = discovered.find((manifest) => manifest.device !== this.deviceId && manifest.device.toLowerCase() === String(this.deviceId).toLowerCase())
+    if (caseTwin) {
+      this.state.collision = { device: this.deviceId, owner: caseTwin.instanceId || 'unknown', kind: 'case', at: this.now() }
+      this.log(`not publishing: usage-badge/${caseTwin.device}/ differs from "${this.deviceId}" only in case, which is one directory on a case-insensitive server`)
+      return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'case' }
+    }
+    // **This installation**, not this name. A rename leaves the previous directory behind with its own
+    // manifest, and every reader treats every manifest it finds as authoritative — so a month withdrawn
+    // after a rename would come straight back through the old name. The `instanceId` is what identifies the
+    // installation across names; two installations never share one, and a manifest without one is not ours
+    // to speak for.
+    const mine = this.instanceId ? discovered.filter((manifest) => manifest.instanceId === this.instanceId) : []
     if (remote && remote.instanceId && remote.instanceId !== this.instanceId) {
       this.state.collision = { device: this.deviceId, owner: remote.instanceId, at: this.now() }
       this.log(
@@ -830,11 +1007,12 @@ export class SyncEngine {
     }
     delete this.state.collision
 
-    const rows = chunkRows(this.store.scan())
+    const rows = chunkRows(this.store.scan(), { owns: [LOCAL_SOURCE, this.deviceId, ...mine.map((manifest) => manifest.device)] })
     // `state.published` is the manifest's **chunks map**, so it is wrapped back into a manifest shape
     // here. Passing the map itself would leave `previous.chunks` empty, and every run would re-upload
-    // every month while looking perfectly healthy.
-    const { pending, unchanged, manifest } = planUpload(rows, { chunks: this.state.published ?? {} }, {
+    // every month while looking perfectly healthy. `retired` rides along for the same reason: it is the
+    // record of withdrawals a reader needs in order to drop what it once imported.
+    const { pending, unchanged, manifest } = planUpload(rows, { chunks: this.state.published ?? {}, retired: this.state.retired ?? {} }, {
       device: this.deviceId,
       label: this.config.deviceLabel ?? '',
       at: this.now(),
@@ -861,13 +1039,6 @@ export class SyncEngine {
       return { uploaded: 0, unchanged: 0, bytes: 0, shards: [], refused: 'diverged' }
     }
 
-    // A month this device published and no longer has any rows for: a session folder was deleted, or a
-    // log version that no longer covers that day superseded the rows. The manifest is republished from
-    // what is actually held, so **new** devices must stop being told about it — otherwise they import
-    // usage its own machine no longer has, which is precisely the "the numbers came back" surprise. The
-    // bytes stay on the server; only the claim that this device still holds them is withdrawn.
-    const retired = Object.keys(this.state.published ?? {}).filter((key) => !rows.has(key))
-
     // The collection comes **before** anything is written into it. `mod_dav_fs` refuses a write whose
     // parent does not exist (403/409, not the 404 an HTTP client might expect), so creating it on the
     // way past would fail the first publish of a device — and mkcol is idempotent anyway, so it costs
@@ -876,6 +1047,9 @@ export class SyncEngine {
     let bytes = 0
     const shards = []
     for (const item of pending) {
+      // Same boundary as the pull: an upload already on the wire cannot be recalled, but the rest of the
+      // list can wait for a run that will still have a store to write to.
+      if (this.aborted) break
       // Upload under a temporary name and rename: a crash mid-upload leaves a name nobody reads,
       // instead of a shard that looks complete and is half a month.
       const temp = `${item.entry.path}.upload-${process.pid}-${Math.random().toString(16).slice(2, 8)}`
@@ -894,27 +1068,70 @@ export class SyncEngine {
     // **last** on purpose: a manifest naming a shard that has not landed yet would send every other
     // device after a 404 on their next run.
     await client.put(`${REMOTE_PREFIX}${this.deviceId}/${DEVICE_MARKER}`, Buffer.from(`${this.deviceId}\n`))
+    // ── what every other name of this installation must stop claiming ──────────
+    //
+    // A rename leaves the previous directory behind, and **every reader treats every manifest it finds as
+    // authoritative**. So a month that was corrected or dropped after a rename keeps feeding readers its old
+    // rows through the old name: the fix is that this installation's own months live in exactly one
+    // directory, and every other directory of ours is brought in line with what we actually hold.
+    //
+    // The comparison is by content hash, not by month: a month can stay while one of its rows goes away
+    // (a log version that no longer covers that day), and that difference is invisible to a key-vs-key test.
+    // A key we no longer publish at all is simply missing from the current chunks, which is the same test.
+    //
+    // The current name's manifest is the one `planUpload` built; the others are rebuilt through
+    // `buildManifest` — **not** written back as parsed. A parsed manifest carries `version` while the file
+    // format is `v`, so writing one back produces a manifest no reader can parse, which would hide that
+    // directory's shards from everyone instead of correcting them.
+    const targets = new Map([[this.deviceId, manifest]])
+    for (const other of mine) {
+      if (targets.has(other.device)) continue
+      targets.set(other.device, buildManifest({ device: other.device, label: other.label, chunks: { ...other.chunks }, at: other.generatedAt || this.now(), instanceId: other.instanceId, retired: { ...(other.retired ?? {}) } }))
+    }
+    const withdrawals = []
+    for (const [key, entry] of Object.entries(manifest.chunks)) {
+      if (rows.has(key)) continue
+      withdrawals.push({ key, device: this.deviceId, manifest, path: entry.path ?? shardPath(this.deviceId, key) })
+    }
+    for (const [device, target] of targets) {
+      if (device === this.deviceId) continue
+      for (const [key, entry] of Object.entries(target.chunks)) {
+        const current = manifest.chunks[key]
+        if (current && current.sha256 === entry.sha256) continue
+        withdrawals.push({ key, device, manifest: target, path: entry.path ?? shardPath(device, key) })
+      }
+    }
+    // **Delete first, then write.** Each deletion is recorded, and a month whose shard could not be removed
+    // stays in the manifest that lists it — so the claim and the bytes never disagree, and the failure is
+    // retried on the next run instead of being half-recorded.
     let withdrawn = 0
     const withdrawnKeys = []
-    for (const key of retired) {
+    const withdrawErrors = []
+    for (const item of withdrawals) {
       try {
         // Gone from the manifest *and* from the server: leaving the bytes behind while dropping the claim
-        // would be tidier for a manual rescue and worse for everything else — the next reader that finds
-        // the manifest is authoritative would see a shard nobody lists. The shard is *this device's own*,
-        // which is the one direction where deleting is not the remote deciding anything.
-        await client.delete(shardPath(this.deviceId, key))
-        delete manifest.chunks[key]
+        // would leave a shard nobody lists. These shards are this installation's own, which is the one
+        // direction where deleting is not the remote deciding anything.
+        await client.delete(item.path)
+        const gone = item.manifest.chunks[item.key]
+        delete item.manifest.chunks[item.key]
+        // A **tombstone**, so a device that already imported this month learns that it is gone. Without it
+        // a reader keeps those rows forever — the publisher drops them, and the two machines disagree for
+        // good, in the direction of "everyone else counts more than the machine that has the data".
+        item.manifest.retired = { ...(item.manifest.retired ?? {}), [item.key]: { at: this.now(), sha256: String(gone?.sha256 ?? '') } }
         withdrawn++
-        withdrawnKeys.push(key)
-        this.log(`withdrew ${key}: this device no longer holds any rows for it`)
+        if (!withdrawnKeys.includes(item.key)) withdrawnKeys.push(item.key)
+        this.log(`withdrew ${item.key} from ${item.device}/: this device publishes it under ${this.deviceId}/ or no longer holds it`)
       } catch (error) {
-        // The month could not be withdrawn, so the manifest is not written at all: half-recording it
-        // would leave the claim in place while the shard is gone. Reported, and retried next run.
-        return { uploaded: pending.length, unchanged, bytes, shards, manifest: null, withdrawn: 0, refused: null, withdrawError: `could not withdraw ${key}: ${error?.message ?? error}` }
+        withdrawErrors.push(`could not withdraw ${item.key} from ${item.device}/: ${error?.message ?? error}`)
       }
     }
-    await client.put(manifestPath(this.deviceId), Buffer.from(JSON.stringify(manifest, null, 2)), { contentType: 'application/json' })
-    return { uploaded: pending.length, unchanged, bytes, shards, manifest, withdrawn, withdrawnKeys }
+    // Written even when a withdrawal failed: the manifests now say exactly what the server holds, and the
+    // current one still names every shard that was uploaded above (so `uploaded` is not an exaggeration).
+    for (const [device, target] of targets) {
+      await client.put(manifestPath(device), Buffer.from(JSON.stringify(target, null, 2)), { contentType: 'application/json' })
+    }
+    return { uploaded: pending.length, unchanged, bytes, shards, manifest, withdrawn, withdrawnKeys, withdrawErrors }
   }
 
   /**
@@ -1096,9 +1313,14 @@ export class SyncEngine {
     } catch (error) {
       return this.finish('failed', String(error?.message ?? error), { started })
     }
+    // The host can unload while this is in flight. Checked at every phase boundary rather than only at the
+    // start, because the phases are where a run touches the store — and after `aggregator.close()` its
+    // statements are finalized, so continuing is not harmless.
+    if (this.aborted) return this.finish('aborted', 'the host unloaded during the sync', { started })
 
     try {
-      const manifests = await this.readManifests(client)
+      const { manifests } = await this.readManifests(client)
+      if (this.aborted) return this.finish('aborted', 'the host unloaded during the sync', { started })
       // **Pull first, refuse only the push.** An installation whose device name is taken still wants the
       // history back, and that is the case the feature exists for (reinstall, another machine): the name
       // is what is in the way, not the data. So the clash is noted here — which is what tells `pull` to
@@ -1113,10 +1335,17 @@ export class SyncEngine {
       this.pullOwnDirectory = Boolean(clash)
       const pulled = await this.pull(client, manifests)
       this.pullOwnDirectory = false
+      if (this.aborted) return this.finish('aborted', `the host unloaded during the sync (${pulled.fetched} shard(s) had arrived)`, { started })
       const pushed = await this.push(client)
-      if (pushed.manifest) this.state.published = pushed.manifest.chunks
+      if (pushed.manifest) {
+        this.state.published = pushed.manifest.chunks
+        // The tombstones are state too: the next run has to keep carrying them, or a reader that has not
+        // synced yet would never learn that a month is gone.
+        this.state.retired = pushed.manifest.retired ?? {}
+      }
       // The price table last: it is the only hand-written thing here, and it should not be touched by
       // a run that failed to move the usage it was about to move.
+      if (this.aborted) return this.finish('aborted', 'the host unloaded during the sync', { started })
       const pricing = await this.syncPricing(client)
       const parts = [
         `pulled ${pulled.fetched} shard(s) / ${pulled.rows} row(s)`,
@@ -1128,8 +1357,9 @@ export class SyncEngine {
         // the pull worked and nothing was lost. Naming it is what lets the panel say what to do.
         parts.push(`nothing published: device name "${this.deviceId}" is being written by another installation`)
       }
-      if (pushed.withdrawError) parts.push(pushed.withdrawError)
+      for (const error of pushed.withdrawErrors ?? []) parts.push(error)
       if (pushed.withdrawn) parts.push(`withdrew ${pushed.withdrawn} month(s) this device no longer holds`)
+      if (pulled.dropped) parts.push(`dropped ${pulled.dropped} row(s) another device has withdrawn`)
       if (pricing.action !== 'none') parts.push(`prices: ${pricing.status}`)
       if (pulled.failed.length > 0) parts.push(`${pulled.failed.length} shard(s) failed`)
       const message = parts.join(', ')
@@ -1137,7 +1367,8 @@ export class SyncEngine {
       // Four outcomes, each meaning a different thing to the person reading the panel: `ok` moved
       // everything, `partial` moved almost everything, `clash` deliberately published nothing and needs a
       // decision from them, `failed` is a problem with the server or the network.
-      const result = pushed.refused ? 'clash' : pulled.failed.length > 0 || pushed.withdrawError ? 'partial' : 'ok'
+      const incomplete = pulled.failed.length > 0 || (pushed.withdrawErrors?.length ?? 0) > 0
+      const result = pushed.refused ? 'clash' : incomplete ? 'partial' : 'ok'
       return this.finish(result, message, {
         started,
         extra: {
@@ -1149,6 +1380,7 @@ export class SyncEngine {
           failed: pulled.failed,
           refused: pushed.refused ?? null,
           withdrawn: pushed.withdrawn ?? 0,
+          dropped: pulled.dropped ?? 0,
         },
       })
     } catch (error) {
@@ -1166,7 +1398,10 @@ export class SyncEngine {
    * — which is exactly how a sync ends up re-importing every shard on every run.
    */
   finish(result, message, { started, extra = {} } = {}) {
-    this.state.lastSyncAt = this.now()
+    // `skipped` (sync turned off between scheduling and running) deliberately leaves the timestamp alone:
+    // nothing was synced, and stamping it would show "上次同步：刚刚" for a run that never happened. `locked`
+    // returns before this, for the same reason.
+    if (result !== 'skipped') this.state.lastSyncAt = this.now()
     this.state.lastResult = result
     this.state.message = message
     try {