@khoavu882/doflow 2.0.0-beta.5 → 2.0.0

package/src/runtime/task-state.js

@@ -13,7 +13,10 @@
 // This module is the one seam both stores persist through:
 //
 //   - a per-file lock (`<file>.lock` directory — mkdirSync is atomic on every platform) makes the
-//     read-merge-write below a critical section across processes on this machine;
+//     read-merge-write below a critical section across processes on this machine; the holder
+//     marks the directory with its own `owner-*` entry, so release and the pre-write check act on
+//     that holder's lock only, and a stale lock is reclaimed in place by one process (see the
+//     protocol above `attemptLock`);
 //   - inside the lock, the writer re-reads the file and MERGES what is on disk into its own view
 //     before writing, so an overlapping writer's items survive;
 //   - the payload carries a monotonically increasing `revision`, so an overlapped-and-merged
@@ -24,6 +27,7 @@
 //     the version named, instead of guessing at its shape.
 
 const path = require('node:path');
+const { randomBytes } = require('node:crypto');
 const { isDeepStrictEqual } = require('node:util');
 
 /** Three-way merge against the reader's snapshot. Unchanged local records never overwrite a
@@ -52,74 +56,340 @@ function sleep(ms        )       {
   Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, ms);
 }
 
+// ── The lock protocol (117 Phase C; FU-8acds5, FU-vgkrvd) ───────────────────────────────────────────
+//
+// The lock is the `<file>.lock` directory, created with mkdirSync. A holder marks it with its own
+// `owner-<pid>-<random>` entry. Before this protocol any process that found the lock older than
+// LOCK_STALE_MS removed it by name and every release removed it by name, so two waiters that judged
+// one stale lock could both take it and each one's release could delete the other's. Three facts
+// carry the protocol (the safety argument, one interleaving at a time, is in the 117 feature records:
+// research/phase-c-protocol.md):
+//
+//   - Pinned: no DoFlow runtime renames a lock directory and rmdir removes only an empty one, so a
+//     directory with an entry in it stays at `<file>.lock` until it is empty. A process whose entry
+//     is in the lock knows the directory at that path is the one holding its entry.
+//   - Released: this runtime removes a lock directory only in its release, after removing its own
+//     entry from the lock it held; it reclaims a stale lock in place. Older runtimes remove one by
+//     name in their release, or after judging a lock stale.
+//   - Alone: a process holds the lock when its entry is the only entry in it. Two processes whose
+//     entries are in one directory cannot both have been alone in it.
+//
+//   - Acquire: note the time, mkdir `<file>.lock`, mkdir its entry inside, read the time again. Under
+//     PUBLISH_BOUND_MS, no process can have judged the new directory stale before the entry landed,
+//     so the entry is in this process's own directory. Over it, the entry may have landed in a
+//     directory another process made after reclaiming this one (an older runtime's empty lock looks
+//     exactly like a new one), so the process removes its entry and starts over. Then it lists the
+//     lock: alone, it holds.
+//   - Reclaim (the lock is older than LOCK_STALE_MS): stat, list, stat again (the same directory,
+//     unchanged, so every entry listed has been there that long); mkdir its own entry; stat again:
+//     not the same directory means the judged one was removed and the entry landed in a newer lock,
+//     so it removes its entry and starts over. Then it removes the judged entries by name (names are
+//     unique, so a later holder's entry is never among them) and lists the lock: alone, it holds. The
+//     directory never empties on the way, so an older runtime's rmdir of a stale lock fails instead of
+//     removing it.
+//   - Contention: two entries mean two processes reached the lock. The one whose entry name sorts
+//     first waits for the other to leave (it holds next); the other removes its entry and waits. A
+//     process that removes its entry and finds the lock emptied enters it again (once, under a fresh
+//     name, after checking it is the same directory) instead of leaving an empty lock everyone would
+//     wait 10 s on.
+//   - Release notes the lock's identity while its entry is in it, removes the entry, then removes the
+//     directory only if it is still that one (an older runtime may have made a new, empty lock there
+//     if the release stalled). A holder whose entry is gone (its lock was reclaimed) removes nothing;
+//     a waiting process's entry keeps the directory, and that process holds next.
+//   - `updateTaskState` checks the holder's entry is still in the lock (`release.held()`) after
+//     writing its tmp file and before the rename: a holder that was paused past LOCK_STALE_MS and
+//     reclaimed writes nothing.
+//
+// Runtimes from before this protocol see a directory they did not create: they wait on a fresh one,
+// and on a stale one their rmdir fails (it is not empty), so they fail closed ("Could not lock",
+// `store-locked`) instead of taking it; their release of a lock reclaimed from them fails the same
+// way. Whatever a process leaves when it dies is reclaimed once LOCK_STALE_MS old. The bounds (what
+// this does not cover) are listed once, in docs/architecture.md under "The task-state lock".
+const OWNER_PREFIX = 'owner-';
+// The longest a fresh acquire may take from its mkdir to its entry and still trust the directory is
+// its own: half the stale age, so a process that judged it stale cannot have done so before the
+// entry landed, with room for coarse file timestamps.
+const PUBLISH_BOUND_MS = LOCK_STALE_MS / 2;
+// How long tryAcquireLock waits for a process that reached the lock at the same moment to leave.
+const CONTEND_MS = 40;
+const CONTEND_WAIT_MS = 2;
+
+                                                     
+
 /**
- * Makes one non-waiting attempt to take the cross-process lock for a state file. A lock older than
- * LOCK_STALE_MS may be broken; a live holder is never waited on. Returns a release function or null.
- * @param {Object} fsImpl
- * @param {string} file
- * @returns {(() => void)|null}
+ * Optional lock settings (FU-31dyvs, for a store whose holders are identified by their entry): the
+ * defaults are the protocol above, unchanged.
+ *   - `stale` replaces the age rule of a reclaim. It is asked after the listing re-check, with the
+ *     entries listed and the lock's age, and the reclaim goes ahead only when it says true. It must
+ *     treat an empty lock by age (an acquire in progress makes an empty lock first; see
+ *     PUBLISH_BOUND_MS), and it may judge an entry by its owner: an entry whose owner is dead can be
+ *     reclaimed at any age, because a dead process never acts on it again.
+ *   - `entryName` makes this process's entry name (default `owner-<pid>-<random>`); every name it
+ *     makes must be unique.
+ *   - `removeEntry` removes one judged entry by name (default rmdir); it throws as rmdir would.
+ *   - `retries` is acquireLock's attempt budget (default LOCK_RETRIES, each LOCK_WAIT_MS apart).
  */
-function tryAcquireLock(fsImpl     , file        )                      {
-  const lockDir = `${file}.lock`;
-  const release = () => { try { fsImpl.rmdirSync(lockDir); } catch { /* released is released */ } };
+                    
+                                                                                   
+                           
+                                                                     
+                   
+  
+
+/** Same directory: same device and inode, and the same birth time where the platform reports one. */
+function sameDir(a     , b     )          {
+  return a.dev === b.dev && a.ino === b.ino && (!a.birthtimeMs || !b.birthtimeMs || a.birthtimeMs === b.birthtimeMs);
+}
+
+/** This attempt's entry name; `leave` gives it a fresh one when it re-enters an emptied lock. */
+                              
+
+/** Removes this process's entry; gone already is fine (a reclaimer removed it). */
+function withdraw(fsImpl     , lockDir        , me       )       {
+  try { fsImpl.rmdirSync(path.join(lockDir, me.name)); } catch { /* not there: nothing of ours is left */ }
+}
+
+/**
+ * With this process's entry in the lock: `held` when it is the only entry; `queued` when another
+ * entry is there and ours sorts first (we hold once it leaves); otherwise we leave (`busy`, or `held`
+ * when leaving found everyone else gone). `again` when our entry is gone (the lock was reclaimed from
+ * us). With `rejoin` false (an entry that came back to an emptied lock) it never queues or comes back.
+ */
+function elect(fsImpl     , lockDir        , me       , rejoin = true, options              = {})           {
+  let entries          ;
+  let pinned     ;
   try {
-    fsImpl.mkdirSync(lockDir);
-    return release;
-  } catch (error     ) {
-    if (error.code !== 'EEXIST') throw error;
+    entries = fsImpl.readdirSync(lockDir);
+    if (!entries.includes(me.name)) return 'again';
+    if (entries.length === 1) return 'held';
+    if (rejoin && entries.every((name        ) => name === me.name || me.name < name)) return 'queued';
+    pinned = pinnedDir(fsImpl, lockDir, me);
+  } catch { pinned = null; }
+  if (!pinned) { withdraw(fsImpl, lockDir, me); return 'again'; }
+  if (!rejoin) { withdraw(fsImpl, lockDir, me); return 'busy'; }
+  return leave(fsImpl, lockDir, me, pinned, options);
+}
+
+/** The stat of the lock while our entry is in it (so it is the directory holding our entry), or null. */
+function pinnedDir(fsImpl     , lockDir        , me       )      {
+  try {
+    const stat = fsImpl.statSync(lockDir);
+    return fsImpl.existsSync(path.join(lockDir, me.name)) ? stat : null;
+  } catch { return null; }
+}
+
+/**
+ * Removes our entry from the lock `pinned` (its stat taken while our entry was in it). If that left
+ * it empty, everyone else left after we looked, and an empty lock would block every process until it
+ * is stale: we enter it again, once, under a fresh name (a reclaimer that listed the old name removes
+ * names it listed), and hold it if we are alone. Never removes the directory.
+ */
+function leave(fsImpl     , lockDir        , me       , pinned     , options              = {})           {
+  withdraw(fsImpl, lockDir, me);
+  try {
+    if (fsImpl.readdirSync(lockDir).length > 0) return 'busy';
+    me.name = (options.entryName || newToken)();
+    fsImpl.mkdirSync(path.join(lockDir, me.name));
+  } catch (error     ) { return error.code === 'ENOENT' ? 'again' : 'busy'; }
+  let same = false;
+  try { same = sameDir(pinned, fsImpl.statSync(lockDir)); } catch { /* treat it as changed */ }
+  if (!same) { withdraw(fsImpl, lockDir, me); return 'again'; }
+  return elect(fsImpl, lockDir, me, false, options);
+}
+
+/** A queued attempt that stops waiting: leave the lock (or hold it, when the other left meanwhile). */
+function giveUp(fsImpl     , lockDir        , me       , options              = {})           {
+  const step = elect(fsImpl, lockDir, me, true, options);
+  if (step !== 'queued') return step;
+  const pinned = pinnedDir(fsImpl, lockDir, me);
+  if (!pinned) { withdraw(fsImpl, lockDir, me); return 'again'; }
+  return leave(fsImpl, lockDir, me, pinned, options);
+}
+
+/** Reclaims the lock at `lockDir` in place when it is stale. `again` means look again now. */
+function reclaimIfStale(fsImpl     , lockDir        , me       , options              = {})           {
+  const missing = (error     )           => (error.code === 'ENOENT' ? 'again' : 'busy');
+  let judged     ;
+  let entries          ;
+  try {
+    judged = fsImpl.statSync(lockDir);
+    if (!options.stale && Date.now() - judged.mtimeMs <= LOCK_STALE_MS) return 'busy';
+    entries = fsImpl.readdirSync(lockDir);
+    const listed = fsImpl.statSync(lockDir);
+    if (!sameDir(judged, listed) || listed.mtimeMs !== judged.mtimeMs) return 'again';
+  } catch (error     ) { return missing(error); }
+  if (options.stale && !options.stale({ lockDir, entries, ageMs: Date.now() - judged.mtimeMs })) return 'busy';
+  try { fsImpl.mkdirSync(path.join(lockDir, me.name)); } catch (error     ) { return missing(error); }
+  // Our entry pins the directory it landed in; it must be the one judged stale.
+  let same = false;
+  try { same = sameDir(judged, fsImpl.statSync(lockDir)); } catch { /* treat it as changed */ }
+  if (!same) {
+    withdraw(fsImpl, lockDir, me);
+    return 'again';
   }
-  let lock     ;
-  try { lock = fsImpl.statSync(lockDir); } catch (error     ) {
-    if (error.code !== 'ENOENT') return null;
-    try { fsImpl.mkdirSync(lockDir); return release; } catch (retryError     ) {
-      if (retryError.code === 'EEXIST') return null;
-      throw retryError;
+  for (const name of entries) {
+    try {
+      if (options.removeEntry) options.removeEntry(fsImpl, lockDir, name);
+      else fsImpl.rmdirSync(path.join(lockDir, name));
+    } catch (error     ) {
+      if (error.code === 'ENOENT') continue; // another reclaimer, or its released holder, removed it
+      withdraw(fsImpl, lockDir, me); // not an entry this protocol makes: leave the lock alone
+      return 'busy';
     }
   }
-  if (Date.now() - lock.mtimeMs <= LOCK_STALE_MS) return null;
-  try { fsImpl.rmdirSync(lockDir); } catch (error     ) {
-    if (error.code !== 'ENOENT') return null;
+  return elect(fsImpl, lockDir, me, true, options);
+}
+
+/**
+ * A queued attempt's next look, for a lock with a `stale` rule (FU-31dyvs C-5): the other entries are
+ * judged as a reclaim judges them (listing re-check, `stale`, the directory still the one holding our
+ * entry), and when they are stale they are removed by name, so a queued process does not wait out its
+ * budget behind a holder that died. Without a `stale` rule this is `elect`, unchanged.
+ */
+function queuedStep(fsImpl     , lockDir        , me       , options             )           {
+  if (!options.stale) return elect(fsImpl, lockDir, me, true, options);
+  let judged     ;
+  let others          ;
+  try {
+    judged = fsImpl.statSync(lockDir);
+    const entries           = fsImpl.readdirSync(lockDir);
+    const listed = fsImpl.statSync(lockDir);
+    if (!sameDir(judged, listed) || listed.mtimeMs !== judged.mtimeMs || !entries.includes(me.name)) return elect(fsImpl, lockDir, me, true, options);
+    others = entries.filter((name        ) => name !== me.name);
+  } catch { return elect(fsImpl, lockDir, me, true, options); }
+  if (others.length === 0 || !options.stale({ lockDir, entries: others, ageMs: Date.now() - judged.mtimeMs })) return elect(fsImpl, lockDir, me, true, options);
+  const pinned = pinnedDir(fsImpl, lockDir, me);
+  if (!pinned || !sameDir(pinned, judged)) return elect(fsImpl, lockDir, me, true, options);
+  for (const name of others) {
+    try {
+      if (options.removeEntry) options.removeEntry(fsImpl, lockDir, name);
+      else fsImpl.rmdirSync(path.join(lockDir, name));
+    } catch (error     ) {
+      if (error.code !== 'ENOENT') break; // not removable now: elect decides as before
+    }
+  }
+  return elect(fsImpl, lockDir, me, true, options);
+}
+
+/** One step towards holding the lock under `me`: held, queued (our entry waits in it), busy, or again. */
+function attemptLock(fsImpl     , lockDir        , me       , options              = {})           {
+  const started = Date.now();
+  try {
+    fsImpl.mkdirSync(lockDir);
+  } catch (error     ) {
+    // Windows reports EPERM/EACCES for a lock folder another process is deleting at this moment.
+    if (error.code !== 'EEXIST' && !(process.platform === 'win32' && (error.code === 'EPERM' || error.code === 'EACCES'))) throw error;
+    return reclaimIfStale(fsImpl, lockDir, me, options);
   }
-  try { fsImpl.mkdirSync(lockDir); return release; } catch (error     ) {
-    if (error.code === 'EEXIST') return null;
+  try {
+    fsImpl.mkdirSync(path.join(lockDir, me.name));
+  } catch (error     ) {
+    if (error.code === 'ENOENT') return 'again'; // removed before it was marked: an older runtime's release or stale break
     throw error;
   }
+  if (Date.now() - started >= PUBLISH_BOUND_MS) {
+    // Paused long enough for the new directory to be judged stale and replaced: the entry may sit in
+    // another process's lock. Leave it; the directory (ours or not) is left as found.
+    withdraw(fsImpl, lockDir, me);
+    return 'again';
+  }
+  return elect(fsImpl, lockDir, me, true, options);
 }
 
+/** The release function of a held lock; `release.held()` re-checks ownership before a write. */
+function ownedRelease(fsImpl     , lockDir        , me       )      {
+  const entry = path.join(lockDir, me.name);
+  let released = false;
+  const release      = () => {
+    if (released) return;
+    released = true;
+    const mine = pinnedDir(fsImpl, lockDir, me); // the directory we hold, while our entry pins it
+    try { fsImpl.rmdirSync(entry); } catch { return; } // reclaimed: the lock is not ours to remove
+    // Removed only if it is still that directory: a release stalled here past LOCK_STALE_MS may find
+    // an older runtime's new (empty) lock at the path. What stays is the gap between this stat and the
+    // rmdir (docs/architecture.md, "The task-state lock", bound 3).
+    try {
+      if (mine && sameDir(mine, fsImpl.statSync(lockDir))) fsImpl.rmdirSync(lockDir);
+    } catch { /* a waiting process's entry is in it; that process holds next */ }
+  };
+  release.held = () => !released && fsImpl.existsSync(entry);
+  return release;
+}
+
+const newToken = () => `${OWNER_PREFIX}${process.pid}-${randomBytes(6).toString('hex')}`;
+
 /**
- * Takes the cross-process lock for one state file, retrying for the existing bounded wait interval.
- * A stale lock is broken because its holder died mid-write; the rename-based write keeps the file whole.
+ * Makes one attempt to take the cross-process lock for a state file, without waiting on a holder:
+ * a lock older than LOCK_STALE_MS is reclaimed, a live holder is never waited on. It waits only when
+ * another process reached the lock at the same moment, for at most CONTEND_MS (40 ms) for that
+ * process to leave. Returns a release function or null.
  * @param {Object} fsImpl
  * @param {string} file
+ * @param {LockOptions} [options] see LockOptions; the defaults are the protocol above
+ * @returns {(() => void)|null}
+ */
+function tryAcquireLock(fsImpl     , file        , options              = {})                      {
+  const lockDir = `${file}.lock`;
+  const deadline = Date.now() + CONTEND_MS;
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    const me = { name: (options.entryName || newToken)() };
+    let step = attemptLock(fsImpl, lockDir, me, options);
+    while (step === 'queued' && Date.now() < deadline) {
+      sleep(CONTEND_WAIT_MS);
+      step = queuedStep(fsImpl, lockDir, me, options);
+    }
+    if (step === 'queued') step = giveUp(fsImpl, lockDir, me, options);
+    if (step === 'held') return ownedRelease(fsImpl, lockDir, me);
+    if (step === 'busy') return null;
+  }
+  return null;
+}
+
+/**
+ * Takes the cross-process lock for one state file, waiting up to LOCK_RETRIES * LOCK_WAIT_MS (5 s).
+ * A lock older than LOCK_STALE_MS is reclaimed: its holder died or stalled, and the rename-based write
+ * keeps the file whole.
+ * @param {Object} fsImpl
+ * @param {string} file
+ * @param {LockOptions} [options] see LockOptions; the defaults are the protocol above
  * @returns {() => void}
  */
-function acquireLock(fsImpl     , file        )             {
+function acquireLock(fsImpl     , file        , options              = {})             {
   const lockDir = `${file}.lock`;
+  const retries = options.retries ?? LOCK_RETRIES;
   // The stale rule below is wall-clock, so the wait must be too: on a host where each attempt costs
-  // real time, counting attempts alone would outlive a live holder's staleness window and then
-  // break (and later release) its lock.
-  const deadline = Date.now() + LOCK_RETRIES * LOCK_WAIT_MS;
-  for (let attempt = 0; attempt < LOCK_RETRIES && (attempt === 0 || Date.now() < deadline); attempt += 1) {
-    try {
-      fsImpl.mkdirSync(lockDir);
-      return () => { try { fsImpl.rmdirSync(lockDir); } catch { /* released is released */ } };
-    } catch {
-      try {
-        if (Date.now() - fsImpl.statSync(lockDir).mtimeMs > LOCK_STALE_MS) {
-          fsImpl.rmdirSync(lockDir);
-          continue;
-        }
-      } catch { /* the holder released between our mkdir and stat — retry immediately */ }
-      sleep(LOCK_WAIT_MS);
+  // real time, counting attempts alone would outlive a live holder's staleness window.
+  const deadline = Date.now() + retries * LOCK_WAIT_MS;
+  let step           = 'again';
+  let me        = { name: '' };
+  for (let attempt = 0; attempt < retries && (attempt === 0 || Date.now() < deadline); attempt += 1) {
+    if (step === 'queued') step = queuedStep(fsImpl, lockDir, me, options);
+    else {
+      me = { name: (options.entryName || newToken)() }; // a fresh entry name per attempt: an earlier one's name never comes back
+      step = attemptLock(fsImpl, lockDir, me, options);
     }
+    if (step === 'held') return ownedRelease(fsImpl, lockDir, me);
+    if (step === 'busy' || step === 'queued') sleep(LOCK_WAIT_MS);
   }
-  throw new Error(`Could not lock '${file}' after ${(LOCK_RETRIES * LOCK_WAIT_MS) / 1000}s — another writer holds '${file}.lock'. `
-    + 'If no other session is running, the lock is leftover from a crash younger than its stale timeout; retry shortly.');
+  if (step === 'queued' && giveUp(fsImpl, lockDir, me, options) === 'held') return ownedRelease(fsImpl, lockDir, me);
+  throw new Error(`Could not lock '${file}' after ${(retries * LOCK_WAIT_MS) / 1000}s — another writer holds '${lockDir}'. `
+    + `If no other DoFlow process is running, the lock is left from a crash and is reclaimed once it is ${LOCK_STALE_MS / 1000}s old; retry then.`
+    + strayEntries(fsImpl, lockDir));
+}
+
+/** Names a non-directory entry in the lock (no DoFlow runtime makes one; it blocks the lock until removed). */
+function strayEntries(fsImpl     , lockDir        )         {
+  let stray           = [];
+  try { stray = fsImpl.readdirSync(lockDir, { withFileTypes: true }).filter((d     ) => !d.isDirectory()).map((d     ) => d.name); } catch { return ''; }
+  if (stray.length === 0) return '';
+  return ` It holds ${stray.map((name) => `'${name}'`).join(', ')}, which no DoFlow runtime makes and which keeps it locked: `
+    + `remove ${stray.length > 1 ? 'them' : 'it'} once no DoFlow process is running.`;
 }
 
 /**
  * Takes the cross-process lock for one state file with a bounded wait.
- * Makes 1 try plus `retries` retries (default 3) separated by `waitMs` (default 5).
+ * Makes 1 try plus `retries` retries (default 3) separated by `waitMs` (default 5); each try is one
+ * tryAcquireLock, which waits at most 40 ms, and only on a process that reached the lock at the same moment.
  * Throws an Error('store-locked') after one try plus retries.
  * @param {Object} fsImpl
  * @param {string} file
@@ -166,7 +436,9 @@ function readTaskState(fsImpl     , file        )      {
   try {
     data = JSON.parse(fsImpl.readFileSync(file, 'utf8'));
   } catch (error     ) {
-    const failure      = new Error(`Failed to load state file '${file}': ${error.message}. The write path is atomic `
+    // The parser's own message quotes the first bytes of the file; a state file can hold text the
+    // reader should not echo, so only the error's class is named.
+    const failure      = new Error(`Failed to load state file '${file}': its content is not valid JSON (${error?.name || 'Error'}). The write path is atomic `
       + '(tmp + rename), so a torn file means outside interference; recover it from version control '
       + 'or move it aside and re-record the task\'s items.');
     failure.code = 'store-unreadable';
@@ -181,6 +453,27 @@ function readTaskState(fsImpl     , file        )      {
   return data;
 }
 
+/** How many times the folder is made again when it vanishes before the lock can be made in it. */
+const FOLDER_RACE_RETRIES = 5;
+
+/**
+ * Makes the state file's folder and takes the lock in it. The lock is a folder inside the state
+ * file's folder and is made without `recursive`, so a sweep that removes an empty namespace folder
+ * between the two steps makes the lock fail with ENOENT. The folder is then made again, a bounded
+ * number of times; the sweep never removes a folder that holds a lock, so a taken lock keeps it.
+ */
+function acquireInFolder(fsImpl     , file        , acquire                                           )             {
+  for (let tries = 0; ; tries += 1) {
+    fsImpl.mkdirSync(path.dirname(file), { recursive: true });
+    try {
+      return acquire(fsImpl, file);
+    } catch (error     ) {
+      if (error && error.code === 'ENOENT' && tries < FOLDER_RACE_RETRIES) continue;
+      throw error;
+    }
+  }
+}
+
 /**
  * The one durable-update path: lock, re-read, merge, bump revision, atomic write.
  *
@@ -194,6 +487,9 @@ function readTaskState(fsImpl     , file        )      {
  *   that must inspect or set aside what is there under the same exclusion as the write
  * @param {(fsImpl: Object, file: string) => () => void} [options.acquire] optional lock acquirer
  *   defaulting to acquireLock
+ * @param {number} [options.lostLockRetries=0] how many times to run the whole locked update again
+ *   when the lock was reclaimed from this writer before its rename (nothing was written); only for a
+ *   caller whose `prepare` and `build` have no effect outside the payload they return
  * @returns {string|null} the file path written, or null when build declines a write
  */
 function updateTaskState({
@@ -202,37 +498,51 @@ function updateTaskState({
   build,
   prepare,
   acquire = acquireLock,
+  lostLockRetries = 0,
 }   
               
                
                             
                        
                                                       
+                           
  )                {
-  fsImpl.mkdirSync(path.dirname(file), { recursive: true });
-  const release = acquire(fsImpl, file);
-  try {
-    if (prepare) prepare();
-    const disk = readTaskState(fsImpl, file);
-    const built = build(disk);
-    if (built === null) return null;
-    const payload = {
-      ...built,
-      version: SCHEMA_VERSION,
-      revision: (disk && Number.isInteger(disk.revision) ? disk.revision : 0) + 1,
-    };
-    const tmp = `${file}.${process.pid}.tmp`;
-    fsImpl.writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8');
-    fsImpl.renameSync(tmp, file);
-    return file;
-  } finally {
-    release();
+  for (let attempt = 0; ; attempt += 1) {
+    const release      = acquireInFolder(fsImpl, file, acquire);
+    try {
+      if (prepare) prepare();
+      const disk = readTaskState(fsImpl, file);
+      const built = build(disk);
+      if (built === null) return null;
+      const payload = {
+        ...built,
+        version: SCHEMA_VERSION,
+        revision: (disk && Number.isInteger(disk.revision) ? disk.revision : 0) + 1,
+      };
+      const tmp = `${file}.${process.pid}.tmp`;
+      fsImpl.writeFileSync(tmp, JSON.stringify(payload, null, 2), 'utf8');
+      // A writer paused past LOCK_STALE_MS may have been reclaimed; its read is then stale.
+      if (typeof release.held === 'function' && !release.held()) {
+        try { fsImpl.rmSync(tmp, { force: true }); } catch { /* stale-temp pruning removes it later */ }
+        if (attempt < lostLockRetries) continue;
+        const lost      = new Error(`Lost the lock on '${file}' to another writer before writing (this writer stalled past `
+          + `${LOCK_STALE_MS / 1000}s${lostLockRetries ? `, ${lostLockRetries + 1} times` : ''}). Nothing was written; run the command again.`);
+        lost.code = 'lock-lost';
+        throw lost;
+      }
+      fsImpl.renameSync(tmp, file);
+      return file;
+    } finally {
+      release();
+    }
   }
 }
 
 module.exports = {
   acquireLock,
   tryAcquireLock,
+  LOCK_STALE_MS,
+  LOCK_WAIT_MS,
   boundedAcquire,
   updateTaskState,
   readTaskState,