@ngockhoale/ukit 2.7.7 → 2.7.9

package/templates/.claude/ukit/runtime/async-lock.mjs

@@ -2,7 +2,10 @@
 // TASK-028: the one async lock utility for hook processes. Every hook lock used to
 // carry its own inline mkdir/owner/backoff copy (auto-allow-bash, verification-guard,
 // token-utils withFileLock, src/core/fileOps) with slightly divergent policies; this
-// module is the shared protocol-compatible implementation for hook-side callers:
+// module is now the single implementation for all of them — the two withFileLock
+// twins (token-utils.mjs and src/core/fileOps.js) delegate here, so the lock
+// protocol, pstart/recycled-pid validation, and claim+quarantine reclaim exist
+// exactly once and cannot drift between copies. For hook-side callers:
 //
 // - Acquisition is an atomic `mkdir` of `<target>.lock`; the owner is stamped into
 //   `owner` (pid + random token + ts) so stale reclaim can be validated instead of
@@ -26,6 +29,7 @@
 //   ownerless lock: a contender reads a missing owner as reclaimable after `staleMs`,
 //   so an unstamped live holder could have its lock deleted mid-critical-section.
 import crypto from 'node:crypto';
+import { spawnSync } from 'node:child_process';
 import fs from 'node:fs/promises';
 import path from 'node:path';
 
@@ -40,6 +44,10 @@ export const LOCK_MAX_SLICE_MS = 1_200;
 const BACKOFF_MIN_MS = 3;
 const BACKOFF_SPREAD_MS = 9;
 
+// Tolerance for comparing a stamped process start time against `ps`-reported start:
+// etimes has 1s granularity and the stamp is taken a few ms after the real start.
+const PID_START_TOLERANCE_MS = 3_000;
+
 /**
  * Derive the lock acquisition budget from a hook's wall-clock deadline: the time
  * remaining before that deadline, minus a reserve for release/cleanup, capped at the
@@ -86,6 +94,58 @@ function sleepWithAbort(ms, signal) {
   });
 }
 
+// Transient kernel-level fs failures worth a bounded retry (C44): observed on
+// external APFS volumes under metadata churn — `open()`/`mkdir()`/`rename()`
+// can return EAGAIN once and succeed on immediate retry. EEXIST is deliberately
+// absent: it is the lock-contend signal, not a flake.
+export const TRANSIENT_FS_CODES = new Set(['EAGAIN', 'EBUSY', 'EMFILE', 'ENFILE', 'ESTALE']);
+
+/**
+ * Retry an fs operation that can surface a transient kernel error
+ * (TRANSIENT_FS_CODES) with jittered exponential backoff. Non-transient codes
+ * throw immediately — EEXIST contention and EISDIR mapping are unaffected.
+ * The LAST error rethrows on retry exhaustion, deadline, or abort: a retry
+ * never extends the caller past `deadlineMs` and never swallows the failure.
+ * @param {() => Promise<*>} op
+ * @param {{
+ *   retries?: number,
+ *   baseDelayMs?: number,
+ *   maxDelayMs?: number,
+ *   deadlineMs?: number,
+ *   signal?: AbortSignal,
+ * }} [options]
+ * @returns {Promise<*>} op's resolved value
+ */
+export async function withTransientFsRetry(op, {
+  retries = 3,
+  baseDelayMs = 15,
+  maxDelayMs = 150,
+  deadlineMs = Infinity,
+  signal,
+} = {}) {
+  const startedAt = Date.now();
+  const maxAttempts = Number.isFinite(retries) && retries >= 0 ? Math.floor(retries) + 1 : 1;
+  let lastError = null;
+  for (let attempt = 0; attempt < maxAttempts; attempt += 1) {
+    if (signal?.aborted) {
+      throw lastError ?? signal.reason ?? new Error('withTransientFsRetry aborted');
+    }
+    try {
+      return await op();
+    } catch (error) {
+      lastError = error;
+      if (!TRANSIENT_FS_CODES.has(error?.code) || attempt + 1 >= maxAttempts) throw error;
+      const remaining = deadlineMs - (Date.now() - startedAt);
+      if (remaining <= 0) throw error;
+      const backoff = Math.min(baseDelayMs * (2 ** attempt), maxDelayMs);
+      const waitMs = Math.min(backoff * (0.5 + Math.random()), remaining);
+      const sleptFully = await sleepWithAbort(Math.max(0, waitMs), signal);
+      if (!sleptFully || Date.now() - startedAt >= deadlineMs) throw error;
+    }
+  }
+  throw lastError;
+}
+
 function isPidAlive(pid) {
   try {
     process.kill(pid, 0);
@@ -96,12 +156,66 @@ function isPidAlive(pid) {
   }
 }
 
-async function readLockOwner(lockPath) {
+// Wall-clock ms when `pid` started, or null when it cannot be determined. `lstart`
+// is POSIX-portable across macOS and Linux ps (BSD ps has no `etimes`); the ctime
+// shape is parsed manually so no Date.parse implementation quirk can misread it.
+// A failed probe is treated as "unknown", never as "dead".
+const LSTART_MONTHS = { Jan: 0, Feb: 1, Mar: 2, Apr: 3, May: 4, Jun: 5, Jul: 6, Aug: 7, Sep: 8, Oct: 9, Nov: 10, Dec: 11 };
+const LSTART_RE = /^[A-Za-z]{3}\s+([A-Za-z]{3})\s+(\d{1,2})\s+(\d{1,2}):(\d{2}):(\d{2})\s+(\d{4})$/;
+function processStartMs(pid) {
+  try {
+    const result = spawnSync('ps', ['-o', 'lstart=', '-p', String(pid)], {
+      encoding: 'utf8',
+      timeout: 2_000,
+    });
+    if (result.status !== 0) return null;
+    const match = String(result.stdout || '').replace(/\s+/g, ' ').trim().match(LSTART_RE);
+    if (!match) return null;
+    const month = LSTART_MONTHS[match[1]];
+    if (month === undefined) return null;
+    const ms = new Date(
+      Number(match[6]), month, Number(match[2]),
+      Number(match[3]), Number(match[4]), Number(match[5]),
+    ).getTime();
+    return Number.isFinite(ms) ? ms : null;
+  } catch {
+    return null;
+  }
+}
+
+// This process's own start time, stamped into every owner/reclaim record so a later
+// observer can tell a recycled pid (same number, different process) from the real
+// holder. `process.kill(pid, 0)` alone cannot: a reused pid probes alive forever and
+// used to pin every stale lock unreclaimable (RC-4 PID-reuse residual).
+const PROCESS_START_MS = Date.now() - Math.floor(process.uptime() * 1000);
+
+// Provably-gone check for a recorded owner/claim { pid, token, pstart? }:
+//  - dead pid probe → gone;
+//  - live pid + stamped pstart that does not match the running process's start → the
+//    pid was recycled after the holder died → gone;
+//  - live pid with no stamp (legacy record) or an undeterminable start → alive
+//    (conservative: a live holder's lock is never stolen).
+function recordedProcessGone(owner) {
+  if (!owner) return true;
+  if (!isPidAlive(owner.pid)) return true;
+  const stamped = Number(owner.pstart);
+  if (!Number.isFinite(stamped) || stamped <= 0) return false;
+  const actual = processStartMs(owner.pid);
+  if (actual === null) return false;
+  return Math.abs(actual - stamped) > PID_START_TOLERANCE_MS;
+}
+
+async function readLockOwner(lockPath, retry = withTransientFsRetry) {
   try {
-    const raw = JSON.parse(await fs.readFile(path.join(lockPath, 'owner'), 'utf8'));
+    const raw = JSON.parse(await retry(() => fs.readFile(path.join(lockPath, 'owner'), 'utf8')));
     const pid = Number(raw?.pid);
+    const pstart = Number(raw?.pstart);
     return Number.isInteger(pid) && pid > 0
-      ? { pid, token: typeof raw?.token === 'string' ? raw.token : null }
+      ? {
+        pid,
+        token: typeof raw?.token === 'string' ? raw.token : null,
+        pstart: Number.isFinite(pstart) && pstart > 0 ? pstart : null,
+      }
       : null;
   } catch {
     return null;
@@ -115,31 +229,37 @@ function sameLockOwner(left, right) {
   return left.pid === right.pid && left.token === right.token;
 }
 
-async function readReclaimOwner(lockPath) {
+async function readReclaimOwner(lockPath, retry = withTransientFsRetry) {
   try {
-    const raw = JSON.parse(await fs.readFile(path.join(lockPath, RECLAIM_FILE), 'utf8'));
+    const raw = JSON.parse(await retry(() => fs.readFile(path.join(lockPath, RECLAIM_FILE), 'utf8')));
     const pid = Number(raw?.pid);
+    const pstart = Number(raw?.pstart);
     return Number.isInteger(pid) && pid > 0
-      ? { pid, token: typeof raw?.token === 'string' ? raw.token : null }
+      ? {
+        pid,
+        token: typeof raw?.token === 'string' ? raw.token : null,
+        pstart: Number.isFinite(pstart) && pstart > 0 ? pstart : null,
+      }
       : null;
   } catch {
     return null;
   }
 }
 
+
 // Stale observers must claim the existing lock before removing it. The claim lives
 // inside the old generation, so a second observer cannot remove that generation while
 // the first observer is between its owner check and rm(). This closes the TOCTOU race
 // where a delayed stale observer deleted a freshly acquired successor lock.
-async function claimReclaim(lockPath, ownerToken, staleMs) {
+async function claimReclaim(lockPath, ownerToken, staleMs, retry = withTransientFsRetry) {
   const reclaimPath = path.join(lockPath, RECLAIM_FILE);
   try {
-    const handle = await fs.open(reclaimPath, 'wx');
+    const handle = await retry(() => fs.open(reclaimPath, 'wx'));
     try {
-      await handle.writeFile(
-        `${JSON.stringify({ pid: process.pid, token: ownerToken, ts: Date.now() })}\n`,
+      await retry(() => handle.writeFile(
+        `${JSON.stringify({ pid: process.pid, token: ownerToken, ts: Date.now(), pstart: PROCESS_START_MS })}\n`,
         'utf8',
-      );
+      ));
     } finally {
       await handle.close();
     }
@@ -150,11 +270,11 @@ async function claimReclaim(lockPath, ownerToken, staleMs) {
     // A killed reclaimer can leave its claim behind. Only clear an old claim whose
     // recorded process is gone; a live claim remains the exclusive reclaim authority.
     try {
-      const stat = await fs.stat(reclaimPath);
+      const stat = await retry(() => fs.stat(reclaimPath));
       if (Date.now() - stat.mtimeMs > staleMs) {
-        const claim = await readReclaimOwner(lockPath);
-        if (!claim || !isPidAlive(claim.pid)) {
-          await fs.rm(reclaimPath, { force: true });
+        const claim = await readReclaimOwner(lockPath, retry);
+        if (!claim || recordedProcessGone(claim)) {
+          await retry(() => fs.rm(reclaimPath, { force: true }));
         }
       }
     } catch {
@@ -164,38 +284,38 @@ async function claimReclaim(lockPath, ownerToken, staleMs) {
   }
 }
 
-async function releaseReclaimClaim(lockPath, ownerToken) {
+async function releaseReclaimClaim(lockPath, ownerToken, retry = withTransientFsRetry) {
   try {
-    const claim = await readReclaimOwner(lockPath);
+    const claim = await readReclaimOwner(lockPath, retry);
     if (claim?.token === ownerToken) {
-      await fs.rm(path.join(lockPath, RECLAIM_FILE), { force: true });
+      await retry(() => fs.rm(path.join(lockPath, RECLAIM_FILE), { force: true }));
     }
   } catch {
     // best-effort claim release; the stale-claim path handles an interrupted cleanup
   }
 }
 
-async function quarantineReclaim(lockPath, owner, ownerToken) {
+async function quarantineReclaim(lockPath, owner, ownerToken, retry = withTransientFsRetry) {
   const quarantinePath = `${lockPath}.reclaim-${ownerToken}`;
   try {
     // The claim serializes stale observers. Re-read before the atomic rename so an
     // observer never detaches a generation different from the one it validated.
-    const current = await readLockOwner(lockPath);
+    const current = await readLockOwner(lockPath, retry);
     if (!sameLockOwner(current, owner)) {
-      await releaseReclaimClaim(lockPath, ownerToken);
+      await releaseReclaimClaim(lockPath, ownerToken, retry);
       return false;
     }
     // Rename is atomic within the lock's parent directory: the old generation is
     // detached as one filesystem operation, so a successor created at lockPath can
     // never be reached by cleanup of this quarantined generation.
-    await fs.rename(lockPath, quarantinePath);
-    await fs.rm(quarantinePath, { recursive: true, force: true });
+    await retry(() => fs.rename(lockPath, quarantinePath));
+    await retry(() => fs.rm(quarantinePath, { recursive: true, force: true }));
     return true;
   } catch {
     // Never recursively remove lockPath here. If rename lost a race or failed, the
     // original path belongs to whoever currently holds it; a later poll can retry.
     try {
-      await fs.rm(quarantinePath, { recursive: true, force: true });
+      await retry(() => fs.rm(quarantinePath, { recursive: true, force: true }));
     } catch {
       // best-effort cleanup of only this acquisition's quarantine path
     }
@@ -207,6 +327,56 @@ async function quarantineReclaim(lockPath, owner, ownerToken) {
 // a pid probe cannot prove, so the module tracks them itself (same as withFileLock).
 const inProcessLockHolders = new Map();
 
+// --- TASK-004: dropped-update journal -------------------------------------------
+// Fail-closed means a skipped mutation is LOST — the journal makes that loss
+// explicit and auditable instead of a silent unlocked write. One JSONL record per
+// drop, appended under a journal-local async lock (the main lock is unavailable —
+// that is why this path runs). Bounded: a full journal applies backpressure and
+// rejects the record rather than rewriting history, same as the ledger journal.
+// Shared by both withFileLock twins (token-utils.mjs and src/core/fileOps.js) so
+// the drop contract can never diverge between the protocol copies.
+const LOCK_DROP_JOURNAL_MAX_RECORDS = 128;
+const LOCK_DROP_JOURNAL_BUDGET_MS = 400;
+
+export function lockDropJournalPath(filePath) {
+  return `${filePath}.lock-drops.jsonl`;
+}
+
+export async function journalDroppedLockMutation(filePath, { reason, waitedMs }) {
+  const journalPath = lockDropJournalPath(filePath);
+  const record = {
+    v: 1,
+    ts: Date.now(),
+    pid: process.pid,
+    file: path.basename(filePath),
+    reason,
+    waitedMs: Math.max(0, Math.round(waitedMs)),
+  };
+  try {
+    const outcome = await withAsyncLock(
+      journalPath,
+      { deadlineMs: LOCK_DROP_JOURNAL_BUDGET_MS },
+      async () => {
+        let existing = null;
+        try {
+          existing = await fs.readFile(journalPath, 'utf8');
+        } catch (error) {
+          if (error?.code !== 'ENOENT') return false;
+        }
+        const lines = existing === null
+          ? []
+          : existing.split('\n').filter((line) => line.trim());
+        if (lines.length >= LOCK_DROP_JOURNAL_MAX_RECORDS) return false;
+        await fs.appendFile(journalPath, `${JSON.stringify(record)}\n`, 'utf8');
+        return true;
+      },
+    );
+    return outcome?.ok === true && outcome.value === true;
+  } catch {
+    return false; // journaling is best-effort; never resurrect the mutation over it
+  }
+}
+
 /**
  * Serialize a mutation of a shared state file across hook processes and concurrent
  * async flows. Protocol-compatible with token-utils withFileLock (`<file>.lock`
@@ -239,20 +409,31 @@ export async function withAsyncLock(filePath, { signal, deadlineMs = LOCK_MAX_SL
   // True only when THIS acquisition holds a lock it can prove it owns: the atomic
   // mkdir succeeded AND the owner stamp is on disk. Release is verified against it.
   let owned = false;
+  // C44: every lock-protocol fs op retries transient kernel errors (EAGAIN et
+  // al.) inside the caller's remaining acquisition budget — a kernel flake must
+  // never escape as an untyped throw while budget remains, and a retry must
+  // never extend the wait past it.
+  const retry = (op) => withTransientFsRetry(op, {
+    deadlineMs: Math.max(0, budget - (Date.now() - startedAt)),
+    signal,
+  });
+  // Post-acquisition ops (release, claim cleanup) run after the budget is spent;
+  // they get the fixed cleanup reserve instead of the acquisition slice.
+  const releaseRetry = (op) => withTransientFsRetry(op, { deadlineMs: LOCK_RESERVE_MS });
 
   while (true) {
     try {
       // The lock parent must exist before the atomic acquire — a first-ever run in a
       // fresh project would otherwise fail mkdir with ENOENT.
-      await fs.mkdir(path.dirname(lockPath), { recursive: true });
-      await fs.mkdir(lockPath); // atomic acquire — EEXIST means another holder exists
+      await retry(() => fs.mkdir(path.dirname(lockPath), { recursive: true }));
+      await retry(() => fs.mkdir(lockPath)); // atomic acquire — EEXIST means another holder exists
       inProcessLockHolders.set(lockPath, ownerToken);
       try {
-        await fs.writeFile(
+        await retry(() => fs.writeFile(
           path.join(lockPath, 'owner'),
-          `${JSON.stringify({ pid: process.pid, token: ownerToken, ts: Date.now() })}\n`,
+          `${JSON.stringify({ pid: process.pid, token: ownerToken, ts: Date.now(), pstart: PROCESS_START_MS })}\n`,
           'utf8',
-        );
+        ));
       } catch {
         // Fail closed: an unstamped lock is not ours to enter. Running the callback
         // anyway would leave an ownerless directory, and every other process reads a
@@ -260,11 +441,11 @@ export async function withAsyncLock(filePath, { signal, deadlineMs = LOCK_MAX_SL
         // lock mid-critical-section and letting two mutations interleave. Undo the
         // acquire and report the typed busy outcome callers already handle.
         try {
-          const current = await readLockOwner(lockPath);
+          const current = await readLockOwner(lockPath, retry);
           // Never remove a directory some other holder has since stamped (only
           // possible if this one was reclaimed in the window above).
           if (!current || current.token === ownerToken) {
-            await fs.rm(lockPath, { recursive: true, force: true });
+            await retry(() => fs.rm(lockPath, { recursive: true, force: true }));
           }
         } catch {
           // best-effort undo; a leftover dir is unheld and reclaimed by the next waiter
@@ -281,20 +462,37 @@ export async function withAsyncLock(filePath, { signal, deadlineMs = LOCK_MAX_SL
     // Someone holds the lock. Reclaim it only after the stale threshold AND only when
     // the holder is provably gone — stealing a live holder reintroduces the exact
     // interleaved-write race this lock exists to prevent.
+    //
+    // RC-4: the effective stale threshold for a budgeted waiter is
+    // min(staleMs, its remaining budget) — a waiter that cannot outlast the full
+    // LOCK_STALE_MS still reclaims a genuinely orphaned lock instead of freezing the
+    // liveness breakers behind it. A zero/negative remaining budget keeps the full
+    // threshold (a no-wait caller never steals). A lock whose recorded owner is a
+    // LIVE process is never reclaimed early; "provably gone" includes a recycled pid
+    // (stamped pstart no longer matches the running process).
     let reclaimed = false;
     try {
-      const stat = await fs.stat(lockPath);
-      if (Date.now() - stat.mtimeMs > stale) {
-        const owner = await readLockOwner(lockPath);
+      const stat = await retry(() => fs.stat(lockPath));
+      const ageMs = Date.now() - stat.mtimeMs;
+      const remainingMs = budget - (Date.now() - startedAt);
+      const effectiveStaleMs = remainingMs > 0 ? Math.min(stale, remainingMs) : stale;
+      if (ageMs > effectiveStaleMs) {
+        const owner = await readLockOwner(lockPath, retry);
         const liveInProcess = inProcessLockHolders.has(lockPath);
-        const reclaimable = !owner || owner.pid === process.pid
-          ? !liveInProcess
-          : !isPidAlive(owner.pid);
-        if (reclaimable && await claimReclaim(lockPath, ownerToken, stale)) {
+        // An ownerless lock may be a holder mid-stamp — only the FULL stale
+        // threshold proves abandonment there. A stamped owner provably gone
+        // (dead pid, recycled pid, or a same-pid record with no live in-process
+        // holder) is reclaimable at the budgeted threshold.
+        const reclaimable = !owner
+          ? (!liveInProcess && ageMs > stale)
+          : owner.pid === process.pid
+            ? !liveInProcess
+            : recordedProcessGone(owner);
+        if (reclaimable && await claimReclaim(lockPath, ownerToken, stale, retry)) {
           // Detach and clean only the generation that was validated. The atomic rename
           // makes this safe even when another process acquires lockPath immediately
           // after the stale generation is removed.
-          reclaimed = await quarantineReclaim(lockPath, owner, ownerToken);
+          reclaimed = await quarantineReclaim(lockPath, owner, ownerToken, retry);
         }
       }
     } catch {
@@ -327,9 +525,9 @@ export async function withAsyncLock(filePath, { signal, deadlineMs = LOCK_MAX_SL
         // Remove the lock only if THIS acquisition still owns it: after a stale
         // reclaim another holder may already own the dir, and deleting it would
         // unlock their critical section for a third waiter.
-        const current = await readLockOwner(lockPath);
+        const current = await readLockOwner(lockPath, releaseRetry);
         if (current && current.token === ownerToken) {
-          await fs.rm(lockPath, { recursive: true, force: true });
+          await releaseRetry(() => fs.rm(lockPath, { recursive: true, force: true }));
         }
       } catch {
         // best-effort release; a leaked dir is reclaimed by the next waiter

package/templates/.claude/ukit/runtime/compact-threshold.mjs

@@ -1398,13 +1398,16 @@ async function runCli() {
       routeSummary: routeState?.routeSummary ?? null,
     }, sessionConfig);
 
-    if (nextState.phase === 'hard') {
+    // TASK-004: withFileLock is fail-closed — a contended pressure lock skips the
+    // mutation and resolves undefined (journaled). Skip the advisory notice too;
+    // the next prompt re-evaluates pressure from the last committed state.
+    if (nextState?.phase === 'hard') {
       process.stdout.write(
         `[ukit-skill-router] Context pressure: hard (est ${nextState.estimatedTotalTokens} / soft ${nextState.softThreshold} / hard ${nextState.hardThreshold}). `
           + 'FIRST THING IN YOUR REPLY, tell the user plainly — do not stay silent: "Context sắp đầy — hãy gõ /compact ngay bây giờ (run /compact now)". '
           + 'Then hold heavy work: no new investigations, subagents, or large edits until the user has compacted.\n',
       );
-    } else if (nextState.phase === 'soft') {
+    } else if (nextState?.phase === 'soft') {
       process.stdout.write(
         `[ukit-skill-router] Context pressure: soft (est ${nextState.estimatedTotalTokens} / soft ${nextState.softThreshold} / hard ${nextState.hardThreshold}). `
           + 'Finish the current step, then tell the user to run /compact at the next natural pause instead of continuing indefinitely.\n',