mandrel 2.3.0 → 2.5.0

package/.agents/scripts/lib/orchestration/ticket-lease.js

@@ -211,6 +211,16 @@ export async function describeLease(opts) {
  *                                     `reason: 'reclaimed'`.
  *   - Foreign claim + `steal:true` → reassign operator, `acquired: true`,
  *                                     `reason: 'stolen'`.
+ *   - Lost a write race            → a foreign login co-assigned between our
+ *                                     PATCH and the verify re-read; back the
+ *                                     operator out, `acquired: false`,
+ *                                     `owner: <foreign>`, `reason: 'lost-race'`.
+ *
+ * Every claiming write is verified: GitHub's assignee PATCH is not a
+ * compare-and-set, so two runs that both read the ticket unassigned will both
+ * write themselves. {@link claimAndVerify} re-reads after the write and refuses
+ * (fail-closed) when a foreign login is present, so the loser of a simultaneous
+ * claim never proceeds as though it holds the lease.
  *
  * @param {object} opts
  * @param {object} opts.provider              Ticketing provider.
@@ -225,7 +235,7 @@ export async function describeLease(opts) {
  *   acquired: boolean,
  *   owner: string,
  *   previousOwner: string|null,
- *   reason: 'unclaimed'|'already-held'|'reclaimed'|'stolen'|'held',
+ *   reason: 'unclaimed'|'already-held'|'reclaimed'|'stolen'|'held'|'lost-race',
  * }>}
  */
 export async function acquireLease(opts) {
@@ -240,13 +250,13 @@ export async function acquireLease(opts) {
 
   // Unclaimed → take it.
   if (owner === null) {
-    await provider.updateTicket(ticketId, { assignees: [operator] });
-    return {
-      acquired: true,
-      owner: operator,
+    return claimAndVerify({
+      provider,
+      ticketId,
+      operator,
       previousOwner: null,
       reason: 'unclaimed',
-    };
+    });
   }
 
   // Already ours → no write needed.
@@ -270,12 +280,70 @@ export async function acquireLease(opts) {
     };
   }
 
-  await provider.updateTicket(ticketId, { assignees: [operator] });
-  return {
-    acquired: true,
-    owner: operator,
+  return claimAndVerify({
+    provider,
+    ticketId,
+    operator,
     previousOwner: owner,
     reason: steal && live ? 'stolen' : 'reclaimed',
+  });
+}
+
+/**
+ * Write the operator to a ticket's assignees, then re-read to confirm the
+ * claim actually stuck before reporting success.
+ *
+ * The assignee write is not atomic — GitHub offers no compare-and-set on the
+ * assignees surface — so two runs that both observed the ticket unassigned (or
+ * a stale foreign claim) will both PATCH themselves in. Without a check the
+ * loser of that race returns `acquired: true` and marches into the worktree
+ * the winner is already building. The verify closes that window: it re-reads
+ * with `fresh: true` (bypassing any provider cache so it sees the other run's
+ * write, not our own), and if a foreign login is present it concedes — removes
+ * the operator from the assignee set so no phantom co-owner lingers, and
+ * returns `acquired: false` / `reason: 'lost-race'` so the fail-closed caller
+ * refuses. A clean read (assignees exactly `[operator]`) confirms the claim.
+ *
+ * It does not eliminate the race — two writes still happen — but it makes the
+ * outcome deterministic: exactly one operator survives as the sole assignee,
+ * and the other is told it lost.
+ *
+ * @param {object} args
+ * @param {object} args.provider              Ticketing provider.
+ * @param {number} args.ticketId              Ticket being claimed.
+ * @param {string} args.operator              Operator acquiring the lease.
+ * @param {string|null} args.previousOwner    Owner before this write (for the result).
+ * @param {string} args.reason                Success reason when the claim holds.
+ * @returns {Promise<{ acquired: boolean, owner: string, previousOwner: string|null, reason: string }>}
+ */
+async function claimAndVerify({
+  provider,
+  ticketId,
+  operator,
+  previousOwner,
+  reason,
+}) {
+  await provider.updateTicket(ticketId, { assignees: [operator] });
+
+  const after = await provider.getTicket(ticketId, { fresh: true });
+  const assignees = Array.isArray(after?.assignees) ? after.assignees : [];
+  const foreign = assignees.filter((login) => login !== operator);
+
+  if (foreign.length === 0) {
+    return { acquired: true, owner: operator, previousOwner, reason };
+  }
+
+  // A foreign login co-assigned after our write — we lost a simultaneous
+  // claim. Back ourselves out so the winner is the sole assignee, and report
+  // the loss so the fail-closed caller refuses rather than double-delivering.
+  await provider
+    .updateTicket(ticketId, { assignees: foreign })
+    .catch(() => undefined);
+  return {
+    acquired: false,
+    owner: foreign[0],
+    previousOwner,
+    reason: 'lost-race',
   };
 }
 

package/.agents/scripts/lib/orchestration/ticketing/transition.js

@@ -38,6 +38,7 @@ import {
   renderTransitionMessage,
 } from '../../notifications/notifier.js';
 import {
+  emitBlockRecoveredFriction,
   emitRuntimeFriction,
   RUNTIME_FRICTION_CATEGORIES,
 } from '../../observability/runtime-friction.js';
@@ -122,20 +123,41 @@ function validateTransitionInputs(newState) {
   return newState;
 }
 
+/**
+ * Active states a `agent::blocked` Story can recover into (Story #4622). A
+ * `blocked → {executing|ready}` transition is a self-resolved block; every
+ * other target (`done`, `closing`) is a real terminal outcome, not a
+ * recovery.
+ */
+const BLOCK_RECOVERY_TARGETS = [STATE_LABELS.EXECUTING, STATE_LABELS.READY];
+
 /**
  * Resolve the pre-transition ticket snapshot that drives the notify
  * payload and the provider's label-merge path. Honors the caller-supplied
  * `opts.ticketSnapshot` (Story #1795) when present; otherwise issues a
  * best-effort `getTicket` and returns `null` on transient failure.
  *
+ * The snapshot is loaded when a caller threads `notify` (its `fromState`
+ * feeds the notification payload) OR when `needFromState` is set — Story
+ * #4622's recovery detection needs the *prior* state, and `getTicket` after
+ * `updateTicket` would already read the new label. Bounding the extra read
+ * to recovery-target transitions keeps every other flip on the snapshot-free
+ * fast path.
+ *
  * @param {object} provider
  * @param {{ notify?: Function, ticketSnapshot?: object|null }} opts
  * @param {number} ticketId
+ * @param {boolean} [needFromState]
  * @returns {Promise<object|null>}
  */
-async function loadTicketSnapshot(provider, opts, ticketId) {
+async function loadTicketSnapshot(provider, opts, ticketId, needFromState) {
   if (opts.ticketSnapshot) return opts.ticketSnapshot;
-  if (!opts.notify || typeof provider.getTicket !== 'function') return null;
+  if (
+    (!opts.notify && !needFromState) ||
+    typeof provider.getTicket !== 'function'
+  ) {
+    return null;
+  }
   try {
     return await provider.getTicket(ticketId);
   } catch (err) {
@@ -301,26 +323,50 @@ function dispatchTransitionNotification(args) {
  * (see `frictionForTerminal`): the two would otherwise count one incident
  * twice.
  *
- * Best-effort and awaited: `emitRuntimeFriction` swallows its own failures
- * and resolves `false`, so this can neither throw nor block the transition.
+ * Story #4622 extends the hook to the inverse edge: a `blocked → active`
+ * transition emits a recovery marker so a transient block that self-resolved
+ * can be netted out of the retro's `story-blocked` recurrence total.
+ *
+ * Best-effort and awaited: the friction emitters swallow their own failures
+ * and resolve `false`, so this can neither throw nor block the transition.
  * It is awaited rather than fire-and-forget because CLI entry points exit
  * via `process.exit` as soon as `main` resolves (`cli-utils.runAsCli` with
  * `propagateExitCode`), which would discard a still-pending append.
  *
  * @param {number} ticketId
+ * @param {string|null} fromState  Prior state label, or null.
  * @param {string} newState
  * @param {{ config?: object }} opts
  * @returns {Promise<void>}
  */
-async function emitBlockedFriction(ticketId, newState, opts) {
-  if (newState !== STATE_LABELS.BLOCKED) return;
-  await emitRuntimeFriction({
-    storyId: ticketId,
-    category: RUNTIME_FRICTION_CATEGORIES.STORY_BLOCKED,
-    tool: 'transitionTicketState',
-    details: { toState: newState },
-    config: opts?.config,
-  });
+async function emitBlockedFriction(ticketId, fromState, newState, opts) {
+  if (newState === STATE_LABELS.BLOCKED) {
+    await emitRuntimeFriction({
+      storyId: ticketId,
+      category: RUNTIME_FRICTION_CATEGORIES.STORY_BLOCKED,
+      tool: 'transitionTicketState',
+      details: { toState: newState },
+      config: opts?.config,
+    });
+    return;
+  }
+  // Story #4622 — a transition *out* of `agent::blocked` into an active state
+  // is a recovery: the earlier block self-resolved. Emit its recovery marker
+  // so the retro composer can net the transient block out of the
+  // `story-blocked` recurrence total (swarm-os friction #581). Only a genuine
+  // block→active recovery qualifies; blocked→done/closing is a real
+  // terminal outcome, not a recovery, so it is left counted.
+  if (
+    fromState === STATE_LABELS.BLOCKED &&
+    BLOCK_RECOVERY_TARGETS.includes(newState)
+  ) {
+    await emitBlockRecoveredFriction({
+      storyId: ticketId,
+      fromState,
+      toState: newState,
+      config: opts?.config,
+    });
+  }
 }
 
 /**
@@ -382,7 +428,12 @@ export async function transitionTicketState(
   // snapshot is also forwarded to `provider.updateTicket` so the label
   // merge path skips its own `getTicket` call (the second of the two
   // round-trips this seam eliminates).
-  const ticketSnapshot = await loadTicketSnapshot(provider, opts, ticketId);
+  const ticketSnapshot = await loadTicketSnapshot(
+    provider,
+    opts,
+    ticketId,
+    BLOCK_RECOVERY_TARGETS.includes(newState),
+  );
   const fromState =
     ticketSnapshot?.labels?.find((l) => ALL_STATES.includes(l)) ?? null;
 
@@ -406,8 +457,9 @@ export async function transitionTicketState(
   });
 
   // Story #4578 — derive a friction signal from the block, at the point the
-  // runtime already knows. Best-effort; never blocks the transition.
-  await emitBlockedFriction(ticketId, newState, opts);
+  // runtime already knows. Story #4622 also emits the recovery marker on the
+  // inverse block→active transition. Best-effort; never blocks the transition.
+  await emitBlockedFriction(ticketId, fromState, newState, opts);
 
   // Story #2548 — mirror the new state onto the Projects v2 Status
   // column. Best-effort; never blocks the transition.

package/.agents/scripts/lib/single-story-sweep/sweep-lock.js

@@ -167,3 +167,76 @@ function buildAcquired(lockPath, ownerId, fsImpl) {
   }
   return { acquired: true, release, ownerId };
 }
+
+const DEFAULT_WAIT_MS = 8_000;
+const DEFAULT_POLL_MS = 150;
+
+/**
+ * Promise-based delay. Injectable so tests can drive the wait loop on a fake
+ * clock without a real timer.
+ *
+ * @param {number} ms
+ * @returns {Promise<void>}
+ */
+function defaultSleep(ms) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}
+
+/**
+ * Bounded-wait wrapper over {@link acquireSweepLock}.
+ *
+ * `acquireSweepLock` is single-attempt on purpose: a *skipped* sweep is
+ * harmless, so the sweep caller proceeds immediately on contention. The
+ * post-land tail is the opposite case — proceeding immediately IS the race
+ * two concurrent closes hit on a shared main checkout — so this wrapper
+ * polls the primitive with short backoff up to `waitMs` before giving up.
+ *
+ * It is still **never load-bearing**: on `waitMs` exhaustion it returns
+ * `{ acquired: false, reason: 'contended-after-wait' }` and the caller is
+ * expected to proceed anyway. The bounded wait is a best-effort collision
+ * damper, not a mutual-exclusion guarantee. A hard I/O error short-circuits
+ * the loop (spinning would just re-hit it).
+ *
+ * @param {object} opts
+ * @param {string} opts.lockPath
+ * @param {number} [opts.waitMs]     Max total time to wait for the lock.
+ * @param {number} [opts.pollMs]     Delay between acquire attempts.
+ * @param {number} [opts.timeoutMs]  Stale-lock expiry, forwarded to the
+ *                                   underlying acquire.
+ * @param {string} [opts.ownerId]
+ * @param {() => number} [opts.nowFn]
+ * @param {(ms: number) => Promise<void>} [opts.sleepFn]
+ * @param {object} [opts.fsImpl]
+ * @returns {Promise<{ acquired: true, release: () => void, ownerId: string }
+ *          | { acquired: false, reason: 'contended-after-wait' | 'error', detail?: string }>}
+ */
+export async function acquireLockWithWait({
+  lockPath,
+  waitMs = DEFAULT_WAIT_MS,
+  pollMs = DEFAULT_POLL_MS,
+  timeoutMs = DEFAULT_TIMEOUT_MS,
+  ownerId,
+  nowFn = Date.now,
+  sleepFn = defaultSleep,
+  fsImpl = fs,
+} = {}) {
+  const deadline = nowFn() + Math.max(0, waitMs);
+  for (;;) {
+    const res = acquireSweepLock({
+      lockPath,
+      timeoutMs,
+      ownerId,
+      nowFn,
+      fsImpl,
+    });
+    if (res.acquired) return res;
+    // A hard error will not resolve by retrying — surface it immediately.
+    if (res.reason === 'error') return res;
+    if (nowFn() >= deadline) {
+      return { acquired: false, reason: 'contended-after-wait' };
+    }
+    await sleepFn(Math.max(0, pollMs));
+  }
+}

package/.agents/scripts/lib/wave-runner/live-probe.js

@@ -25,11 +25,16 @@
  *     labels, **unioned with the ids the host says it has dispatched**
  *     (`--dispatched`). The label alone is not sufficient: the kernel's
  *     contract counts "executing / closing / dispatched-not-yet-labelled" as
- *     in-flight, and `single-story-init.js` flips `agent::executing` at step 6
- *     of 6 — *after* a 3–6 minute worktree install. For that whole window a
- *     dispatched Story still reads `agent::ready`, so a label-only derivation
- *     re-emits it in the next beat's `ready[]` and the host dispatches it a
- *     second time onto the same branch and worktree (Story #4601).
+ *     in-flight, and there is still a window between the host spawning a
+ *     sub-agent and that sub-agent's `single-story-init.js` publishing the
+ *     `agent::executing` label. Story #4620 shrank that window sharply — the
+ *     flip now lands before the multi-minute worktree install rather than
+ *     after it — but it is not zero (init still runs the lease acquire and a
+ *     branch fetch first), so a label-only derivation could still re-emit a
+ *     just-dispatched Story in the next beat's `ready[]` and dispatch it a
+ *     second time onto the same branch (Story #4601). `--dispatched` closes
+ *     the residual window; foreign runs are covered by the assignee lease
+ *     (see `deriveForeignHeld`).
  *   - **blocked** — the ids carrying `agent::blocked`. `classifyStory` has
  *     always returned this class; nothing consumed it, so a blocked Story was
  *     neither done, ready, nor in-flight and the beat reported a permanent
@@ -56,6 +61,10 @@ import {
 } from '../../resolve-stories.js';
 import { AGENT_LABELS } from '../label-constants.js';
 import { buildStoriesEnvelope } from '../orchestration/resolve-stories.js';
+import {
+  currentOwner,
+  normalizeOperatorHandle,
+} from '../orchestration/ticket-lease.js';
 import { classifyStory, storyIdOf } from './ready-set.js';
 
 /**
@@ -67,10 +76,13 @@ import { classifyStory, storyIdOf } from './ready-set.js';
  *   1. **Live labels.** `classifyStory` folds `agent::executing` and
  *      `agent::closing` into one `executing` class — both are in-flight and
  *      neither may be re-dispatched.
- *   2. **`dispatched`** — ids the host has spawned. This closes the init
- *      window: `single-story-init.js` flips `agent::executing` last, after a
- *      3–6 minute install, so between spawn and flip a dispatched Story reads
- *      `agent::ready` and a label-only derivation hands it back as ready.
+ *   2. **`dispatched`** — ids the host has spawned. This closes the residual
+ *      init window: between the host spawning a sub-agent and that agent's
+ *      `single-story-init.js` publishing `agent::executing`, a dispatched Story
+ *      still reads `agent::ready` and a label-only derivation hands it back as
+ *      ready. Story #4620 moved the flip ahead of the worktree install, so the
+ *      window is now short rather than minutes-long, but `--dispatched` still
+ *      covers it deterministically.
  *
  * `dispatched` is deliberately **not** the `--done`-style accounting probe
  * mode retired. Three properties keep it from becoming one:
@@ -129,6 +141,54 @@ function deriveBlockedIds(storyRecords) {
     .sort((a, b) => a - b);
 }
 
+/**
+ * Identify Stories claimed by a **different** operator's lease.
+ *
+ * The Story lease rides the ticket's assignees (`ticket-lease.js`): the sole
+ * assignee is the operator driving that Story's run. `single-story-init.js`
+ * takes the lease at init, but flips `agent::executing` only after a 3–6 minute
+ * worktree install — so for that whole window a Story another operator is
+ * actively delivering still reads `agent::ready` with no in-flight label. A
+ * label-only probe classifies it `ready` and hands it to this run, which then
+ * dispatches into a guaranteed init failure (the fail-closed lease refuses a
+ * foreign assignee) mid-batch. Reading the assignee lets the probe withhold it
+ * up front and report who holds it instead.
+ *
+ * Only Stories that would otherwise be `ready` are considered — a `done`,
+ * `blocked`, or already-`executing` Story is handled by its own class, and a
+ * self-held assignee is this run's own claim and never withholds.
+ *
+ * When `self` is unresolved (no `github.operatorHandle`), foreign cannot be
+ * told from self, so this returns empty and warns once: the probe is a
+ * read-only path that must not fail closed, and init's lease acquire remains
+ * the backstop.
+ *
+ * @param {Array<{id?: number, number?: number, labels?: string[], state?: string, assignees?: string[]}>} storyRecords
+ * @param {string|null|undefined} self  Resolved bare operator login for this run.
+ * @param {(msg: string) => void} [warn]
+ * @returns {Map<number, string>} Foreign-held Story id → holder login.
+ */
+function deriveForeignHeld(storyRecords, self, warn) {
+  const held = new Map();
+  if (!self) {
+    warn?.(
+      '[live-probe] github.operatorHandle is unset (or the shipped ' +
+        '@[USERNAME] placeholder), so a foreign lease cannot be told from ' +
+        'this run’s own claim — skipping assignee-based withholding. ' +
+        'Set your handle in .agentrc.local.json to de-conflict concurrent ' +
+        'runs at probe time; init’s lease still refuses a foreign claim.',
+    );
+    return held;
+  }
+  for (const rec of storyRecords) {
+    const id = storyIdOf(rec);
+    if (id === null || classifyStory(rec) !== 'ready') continue;
+    const owner = currentOwner(rec.assignees);
+    if (owner && owner !== self) held.set(id, owner);
+  }
+  return held;
+}
+
 /**
  * Resolve the provider + repo coordinates the probe reads through.
  *
@@ -138,13 +198,21 @@ function deriveBlockedIds(storyRecords) {
  *
  * @param {object} [deps]
  * @param {Function} [deps.resolveProvider] Injection seam for tests.
- * @returns {{ provider: object, owner: string|undefined, repo: string|undefined }}
+ * @returns {{ provider: object, owner: string|undefined, repo: string|undefined, self: string|null }}
  */
 export function createProbeContext({
   resolveProvider = resolveStoriesProvider,
 } = {}) {
   const { provider, config } = resolveProvider();
-  return { provider, owner: config?.github?.owner, repo: config?.github?.repo };
+  return {
+    provider,
+    owner: config?.github?.owner,
+    repo: config?.github?.repo,
+    // Bare login this run claims leases under. Normalised (leading `@` stripped,
+    // `@[USERNAME]` placeholder → null) so it compares against the bare assignee
+    // logins GitHub returns; `null` disables assignee-based withholding.
+    self: normalizeOperatorHandle(config?.github?.operatorHandle),
+  };
 }
 
 /**
@@ -165,6 +233,10 @@ export function createProbeContext({
  * @param {boolean} [args.native=true]   Read native `blocked_by` edges.
  * @param {number[]} [args.dispatched=[]] Ids the host has spawned but may not
  *   yet have observed labelled `agent::executing` (see `deriveInFlightIds`).
+ * @param {string|null} [args.self]     Resolved bare operator login for this
+ *   run, used to withhold Stories another operator's lease holds
+ *   (`deriveForeignHeld`). Absent/unresolved → assignee-based withholding is
+ *   skipped (the probe never fails closed).
  * @param {(msg: string) => void} [args.warn]
  * Each returned node carries its **live labels**. That is load-bearing, not
  * decoration: `selectReadySet` classifies from labels, so a node stripped of
@@ -177,7 +249,8 @@ export function createProbeContext({
  *   nodes: Array<{id: number, dependsOn: number[], files: string[], labels: string[]}>,
  *   doneIds: Set<number>,
  *   inFlight: number,
- *   blockedIds: number[]
+ *   blockedIds: number[],
+ *   foreignHeld: Array<{id: number, holder: string}>
  * }>}
  */
 export async function probeLiveState({
@@ -187,6 +260,7 @@ export async function probeLiveState({
   repo,
   native = true,
   dispatched = [],
+  self,
   warn,
 }) {
   const stories = await fetchStories(provider, ids);
@@ -209,6 +283,12 @@ export async function probeLiveState({
 
   const labelsById = new Map(stories.map((s) => [s.id, s.labels ?? []]));
   const inFlightIds = deriveInFlightIds(stories, dispatched);
+  // A Story another operator's lease holds occupies a (global) dispatch slot
+  // just like an in-flight one: fold it into the in-flight set so it is both
+  // withheld (via the projected label) and excluded from a false wedge, but
+  // never dispatched by this run.
+  const foreignHeld = deriveForeignHeld(stories, self, warn);
+  for (const id of foreignHeld.keys()) inFlightIds.add(id);
   return {
     nodes: envelope.dag.map((node) => ({
       ...node,
@@ -220,6 +300,7 @@ export async function probeLiveState({
     doneIds: new Set(envelope.done),
     inFlight: inFlightIds.size,
     blockedIds: deriveBlockedIds(stories),
+    foreignHeld: [...foreignHeld].map(([id, holder]) => ({ id, holder })),
   };
 }