input-kanban 0.0.8 → 0.0.10

package/src/orchestrator.js

@@ -4,7 +4,7 @@ import fsp from 'node:fs/promises';
 import path from 'node:path';
 import { promisify } from 'node:util';
 import {
-  DEFAULT_REPO, RUNS_DIR, ensureDir, nowIso, makeRunId, readJson,
+  DEFAULT_WORKSPACE, DEFAULT_REPO, RUNS_DIR, ensureDir, nowIso, makeRunId, readJson,
   writeJsonAtomic, fileInfo, readTextMaybe, extractFirstJsonObject, listRunDirs,
   pathForRun, roleDir, safeIdPart, RUNNER
 } from './utils.js';
@@ -17,6 +17,10 @@ const runner = defaultRunner;
 const VALID_SANDBOXES = new Set(['read-only', 'workspace-write', 'danger-full-access']);
 const MISSING_RUNNER_GRACE_MS = 10000;
 const MAX_DERIVED_LABEL_DISPLAY_WIDTH = 40;
+const RUN_STATE_LOCK_NAME = 'run_state.lock';
+const RUN_STATE_LOCK_STALE_MS = 30000;
+const RUN_STATE_LOCK_TIMEOUT_MS = 30000;
+const sleep = ms => new Promise(resolve => setTimeout(resolve, ms));
 
 function normalizeSandbox(value, fallback = 'workspace-write') {
   const sandbox = String(value || '').trim();
@@ -26,6 +30,120 @@ function normalizeSandbox(value, fallback = 'workspace-write') {
 
 function statePath(runDir) { return path.join(runDir, 'run_state.json'); }
 function planPath(runDir) { return path.join(runDir, 'plan.json'); }
+function lockPath(runDir) { return path.join(runDir, RUN_STATE_LOCK_NAME); }
+function workspacePathOf(state) { return path.resolve(state?.workspacePath || state?.repo || DEFAULT_WORKSPACE || DEFAULT_REPO); }
+function workspaceNameOf(state) { return state?.workspaceName || path.basename(workspacePathOf(state)) || workspacePathOf(state); }
+async function detectWorkspaceMetadata(workspacePath) {
+  const resolvedWorkspace = path.resolve(workspacePath || DEFAULT_WORKSPACE || DEFAULT_REPO || process.cwd());
+  const metadata = {
+    path: resolvedWorkspace,
+    name: path.basename(resolvedWorkspace) || resolvedWorkspace,
+    isGit: false
+  };
+  try {
+    const { stdout: rootStdout } = await execFileAsync('git', ['-C', resolvedWorkspace, 'rev-parse', '--show-toplevel'], { timeout: 5000 });
+    const gitRoot = rootStdout.trim();
+    if (gitRoot) {
+      metadata.isGit = true;
+      metadata.gitRoot = gitRoot;
+      try {
+        const { stdout: branchStdout } = await execFileAsync('git', ['-C', resolvedWorkspace, 'branch', '--show-current'], { timeout: 5000 });
+        metadata.branch = branchStdout.trim() || 'detached';
+      } catch {
+        metadata.branch = 'unknown';
+      }
+      try {
+        const { stdout: dirtyStdout } = await execFileAsync('git', ['-C', resolvedWorkspace, 'status', '--porcelain'], { timeout: 5000 });
+        metadata.dirty = dirtyStdout.trim().length > 0;
+      } catch {
+        metadata.dirty = null;
+      }
+    }
+  } catch {}
+  return metadata;
+}
+async function assertWorkspacePath(workspace) {
+  const resolvedWorkspace = path.resolve(workspace || DEFAULT_WORKSPACE || DEFAULT_REPO || process.cwd());
+  let stat;
+  try { stat = await fsp.stat(resolvedWorkspace); }
+  catch { throw userInputError(`workspace does not exist: ${resolvedWorkspace}`); }
+  if (!stat.isDirectory()) throw userInputError(`workspace is not a directory: ${resolvedWorkspace}`);
+  return resolvedWorkspace;
+}
+function normalizeWorkspaceState(state) {
+  const workspacePath = path.resolve(state?.workspacePath || state?.repo || DEFAULT_WORKSPACE || DEFAULT_REPO);
+  const workspaceName = state.workspaceName || path.basename(workspacePath) || workspacePath;
+  const git = state.git || state.workspace?.git || null;
+  state.workspacePath = workspacePath;
+  state.workspaceName = workspaceName;
+  state.repo = state.repo || workspacePath;
+  state.git = git;
+  state.workspace = state.workspace || {
+    path: workspacePath,
+    name: workspaceName,
+    git
+  };
+  if (!state.workspace.git && git) state.workspace.git = git;
+  return state;
+}
+
+async function isStaleRunLock(lockFile) {
+  const info = await fileInfo(lockFile);
+  if (!info.exists) return false;
+  const modifiedAt = Date.parse(info.mtime || '');
+  if (!Number.isFinite(modifiedAt)) return true;
+  if (Date.now() - modifiedAt < RUN_STATE_LOCK_STALE_MS) return false;
+  const lockData = await readJson(lockFile, null);
+  const pid = Number(lockData?.pid);
+  if (!Number.isFinite(pid) || pid <= 0) return true;
+  return !isPidAlive(pid);
+}
+
+export async function acquireRunStateLock(runId, { timeoutMs = RUN_STATE_LOCK_TIMEOUT_MS, staleMs = RUN_STATE_LOCK_STALE_MS } = {}) {
+  const runDir = pathForRun(runId);
+  await ensureDir(runDir);
+  const lockFile = lockPath(runDir);
+  const startedAt = Date.now();
+  let waitMs = 50;
+  while (true) {
+    try {
+      const handle = await fsp.open(lockFile, 'wx');
+      try {
+        await handle.writeFile(JSON.stringify({ runId, pid: process.pid, createdAt: nowIso() }, null, 2));
+        await handle.sync().catch(() => {});
+      } catch (error) {
+        await handle.close().catch(() => {});
+        await fsp.unlink(lockFile).catch(() => {});
+        throw error;
+      }
+      let released = false;
+      return async () => {
+        if (released) return;
+        released = true;
+        try { await handle.close(); } catch {}
+        await fsp.unlink(lockFile).catch(() => {});
+      };
+    } catch (error) {
+      if (error?.code !== 'EEXIST') throw error;
+      if (await isStaleRunLock(lockFile) && Date.now() - startedAt >= staleMs) {
+        await fsp.unlink(lockFile).catch(() => {});
+        continue;
+      }
+      if (Date.now() - startedAt >= timeoutMs) throw new Error(`run state lock busy: ${runId}`);
+      await sleep(waitMs);
+      waitMs = Math.min(waitMs * 1.5, 1000);
+    }
+  }
+}
+
+async function withRunStateLock(runId, fn, options = {}) {
+  const release = await acquireRunStateLock(runId, options);
+  try {
+    return await fn();
+  } finally {
+    await release();
+  }
+}
 
 function shouldMarkRunnerUnknown(target) {
   const missingSince = Date.parse(target.missingRunnerAt || '');
@@ -102,28 +220,29 @@ function deriveRunLabel(label, taskText) {
   return truncateDisplayWidth(cleaned, MAX_DERIVED_LABEL_DISPLAY_WIDTH) || 'task';
 }
 
-async function assertGitWorkTree(repo) {
-  const resolvedRepo = path.resolve(repo || DEFAULT_REPO);
-  let stat;
-  try { stat = await fsp.stat(resolvedRepo); }
-  catch { throw userInputError(`target repository does not exist: ${resolvedRepo}`); }
-  if (!stat.isDirectory()) throw userInputError(`target repository is not a directory: ${resolvedRepo}`);
-  try {
-    const { stdout } = await execFileAsync('git', ['-C', resolvedRepo, 'rev-parse', '--is-inside-work-tree'], { timeout: 5000 });
-    if (stdout.trim() === 'true') return resolvedRepo;
-  } catch {}
-  throw userInputError(`target repository is not a git work tree: ${resolvedRepo}`);
-}
 
-export async function createRun({ label = '', taskText = '', repo = DEFAULT_REPO, maxParallel = 3, workerSandbox = 'workspace-write' } = {}) {
-  const resolvedRepo = await assertGitWorkTree(repo);
+export async function createRun({ label = '', taskText = '', workspace = '', repo = DEFAULT_REPO, maxParallel = 3, workerSandbox = 'workspace-write' } = {}) {
+  const resolvedWorkspace = await assertWorkspacePath(workspace || repo || DEFAULT_WORKSPACE);
+  const workspaceMeta = await detectWorkspaceMetadata(resolvedWorkspace);
   const runLabel = deriveRunLabel(label, taskText);
   const runId = makeRunId(runLabel);
   const runDir = pathForRun(runId);
   await ensureDir(runDir);
   await fsp.writeFile(path.join(runDir, 'task.md'), taskText || '');
   const state = {
-    runId, label: runLabel, repo: resolvedRepo, maxParallel: Number(maxParallel) || 3, workerSandbox: normalizeSandbox(workerSandbox),
+    runId,
+    label: runLabel,
+    workspacePath: resolvedWorkspace,
+    workspaceName: workspaceMeta.name,
+    workspace: {
+      path: resolvedWorkspace,
+      name: workspaceMeta.name,
+      git: workspaceMeta
+    },
+    git: workspaceMeta,
+    repo: resolvedWorkspace,
+    maxParallel: Number(maxParallel) || 3,
+    workerSandbox: normalizeSandbox(workerSandbox),
     runner: RUNNER,
     status: 'created', createdAt: nowIso(), updatedAt: nowIso(),
     planner: { status: 'pending' }, batches: [], tasks: [], judge: { status: 'pending' }
@@ -132,23 +251,57 @@ export async function createRun({ label = '', taskText = '', repo = DEFAULT_REPO
   return state;
 }
 
-export async function listRuns({ includeArchived = false } = {}) {
+function normalizeWorkspaceFilter(workspace) {
+  const value = String(workspace || '').trim();
+  if (!value || value === 'all') return '';
+  return path.resolve(value);
+}
+
+function runMatchesWorkspace(run, workspaceFilter) {
+  if (!workspaceFilter) return true;
+  const runWorkspace = path.resolve(run?.workspacePath || run?.repo || '');
+  return runWorkspace === workspaceFilter;
+}
+
+export function isTerminalRunStatus(status) {
+  return ['judged', 'judge_failed', 'batch_blocked', 'plan_failed', 'plan_empty', 'stopped'].includes(status);
+}
+
+export function isFailureRunStatus(status) {
+  return ['judge_failed', 'batch_blocked', 'plan_failed', 'plan_empty', 'stopped'].includes(status);
+}
+
+export async function listRuns({ includeArchived = false, workspace = '' } = {}) {
+  const workspaceFilter = normalizeWorkspaceFilter(workspace);
   const dirs = await listRunDirs();
   const rows = [];
   for (const dir of dirs) {
-    const s = await loadAndRefreshRun(path.basename(dir), null, { light: true });
-    if (s && (includeArchived || !s.archived)) rows.push(summaryOfRun(s));
+    const s = await loadRun(path.basename(dir));
+    if (!s || (!includeArchived && s.archived)) continue;
+    const summary = summaryOfRun(s);
+    if (!runMatchesWorkspace(summary, workspaceFilter)) continue;
+    rows.push(summary);
   }
   return rows;
 }
 
 export async function loadRun(runId) {
   const state = await readJson(statePath(pathForRun(runId)), null);
-  if (state) ensureBatchShape(state);
+  if (state) {
+    normalizeWorkspaceState(state);
+    if (!state.git || typeof state.git.isGit !== 'boolean') {
+      const workspaceMeta = await detectWorkspaceMetadata(workspacePathOf(state));
+      state.git = workspaceMeta;
+      state.workspace = state.workspace || { path: workspacePathOf(state), name: state.workspaceName, git: workspaceMeta };
+      state.workspace.git = workspaceMeta;
+    }
+    ensureBatchShape(state);
+  }
   return state;
 }
 
 async function saveRun(state) {
+  normalizeWorkspaceState(state);
   ensureBatchShape(state);
   state.updatedAt = nowIso();
   await writeJsonAtomic(statePath(pathForRun(state.runId)), state);
@@ -238,38 +391,42 @@ Plan: ${path.join(pathForRun(state.runId), 'plan.json')}
 }
 
 export async function startPlanner(runId) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  if (state.archived) throw new Error('archived run cannot be planned');
-  if (state.status === 'stopped') throw new Error('stopped run cannot be planned; create a new run after modifications');
-  if (state.planner.status === 'running') throw new Error('planner already running');
-  if (hasStartedExecution(state)) throw new Error('planner retry is allowed only before any worker/judge starts');
-  const runDir = pathForRun(runId);
-  const previousPlanner = state.planner;
-  if (previousPlanner?.status && previousPlanner.status !== 'pending') await rotatePlannerAttempt(state, runDir);
-  state.batches = [];
-  state.tasks = [];
-  state.judge = { status: 'pending' };
-  const outDir = roleDir(runDir, 'planner');
-  await ensureDir(outDir);
-  await fsp.rm(planPath(runDir), { force: true });
-  const taskText = await fsp.readFile(path.join(runDir, 'task.md'), 'utf8');
-  const prompt = defaultPlannerPrompt(state, taskText);
-  const child = await runner.startCodexTask({ runId: state.runId, taskId: 'planner', batchId: 'planner', runStatePath: statePath(runDir), prompt, sandbox: 'read-only', cwd: state.repo, outDir });
-  state.status = 'planning';
-  state.planner = { status: 'running', pid: child.pid, startedAt: nowIso(), dir: outDir, attempt: (state.plannerAttempts?.length || 0) + 1 };
-  await saveRun(state);
-  child.onExit(async code => {
-    const s = await loadRun(runId); if (!s || s.status === 'stopped') return;
-    s.planner.exitCode = code; s.planner.endedAt = nowIso(); s.planner.status = code === 0 ? 'completed' : 'failed';
-    const planResult = await materializePlan(s);
-    if (s.planner.status !== 'completed') s.status = 'plan_failed';
-    else if (planResult.ok) s.status = 'planned';
-    else if (planResult.empty) s.status = 'plan_empty';
-    else s.status = 'plan_failed';
-    await saveRun(s);
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    if (state.archived) throw new Error('archived run cannot be planned');
+    if (state.status === 'stopped') throw new Error('stopped run cannot be planned; create a new run after modifications');
+    if (state.planner.status === 'running') throw new Error('planner already running');
+    if (hasStartedExecution(state)) throw new Error('planner retry is allowed only before any worker/judge starts');
+    const runDir = pathForRun(runId);
+    const previousPlanner = state.planner;
+    if (previousPlanner?.status && previousPlanner.status !== 'pending') await rotatePlannerAttempt(state, runDir);
+    state.batches = [];
+    state.tasks = [];
+    state.judge = { status: 'pending' };
+    const outDir = roleDir(runDir, 'planner');
+    await ensureDir(outDir);
+    await fsp.rm(planPath(runDir), { force: true });
+    const taskText = await fsp.readFile(path.join(runDir, 'task.md'), 'utf8');
+    const prompt = defaultPlannerPrompt(state, taskText);
+    const child = await runner.startCodexTask({ runId: state.runId, taskId: 'planner', batchId: 'planner', runStatePath: statePath(runDir), prompt, sandbox: 'read-only', cwd: workspacePathOf(state), outDir });
+    state.status = 'planning';
+    state.planner = { status: 'running', pid: child.pid, startedAt: nowIso(), dir: outDir, attempt: (state.plannerAttempts?.length || 0) + 1 };
+    await saveRun(state);
+    child.onExit(async code => {
+      await withRunStateLock(runId, async () => {
+        const s = await loadRun(runId); if (!s || s.status === 'stopped') return;
+        s.planner.exitCode = code; s.planner.endedAt = nowIso(); s.planner.status = code === 0 ? 'completed' : 'failed';
+        const planResult = await materializePlan(s);
+        if (s.planner.status !== 'completed') s.status = 'plan_failed';
+        else if (planResult.ok) s.status = 'planned';
+        else if (planResult.empty) s.status = 'plan_empty';
+        else s.status = 'plan_failed';
+        await saveRun(s);
+      });
+    });
+    return state;
   });
-  return state;
 }
 
 function normalizeExpectedArtifacts(value, runId, taskId) {
@@ -322,6 +479,43 @@ async function rotatePlannerAttempt(state, runDir) {
   }];
 }
 
+async function rotateWorkerAttempt(state, task) {
+  const runDir = pathForRun(state.runId);
+  const workerDir = roleDir(runDir, 'worker', task.id);
+  if (!fs.existsSync(workerDir)) return null;
+  const attemptsDir = path.join(runDir, 'worker_attempts', task.id);
+  await ensureDir(attemptsDir);
+  const attempt = Number(task.retryCount || 0) + 1;
+  const archivedDir = path.join(attemptsDir, `attempt-${String(attempt).padStart(2, '0')}`);
+  await fsp.rm(archivedDir, { recursive: true, force: true });
+  await fsp.rename(workerDir, archivedDir);
+  task.retryHistory = [...(task.retryHistory || []), {
+    attempt,
+    status: task.status,
+    exitCode: task.exitCode ?? null,
+    startedAt: task.startedAt,
+    endedAt: task.endedAt,
+    archivedDir,
+    archivedAt: nowIso(),
+    reason: task.retryReason || null
+  }];
+  task.retryCount = attempt;
+  task.retryReason = null;
+  delete task.pid;
+  delete task.exitCode;
+  delete task.startedAt;
+  delete task.endedAt;
+  delete task.stoppedAt;
+  delete task.missingRunnerAt;
+  delete task.manualCompletion;
+  delete task.originalStatus;
+  delete task.originalExitCode;
+  delete task.error;
+  delete task.tmux;
+  task.status = 'pending';
+  return archivedDir;
+}
+
 function normalizePlan(plan, defaultMaxParallel, defaultSandbox = 'workspace-write', runId = '') {
   if (Array.isArray(plan.batches)) {
     const batches = plan.batches.map((b, bi) => {
@@ -378,22 +572,24 @@ async function materializePlan(state) {
 }
 
 export async function dispatchRun(runId) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  if (state.archived) throw new Error('archived run cannot be dispatched');
-  if (state.status === 'stopped') throw new Error('stopped run cannot be dispatched; create a new run after modifications');
-  if (!state.tasks?.length) throw new Error('no tasks in plan');
-  if (state.status === 'batch_blocked') throw new Error('current batch is blocked by failed/unknown tasks');
-  if (allBatchesCompleted(state)) throw new Error('all batches completed; run final judge next');
-  state.status = 'running';
-  await scheduleMoreWorkers(state);
-  recomputeRunStatus(state);
-  await saveRun(state);
-  return state;
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    if (state.archived) throw new Error('archived run cannot be dispatched');
+    if (state.status === 'stopped') throw new Error('stopped run cannot be dispatched; create a new run after modifications');
+    if (!state.tasks?.length) throw new Error('no tasks in plan');
+    if (state.status === 'batch_blocked') throw new Error('current batch is blocked by failed/unknown tasks');
+    if (allBatchesCompleted(state)) throw new Error('all batches completed; run final judge next');
+    state.status = 'running';
+    await scheduleMoreWorkers(state);
+    recomputeRunStatus(state);
+    await saveRun(state);
+    return state;
+  });
 }
 
 function artifactPathForState(state, rel) {
-  return path.isAbsolute(rel) ? rel : path.join(state.repo, rel);
+  return path.isAbsolute(rel) ? rel : path.join(workspacePathOf(state), rel);
 }
 
 function workerArtifactInstructions(state, task) {
@@ -423,168 +619,252 @@ ORCHESTRATOR_BATCH_ID: ${task.batchId || 'batch-1'}
 
 ${task.prompt}${workerArtifactInstructions(state, task)}${upstreamArtifactInstructions(state, task)}
 `;
-  const child = await runner.startCodexTask({ runId: state.runId, taskId: task.id, batchId: task.batchId || 'batch-1', runStatePath: statePath(runDir), prompt: fullPrompt, sandbox: task.sandbox || state.workerSandbox || 'workspace-write', cwd: state.repo, outDir });
+  const child = await runner.startCodexTask({ runId: state.runId, taskId: task.id, batchId: task.batchId || 'batch-1', runStatePath: statePath(runDir), prompt: fullPrompt, sandbox: task.sandbox || state.workerSandbox || 'workspace-write', cwd: workspacePathOf(state), outDir });
   Object.assign(task, { status: 'running', pid: child.pid, startedAt: nowIso(), dir: outDir });
 }
 
 export async function stopRun(runId, { reason = 'stopped by user' } = {}) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  const stoppedAt = nowIso();
-  await runner.stopRun(runId);
-  const stoppedPids = new Set();
-  const stopTargetPid = target => {
-    const pid = Number(target?.pid);
-    if (Number.isFinite(pid) && pid > 0 && !stoppedPids.has(pid)) {
-      stoppedPids.add(pid);
-      stopPid(pid);
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    const stoppedAt = nowIso();
+    await runner.stopRun(runId);
+    const stoppedPids = new Set();
+    const stopTargetPid = target => {
+      const pid = Number(target?.pid);
+      if (Number.isFinite(pid) && pid > 0 && !stoppedPids.has(pid)) {
+        stoppedPids.add(pid);
+        stopPid(pid);
+      }
+    };
+    for (const roleState of [state.planner, state.judge]) {
+      if (roleState?.status === 'running') {
+        stopTargetPid(roleState);
+        Object.assign(roleState, { status: 'stopped', stoppedAt, endedAt: stoppedAt });
+      }
     }
-  };
-  for (const roleState of [state.planner, state.judge]) {
-    if (roleState?.status === 'running') {
-      stopTargetPid(roleState);
-      Object.assign(roleState, { status: 'stopped', stoppedAt, endedAt: stoppedAt });
+    for (const task of state.tasks || []) {
+      if (task.status === 'running') {
+        stopTargetPid(task);
+        Object.assign(task, { status: 'stopped', stoppedAt, endedAt: stoppedAt });
+      }
     }
-  }
-  for (const task of state.tasks || []) {
-    if (task.status === 'running') {
-      stopTargetPid(task);
-      Object.assign(task, { status: 'stopped', stoppedAt, endedAt: stoppedAt });
+    for (const batch of state.batches || []) {
+      for (const task of batch.tasks || []) if (task.status === 'running') stopTargetPid(task);
     }
-  }
-  for (const batch of state.batches || []) {
-    for (const task of batch.tasks || []) if (task.status === 'running') stopTargetPid(task);
-  }
-  for (const batch of state.batches || []) {
-    if ((batch.tasks || []).some(t => t.status === 'stopped')) batch.status = 'stopped';
-  }
-  state.status = 'stopped';
-  state.stopInfo = { reason, stoppedAt };
-  await saveRun(state);
-  return state;
+    for (const batch of state.batches || []) {
+      if ((batch.tasks || []).some(t => t.status === 'stopped')) batch.status = 'stopped';
+    }
+    state.status = 'stopped';
+    state.stopInfo = { reason, stoppedAt };
+    await saveRun(state);
+    return state;
+  });
 }
 
 export async function archiveRun(runId, { reason = 'archived by user' } = {}) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  if ((state.tasks || []).some(t => t.status === 'running') || state.planner?.status === 'running' || state.judge?.status === 'running') {
-    throw new Error('cannot archive a run while tasks are running; stop it first');
-  }
-  state.archived = true;
-  state.archivedAt = nowIso();
-  state.archiveInfo = { reason, archivedAt: state.archivedAt };
-  await saveRun(state);
-  return state;
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    if ((state.tasks || []).some(t => t.status === 'running') || state.planner?.status === 'running' || state.judge?.status === 'running') {
+      throw new Error('cannot archive a run while tasks are running; stop it first');
+    }
+    state.archived = true;
+    state.archivedAt = nowIso();
+    state.archiveInfo = { reason, archivedAt: state.archivedAt };
+    await saveRun(state);
+    return state;
+  });
 }
 
 export async function renameRun(runId, { label = '' } = {}) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  const nextLabel = String(label || '').trim();
-  if (!nextLabel) throw userInputError('run label cannot be empty');
-  state.label = nextLabel;
-  state.renamedAt = nowIso();
-  await saveRun(state);
-  return state;
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    const nextLabel = String(label || '').trim();
+    if (!nextLabel) throw userInputError('run label cannot be empty');
+    state.label = nextLabel;
+    state.renamedAt = nowIso();
+    await saveRun(state);
+    return state;
+  });
+}
+
+export async function retryRun(runId, { taskId = '', reason = 'manual retry', maxRetries = 1, auto = false } = {}) {
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    if (state.archived) throw new Error('archived run cannot be retried');
+    if (state.status === 'stopped') throw new Error('stopped run cannot be retried');
+    const selectedTaskId = String(taskId || '').trim();
+    let taskIds = [];
+    if (selectedTaskId) {
+      const task = (state.tasks || []).find(item => item.id === selectedTaskId);
+      if (!task) throw new Error(`task not found: ${selectedTaskId}`);
+      taskIds = [task.id];
+      if (!['failed', 'unknown'].includes(task.status)) throw new Error(`task is not retryable: ${selectedTaskId}`);
+    } else {
+      const batch = currentBlockedBatch(state);
+      if (!batch) throw new Error('no blocked batch to retry');
+      taskIds = (batch.tasks || []).filter(item => ['failed', 'unknown'].includes(item.status) && (!auto || canAutoRetryTask(item, maxRetries))).map(item => item.id);
+      if (!taskIds.length) throw new Error('no retryable tasks in blocked batch');
+    }
+    const result = await retryTasksInState(state, taskIds, { auto, maxRetries, reason });
+    if (!result.retried.length) throw new Error('no tasks were retried');
+    await saveRun(state);
+    return { ...state, retriedTaskIds: result.retried };
+  });
 }
 
 export async function markTaskCompleted(runId, taskId, { reason = 'manual success confirmed by user', resultText = '' } = {}) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  const task = (state.tasks || []).find(t => t.id === taskId);
-  if (!task) throw new Error(`task not found: ${taskId}`);
-  if (task.status === 'running') throw new Error('cannot mark a running task completed');
-  const runDir = pathForRun(runId);
-  const outDir = roleDir(runDir, 'worker', task.id);
-  await ensureDir(outDir);
-  if (task.status !== 'completed') {
-    const manualResult = String(resultText || '').trim();
-    if (manualResult) await fsp.writeFile(path.join(outDir, 'manual_result.md'), manualResult);
-    const override = {
-      type: 'manual_task_completed',
-      runId,
-      taskId,
-      originalStatus: task.originalStatus || task.status,
-      originalExitCode: task.originalExitCode ?? task.exitCode ?? null,
-      previousStatus: task.status,
-      previousExitCode: task.exitCode ?? null,
-      reason,
-      hasManualResult: !!manualResult,
-      manualResultFile: manualResult ? 'manual_result.md' : null,
-      manualResultPreview: manualResult ? manualResult.slice(0, 500) : '',
-      markedAt: nowIso()
-    };
-    await writeJsonAtomic(path.join(outDir, 'manual_completion.json'), override);
-    Object.assign(task, {
-      status: 'completed',
-      originalStatus: override.originalStatus,
-      originalExitCode: override.originalExitCode,
-      manualCompletion: override,
-      completedAt: override.markedAt
-    });
-    const batch = (state.batches || []).find(b => b.id === task.batchId);
-    if (batch) {
-      const batchTask = batch.tasks.find(t => t.id === task.id);
-      if (batchTask && batchTask !== task) Object.assign(batchTask, task);
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    const task = (state.tasks || []).find(t => t.id === taskId);
+    if (!task) throw new Error(`task not found: ${taskId}`);
+    if (task.status === 'running') throw new Error('cannot mark a running task completed');
+    const runDir = pathForRun(runId);
+    const outDir = roleDir(runDir, 'worker', task.id);
+    await ensureDir(outDir);
+    if (task.status !== 'completed') {
+      const manualResult = String(resultText || '').trim();
+      if (manualResult) await fsp.writeFile(path.join(outDir, 'manual_result.md'), manualResult);
+      const override = {
+        type: 'manual_task_completed',
+        runId,
+        taskId,
+        originalStatus: task.originalStatus || task.status,
+        originalExitCode: task.originalExitCode ?? task.exitCode ?? null,
+        previousStatus: task.status,
+        previousExitCode: task.exitCode ?? null,
+        reason,
+        hasManualResult: !!manualResult,
+        manualResultFile: manualResult ? 'manual_result.md' : null,
+        manualResultPreview: manualResult ? manualResult.slice(0, 500) : '',
+        markedAt: nowIso()
+      };
+      await writeJsonAtomic(path.join(outDir, 'manual_completion.json'), override);
+      Object.assign(task, {
+        status: 'completed',
+        originalStatus: override.originalStatus,
+        originalExitCode: override.originalExitCode,
+        manualCompletion: override,
+        completedAt: override.markedAt
+      });
+      const batch = (state.batches || []).find(b => b.id === task.batchId);
+      if (batch) {
+        const batchTask = batch.tasks.find(t => t.id === task.id);
+        if (batchTask && batchTask !== task) Object.assign(batchTask, task);
+      }
     }
-  }
-  recomputeRunStatus(state);
-  if (hasPendingRunnableBatch(state)) state.status = 'running';
-  await scheduleMoreWorkers(state);
-  recomputeRunStatus(state);
-  await saveRun(state);
-  return state;
+    recomputeRunStatus(state);
+    if (hasPendingRunnableBatch(state)) state.status = 'running';
+    await scheduleMoreWorkers(state);
+    recomputeRunStatus(state);
+    await saveRun(state);
+    return state;
+  });
 }
 
 export async function startJudge(runId) {
-  const state = await loadRun(runId);
-  if (!state) throw new Error(`run not found: ${runId}`);
-  recomputeRunStatus(state);
-  if (!allBatchesCompleted(state) && state.tasks?.length) throw new Error('final judge is allowed only after all batches completed');
-  const outDir = roleDir(pathForRun(runId), 'judge');
-  await ensureDir(outDir);
-  const judgeInputPath = path.join(outDir, 'judge_input.json');
-  const judgeInput = await buildJudgeInput(state);
-  await writeJsonAtomic(judgeInputPath, judgeInput);
-  const prompt = defaultJudgePrompt(state, judgeInputPath);
-  const child = await runner.startCodexTask({ runId: state.runId, taskId: 'judge', batchId: 'judge', runStatePath: statePath(pathForRun(runId)), prompt, sandbox: 'read-only', cwd: state.repo, outDir });
-  state.judge = { status: 'running', pid: child.pid, startedAt: nowIso(), dir: outDir };
-  state.status = 'judging';
-  await saveRun(state);
-  child.onExit(async code => {
-    const s = await loadRun(runId); if (!s || s.status === 'stopped') return;
-    s.judge.exitCode = code; s.judge.endedAt = nowIso(); s.judge.status = code === 0 ? 'completed' : 'failed';
-    const text = await readTextMaybe(path.join(outDir, 'last_message.md'), 1000000);
-    const verdict = extractFirstJsonObject(text);
-    if (verdict) { s.judge.verdict = verdict; await writeJsonAtomic(path.join(outDir, 'verdict.json'), verdict); }
-    s.status = s.judge.status === 'completed' ? 'judged' : 'judge_failed';
-    await saveRun(s);
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) throw new Error(`run not found: ${runId}`);
+    recomputeRunStatus(state);
+    if (!allBatchesCompleted(state) && state.tasks?.length) throw new Error('final judge is allowed only after all batches completed');
+    const outDir = roleDir(pathForRun(runId), 'judge');
+    await ensureDir(outDir);
+    const judgeInputPath = path.join(outDir, 'judge_input.json');
+    const judgeInput = await buildJudgeInput(state);
+    await writeJsonAtomic(judgeInputPath, judgeInput);
+    const prompt = defaultJudgePrompt(state, judgeInputPath);
+    const child = await runner.startCodexTask({ runId: state.runId, taskId: 'judge', batchId: 'judge', runStatePath: statePath(pathForRun(runId)), prompt, sandbox: 'read-only', cwd: workspacePathOf(state), outDir });
+    state.judge = { status: 'running', pid: child.pid, startedAt: nowIso(), dir: outDir };
+    state.status = 'judging';
+    await saveRun(state);
+    child.onExit(async code => {
+      await withRunStateLock(runId, async () => {
+        const s = await loadRun(runId); if (!s || s.status === 'stopped') return;
+        s.judge.exitCode = code; s.judge.endedAt = nowIso(); s.judge.status = code === 0 ? 'completed' : 'failed';
+        const text = await readTextMaybe(path.join(outDir, 'last_message.md'), 1000000);
+        const verdict = extractFirstJsonObject(text);
+        if (verdict) { s.judge.verdict = verdict; await writeJsonAtomic(path.join(outDir, 'verdict.json'), verdict); }
+        s.status = s.judge.status === 'completed' ? 'judged' : 'judge_failed';
+        await saveRun(s);
+      });
+    });
+    return state;
   });
-  return state;
 }
 
 export async function refreshRun(runId, appClient = null) {
   return await loadAndRefreshRun(runId, appClient, { light: false });
 }
 
-async function loadAndRefreshRun(runId, appClient = null, { light = false } = {}) {
-  const state = await loadRun(runId);
-  if (!state) return null;
-  state.runner = state.runner || RUNNER;
-  await refreshRole(state, state.planner, roleDir(pathForRun(runId), 'planner'));
-  await recoverCompletedPlanner(state);
-  for (const task of state.tasks || []) await refreshTask(state, task);
-  await refreshRole(state, state.judge, roleDir(pathForRun(runId), 'judge'));
-  await recoverCompletedJudge(state);
-  aggregateRunTmuxMetadata(state);
-  recomputeRunStatus(state);
-  await scheduleMoreWorkers(state);
-  recomputeRunStatus(state);
-  if (appClient && !light) await enrichFromAppServer(state, appClient).catch(e => { state.appServerError = e.message; });
-  await saveRun(state);
+export async function autoAdvanceRun(runId, { appClient = null, startCreated = false, maxRetries = 1, retryReason = 'auto retry from scheduler' } = {}) {
+  let state = await refreshRun(runId, appClient);
+  if (!state || state.archived || state.status === 'stopped') return state;
+  if (startCreated && state.status === 'created') {
+    try { state = await startPlanner(runId); }
+    catch (error) { if (!/planner already running/i.test(error.message || '')) throw error; }
+    state = await refreshRun(runId, appClient);
+  }
+  if (!state || state.archived || state.status === 'stopped') return state;
+  if (state.status === 'batch_blocked') {
+    try { state = await retryRun(runId, { reason: retryReason, maxRetries, auto: true }); }
+    catch (error) { state.autoAdvanceError = error.message || String(error); }
+    return await refreshRun(runId, appClient) || state;
+  }
+  if (state.status === 'planned') {
+    try { state = await dispatchRun(runId); }
+    catch (error) { if (!/all batches completed|current batch is blocked/i.test(error.message || '')) throw error; }
+    return await refreshRun(runId, appClient) || state;
+  }
+  if (state.status === 'batches_completed' && state.judge?.status !== 'running' && state.judge?.status !== 'completed') {
+    try { state = await startJudge(runId); }
+    catch (error) { if (!/final judge is allowed only after all batches completed/i.test(error.message || '')) throw error; }
+    return await refreshRun(runId, appClient) || state;
+  }
   return state;
 }
 
+export async function autoAdvanceActiveRuns({ appClient = null, startCreated = false, maxRetries = 1, retryReason = 'auto retry from scheduler' } = {}) {
+  const dirs = await listRunDirs();
+  const results = [];
+  for (const dir of dirs) {
+    const runId = path.basename(dir);
+    try {
+      const initial = await loadRun(runId);
+      if (!initial || initial.archived || ['judged', 'judge_failed', 'plan_failed', 'plan_empty', 'stopped'].includes(initial.status)) continue;
+      const state = await autoAdvanceRun(runId, { appClient, startCreated, maxRetries, retryReason });
+      if (state) results.push({ runId, status: state.status, ok: true });
+    } catch (error) {
+      results.push({ runId, ok: false, error: error.message || String(error) });
+    }
+  }
+  return results;
+}
+
+async function loadAndRefreshRun(runId, appClient = null, { light = false } = {}) {
+  return await withRunStateLock(runId, async () => {
+    const state = await loadRun(runId);
+    if (!state) return null;
+    state.runner = state.runner || RUNNER;
+    await refreshRole(state, state.planner, roleDir(pathForRun(runId), 'planner'));
+    await recoverCompletedPlanner(state);
+    for (const task of state.tasks || []) await refreshTask(state, task);
+    await refreshRole(state, state.judge, roleDir(pathForRun(runId), 'judge'));
+    await recoverCompletedJudge(state);
+    aggregateRunTmuxMetadata(state);
+    recomputeRunStatus(state);
+    await scheduleMoreWorkers(state);
+    recomputeRunStatus(state);
+    if (appClient && !light) await enrichFromAppServer(state, appClient).catch(e => { state.appServerError = e.message; });
+    await saveRun(state);
+    return state;
+  });
+}
+
 async function recoverCompletedPlanner(state) {
   if (state.planner?.status !== 'completed' || state.tasks?.length || state.batches?.length) return;
   const planResult = await materializePlan(state);
@@ -650,7 +930,7 @@ async function refreshTask(state, task) {
   await attachTmuxMetadata(task, dir);
   delete task.attentionHint;
   task.artifacts = [];
-  for (const rel of task.expectedArtifacts || []) task.artifacts.push({ path: rel, ...(await fileInfo(path.isAbsolute(rel) ? rel : path.join(state.repo, rel))) });
+  for (const rel of task.expectedArtifacts || []) task.artifacts.push({ path: rel, ...(await fileInfo(path.isAbsolute(rel) ? rel : path.join(workspacePathOf(state), rel))) });
   const batch = (state.batches || []).find(b => b.id === task.batchId);
   if (batch) {
     const bt = batch.tasks.find(t => t.id === task.id);
@@ -744,6 +1024,43 @@ function currentBatch(state) {
   return (state.batches || []).find(b => b.status !== 'completed');
 }
 
+function currentBlockedBatch(state) {
+  ensureBatchShape(state);
+  return (state.batches || []).find(b => b.status === 'failed' || b.status === 'blocked' || b.status === 'running' && (b.tasks || []).some(t => ['failed', 'unknown'].includes(t.status)));
+}
+
+function canAutoRetryTask(task, maxRetries = 1) {
+  if (!task) return false;
+  if (!['failed', 'unknown'].includes(task.status)) return false;
+  if (Number(task.retryCount || 0) >= Number(maxRetries || 1)) return false;
+  return true;
+}
+
+async function retryTasksInState(state, taskIds = null, { auto = false, maxRetries = 1, reason = 'retry' } = {}) {
+  ensureBatchShape(state);
+  const selectedTaskIds = taskIds ? new Set(taskIds.map(id => safeIdPart(id))) : null;
+  const tasksToRetry = (state.tasks || []).filter(task => {
+    if (selectedTaskIds && !selectedTaskIds.has(task.id)) return false;
+    if (!['failed', 'unknown'].includes(task.status)) return false;
+    if (auto && !canAutoRetryTask(task, maxRetries)) return false;
+    return true;
+  });
+  if (!tasksToRetry.length) return { retried: [], state };
+  for (const task of tasksToRetry) {
+    if (hasLiveRunnerProcess(state, task.id, task)) throw new Error(`task still has a live process: ${task.id}`);
+    const batch = (state.batches || []).find(item => item.id === task.batchId);
+    task.retryReason = reason;
+    await rotateWorkerAttempt(state, task);
+    const batchTask = batch?.tasks?.find(item => item.id === task.id);
+    if (batchTask && batchTask !== task) Object.assign(batchTask, task);
+  }
+  recomputeRunStatus(state);
+  if (hasPendingRunnableBatch(state)) state.status = 'running';
+  await scheduleMoreWorkers(state);
+  recomputeRunStatus(state);
+  return { retried: tasksToRetry.map(task => task.id), state };
+}
+
 async function scheduleMoreWorkers(state) {
   if (state.status !== 'running') return;
   const batch = currentBatch(state);
@@ -838,7 +1155,10 @@ async function buildJudgeInput(state) {
     run: {
       runId: state.runId,
       label: state.label,
-      repo: state.repo,
+      repo: workspacePathOf(state),
+      workspacePath: workspacePathOf(state),
+      workspaceName: state.workspaceName || path.basename(workspacePathOf(state)) || workspacePathOf(state),
+      git: state.git || state.workspace?.git || null,
       status: state.status,
       runner: state.runner || RUNNER,
       createdAt: state.createdAt,
@@ -882,7 +1202,7 @@ function ensureBatchShape(state) {
 }
 
 async function enrichFromAppServer(state, appClient) {
-  const res = await appClient.listThreads({ cwd: state.repo, limit: 100 });
+  const res = await appClient.listThreads({ cwd: workspacePathOf(state), limit: 100 });
   const threads = res?.data || [];
   const all = [{ id: 'planner', target: state.planner }, ...(state.tasks || []).map(t => ({ id: t.id, target: t })), { id: 'judge', target: state.judge }];
   for (const item of all) {
@@ -904,9 +1224,11 @@ function runDurationEndOfState(s) {
   return times.length ? new Date(Math.max(...times)).toISOString() : s.updatedAt;
 }
 
-function summaryOfRun(s) {
+export function summaryOfRun(s) {
   const tasks = s.tasks || [];
-  return { runId: s.runId, label: s.label, repo: s.repo, status: s.status, runner: s.runner || RUNNER, workerSandbox: s.workerSandbox || 'workspace-write', archived: !!s.archived, createdAt: s.createdAt, updatedAt: s.updatedAt, durationEnd: runDurationEndOfState(s), total: tasks.length, completed: tasks.filter(t => t.status === 'completed').length, failed: tasks.filter(t => ['failed','unknown'].includes(t.status)).length, running: tasks.filter(t => t.status === 'running').length, batches: (s.batches || []).map(b => ({ id: b.id, name: b.name, status: b.status, total: b.tasks?.length || 0, completed: (b.tasks || []).filter(t => t.status === 'completed').length })) };
+  const workspacePath = s.workspacePath || s.repo || '';
+  const git = s.git || s.workspace?.git || null;
+  return { runId: s.runId, label: s.label, repo: s.repo || workspacePath, workspacePath, workspaceName: s.workspaceName || path.basename(workspacePath || ''), git, status: s.status, runner: s.runner || RUNNER, workerSandbox: s.workerSandbox || 'workspace-write', archived: !!s.archived, createdAt: s.createdAt, updatedAt: s.updatedAt, durationEnd: runDurationEndOfState(s), total: tasks.length, completed: tasks.filter(t => t.status === 'completed').length, failed: tasks.filter(t => ['failed','unknown'].includes(t.status)).length, running: tasks.filter(t => t.status === 'running').length, batches: (s.batches || []).map(b => ({ id: b.id, name: b.name, status: b.status, total: b.tasks?.length || 0, completed: (b.tasks || []).filter(t => t.status === 'completed').length })) };
 }
 
 export async function readRunTaskText(runId) {