elanous 0.2.12 → 0.2.14

package/src/harness/harness-queue.ts

@@ -0,0 +1,494 @@
+import { spawn, spawnSync } from 'node:child_process';
+import { createHash, randomUUID } from 'node:crypto';
+import { closeSync, existsSync, mkdirSync, openSync, readFileSync, readlinkSync, renameSync, unlinkSync, writeFileSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { Database } from 'bun:sqlite';
+import { debug } from '../debug/log.js';
+import { effectiveInstanceRoot, prodInstanceRoot } from '../instance/resolve.js';
+import { decideSpawn } from '../loops/budget.js';
+import { listSelfDevRuns, processBirthId, selfDevRunsDir } from '../self-dev/run-store.js';
+import { getUserConfig } from '../user-config.js';
+import { hostLeaseCounts, leaseHasPendingPod } from '../pod-lease/host-lease.js';
+import { leaseKubectl, measurePoolLease } from '../task-orchestrator/surfaces/pod-lease.js';
+import { parsePodPool, podPoolHostLease, resolvePodPoolSpec } from '../task-orchestrator/surfaces/pod-pool.js';
+import { writeHarnessQueueReceipt } from './harness-queue-child.js';
+
+export type QueueSeat = 'OP' | 'TC' | 'MK' | 'UX';
+export type QueueItem = {
+  id: string; seat: QueueSeat; kind: 'say' | 'ask'; input: string; hold: boolean; heavy: boolean;
+  at: string; status: 'queued' | 'launching' | 'launched' | 'finished'; pid?: number; launchId?: string; idempotencyKey?: string;
+};
+export type QueuePool = { running: number; pending: number; reserved: number; limit: number };
+export type QueueTick = { outcome: 'launched' | 'waiting' | 'skipped'; item?: QueueItem; reason: string };
+export type QueueProcess = { pid: number; seat?: QueueSeat; launchId?: string };
+type ProcessProbe = {
+  run?: typeof spawnSync; platform?: NodeJS.Platform;
+  cwd?: (pid: number) => string;
+  seatTrees?: Partial<Record<QueueSeat, string[]>>;
+  runsDir?: string;
+  environ?: (pid: number) => string;
+  birthId?: (pid: number) => string | undefined;
+};
+const queueSeatNames = ['OP', 'TC', 'MK', 'UX'] as const;
+
+export function queueSeatForCwd(cwd: string, trees: Partial<Record<QueueSeat, string[]>> = {}): QueueSeat | undefined {
+  if (!cwd.startsWith('/')) return undefined;
+  const current = resolve(cwd);
+  const matches = queueSeatNames.filter((s) => trees[s]?.some((tree) => {
+    if (!tree.startsWith('/')) return false;
+    const prefix = resolve(tree);
+    return current === prefix || current.startsWith(`${prefix}/`);
+  }));
+  return matches.length === 1 ? matches[0] : undefined;
+}
+
+export interface HarnessQueueDeps {
+  root?: string;
+  configPath?: string;
+  cap?: (seat: QueueSeat) => number;
+  pool?: () => QueuePool;
+  launch?: (item: QueueItem, args: string[], root: string) => Promise<number>;
+  alive?: (pid: number) => boolean;
+  processes?: () => readonly QueueProcess[];
+  receipt?: (root: string, launchId: string) => 'started' | 'finished' | 'not-started' | null;
+  log?: (event: 'enqueued' | 'launched' | 'waiting' | 'skipped', data: Record<string, unknown>) => void;
+}
+
+export function harnessQueuePath(root = effectiveInstanceRoot()): string {
+  return join(root, 'harness', 'queue.json');
+}
+
+function observe(event: 'enqueued' | 'launched' | 'waiting' | 'skipped', data: Record<string, unknown>, deps: HarnessQueueDeps): void {
+  try { (deps.log ?? ((name, payload) => debug.log('harness.queue', name, payload)))(event, data); }
+  catch { /* Logging cannot affect dispatch. */ }
+}
+
+function read(path: string): QueueItem[] {
+  if (!existsSync(path)) return [];
+  const value: unknown = JSON.parse(readFileSync(path, 'utf8'));
+  if (!Array.isArray(value) || !value.every((row) => row && typeof row === 'object'
+    && typeof row.id === 'string' && /^(OP|TC|MK|UX)$/.test(row.seat)
+    && (row.kind === 'say' || row.kind === 'ask') && typeof row.input === 'string'
+    && typeof row.at === 'string' && typeof row.hold === 'boolean' && typeof row.heavy === 'boolean'
+    && ['queued', 'launching', 'launched', 'finished'].includes(row.status)
+    && (row.pid === undefined || Number.isSafeInteger(row.pid))
+    && (row.launchId === undefined || typeof row.launchId === 'string')
+    && (row.idempotencyKey === undefined || typeof row.idempotencyKey === 'string'))) {
+    throw new Error('harness queue: invalid queue file (no launch)');
+  }
+  return value as QueueItem[];
+}
+
+function save(path: string, items: QueueItem[]): void {
+  const temporary = `${path}.${randomUUID()}.tmp`;
+  try {
+    writeFileSync(temporary, `${JSON.stringify(items, null, 2)}\n`, { flag: 'wx', mode: 0o600 });
+    renameSync(temporary, path);
+  } catch (error) {
+    try { unlinkSync(temporary); } catch { /* write did not finish */ }
+    throw error;
+  }
+}
+
+// SQLite OS locks release on process death, including death while awaiting a launch.
+async function locked<T>(root: string, action: (path: string) => Promise<T>): Promise<T> {
+  const path = harnessQueuePath(root);
+  mkdirSync(join(root, 'harness'), { recursive: true, mode: 0o700 });
+  const db = new Database(`${path}.mutex.sqlite`, { create: true, strict: true });
+  try {
+    const deadline = Date.now() + 30_000;
+    for (;;) {
+      try { db.exec('BEGIN IMMEDIATE'); break; }
+      catch (error) {
+        if ((error as { code?: string }).code !== 'SQLITE_BUSY' || Date.now() >= deadline) throw error;
+        await Bun.sleep(25);
+      }
+    }
+    try { return await action(path); }
+    finally { db.exec('ROLLBACK'); }
+  } finally { db.close(); }
+}
+
+function seat(value: string): QueueSeat {
+  if (!/^(OP|TC|MK|UX)$/.test(value)) throw new Error(`harness queue: unknown seat ${value}`);
+  return value as QueueSeat;
+}
+
+export async function addHarnessQueue(input: { seat: string; say?: string; ask?: string; hold?: boolean; heavy?: boolean; idempotencyKey?: string }, deps: HarnessQueueDeps = {}): Promise<QueueItem> {
+  const assigned = seat(input.seat);
+  if ((input.say === undefined) === (input.ask === undefined)) throw new Error('harness queue add: --say 또는 --ask 중 하나만 필요');
+  const kind = input.ask === undefined ? 'say' : 'ask';
+  const text = kind === 'say' ? input.say! : input.ask!;
+  if (!text.trim()) throw new Error('harness queue add: 빈 입력');
+  const value = kind === 'ask' ? resolve(text) : text;
+  if (kind === 'ask' && !existsSync(value)) throw new Error(`harness queue add: goal file not found: ${value}`);
+  const root = deps.root ?? effectiveInstanceRoot();
+  return locked(root, async (path) => {
+    const rows = read(path);
+    if (input.idempotencyKey !== undefined) {
+      if (!input.idempotencyKey.trim()) throw new Error('harness queue add: empty idempotency key');
+      const existing = rows.find((row) => row.idempotencyKey === input.idempotencyKey
+        && (row.status === 'queued' || row.status === 'launching' || row.status === 'launched')
+        && queueRowOutcome(row, root) !== 'retryable');
+      if (existing) {
+        if (existing.seat !== assigned || existing.kind !== kind || existing.input !== value) throw new Error('harness queue: idempotency key collision');
+        return existing;
+      }
+    }
+    const item: QueueItem = { id: `hq-${randomUUID()}`, seat: assigned, kind, input: value,
+      hold: input.hold === true, heavy: input.heavy === true, at: new Date().toISOString(), status: 'queued',
+      ...(input.idempotencyKey === undefined ? {} : { idempotencyKey: input.idempotencyKey }) };
+    save(path, [...rows, item]);
+    observe('enqueued', { id: item.id, seat: item.seat, kind }, deps);
+    return item;
+  });
+}
+
+export function listHarnessQueue(deps: HarnessQueueDeps = {}): QueueItem[] {
+  return read(harnessQueuePath(deps.root ?? effectiveInstanceRoot()));
+}
+
+/** A missing row is not proof of failure unless the queue left a cancellation marker. */
+export function harnessQueueOutcome(id: string, deps: HarnessQueueDeps = {}): 'pending' | 'succeeded' | 'retryable' | 'unknown' {
+  const root = deps.root ?? effectiveInstanceRoot();
+  const row = listHarnessQueue(deps).find((item) => item.id === id);
+  if (!row) {
+    const marker = join(root, 'harness', `${id}.outcome`);
+    if (!existsSync(marker)) return 'unknown';
+    const value = readFileSync(marker, 'utf8');
+    return value === 'succeeded' || value === 'retryable' ? value : 'unknown';
+  }
+  return queueRowOutcome(row, root);
+}
+
+const keyMarkerPath = (root: string, key: string) => join(root, 'harness', 'by-key', `${createHash('sha256').update(key).digest('hex')}.id`);
+
+function recordQueueOutcome(root: string, id: string, outcome: 'succeeded' | 'retryable', key?: string): void {
+  const path = join(root, 'harness', `${id}.outcome`);
+  if (!existsSync(path)) writeFileSync(path, outcome, { flag: 'wx', mode: 0o600 });
+  // The row is about to leave the queue; keep «which id last held this key» so a caller that crashed before
+  // recording the id can still find the outcome (LOOP-LIVE1 · seat loop recovery).
+  if (key) {
+    mkdirSync(join(root, 'harness', 'by-key'), { recursive: true });
+    writeFileSync(keyMarkerPath(root, key), id, { mode: 0o600 });
+  }
+}
+
+/** Last queue id that settled under this idempotency key, or undefined when none was recorded. */
+export function harnessQueueIdForKey(key: string, deps: HarnessQueueDeps = {}): string | undefined {
+  try {
+    const id = readFileSync(keyMarkerPath(deps.root ?? effectiveInstanceRoot(), key), 'utf8').trim();
+    return /^hq-[0-9a-f-]{36}$/i.test(id) ? id : undefined;
+  } catch { return undefined; }
+}
+
+function queueRowOutcome(row: QueueItem, root: string): 'pending' | 'succeeded' | 'retryable' | 'unknown' {
+  if (row.status === 'queued') return 'pending';
+  if (row.status === 'finished') {
+    const marker = join(root, 'harness', `${row.id}.outcome`);
+    if (existsSync(marker)) {
+      const value = readFileSync(marker, 'utf8');
+      if (value === 'succeeded' || value === 'retryable') return value;
+    }
+  }
+  if (!row.launchId) return 'unknown';
+  const receiptPath = harnessQueueReceiptPath(root, row.launchId);
+  if (!existsSync(receiptPath)) return 'pending';
+  const record: unknown = JSON.parse(readFileSync(receiptPath, 'utf8'));
+  if (!record || typeof record !== 'object') return 'unknown';
+  const { state, exitCode } = record as { state?: unknown; exitCode?: unknown };
+  if (state === 'not-started') return 'retryable';
+  if (state === 'finished') return exitCode === 0 ? 'succeeded' : typeof exitCode === 'number' ? 'retryable' : 'unknown';
+  return state === 'started' ? 'pending' : 'unknown';
+}
+
+export async function removeHarnessQueue(id: string, deps: HarnessQueueDeps = {}): Promise<boolean> {
+  const root = deps.root ?? effectiveInstanceRoot();
+  return locked(root, async (path) => {
+    const rows = read(path);
+    const item = rows.find((row) => row.id === id);
+    if (!item) return false;
+    if (item.status !== 'queued' && item.status !== 'finished') {
+      if ((item.status !== 'launched' && item.status !== 'launching') || !item.launchId) return false;
+      const state = (deps.receipt ?? receipt)(root, item.launchId);
+      if (state !== 'finished' && state !== 'not-started'
+        || (deps.processes ?? readHarnessQueueProcesses)().some((row) => row.launchId === item.launchId)) return false;
+    }
+    const outcome = item.status === 'queued' ? 'retryable' : queueRowOutcome(item, root);
+    if (item.idempotencyKey && (outcome === 'retryable' || outcome === 'succeeded')) {
+      recordQueueOutcome(root, id, outcome, item.idempotencyKey);
+    }
+    save(path, rows.filter((row) => row.id !== id));
+    observe('skipped', { id, seat: item.seat, reason: 'removed' }, deps);
+    return true;
+  });
+}
+
+export function queueLaunchArgs(item: QueueItem): string[] {
+  return ['harness', item.kind, item.input, '--substrate', 'pod',
+    ...(item.hold ? ['--no-auto-merge'] : []), ...(item.heavy ? ['--pod-memory', 'high'] : [])];
+}
+
+export function readHarnessQueuePool(): QueuePool {
+  const config = getUserConfig();
+  const spec = resolvePodPoolSpec(undefined, process.env, () => config.harness?.podPool ?? config.pod?.pool);
+  const current = spec ? null : leaseKubectl(['config', 'current-context']);
+  const context = spec ?? (current?.status === 0 ? current.stdout.trim() : '');
+  if (!context) throw new Error('pod lease status: no pool/context');
+  const members = parsePodPool(context);
+  const observed = measurePoolLease(members);
+  if (observed.members.some((member) => member.running === null || member.pending === null)) throw new Error('pod lease status: incomplete running/pending measurement');
+  const pendingJobs = observed.members.flatMap((member) => member.pendingJobs ?? []);
+  const records = podPoolHostLease(members).live();
+  return {
+    running: observed.members.reduce((sum, member) => sum + member.running!, 0),
+    pending: observed.members.reduce((sum, member) => sum + member.pending!, 0),
+    reserved: hostLeaseCounts(records).reserved + records.filter((row) => row.stage === 'job' && !leaseHasPendingPod(row, pendingJobs)).length,
+    limit: members.reduce((sum, member) => sum + member.capacity, 0),
+  };
+}
+
+function alive(pid: number): boolean {
+  try { process.kill(pid, 0); return true; } catch { return false; }
+}
+
+/** Host inventory includes harness authoring before it has a run ledger. Unattributed processes count for every seat. */
+export function readHarnessQueueProcesses(probe: ProcessProbe = {}): readonly QueueProcess[] {
+  const run = probe.run ?? spawnSync;
+  const platform = probe.platform ?? process.platform;
+  const result = run('ps', ['-eo', 'pid=,ppid=,args='], { encoding: 'utf8', timeout: 5_000, maxBuffer: 16 * 1024 * 1024 });
+  if (result.error || result.status !== 0 || typeof result.stdout !== 'string') throw new Error('harness process inventory unavailable');
+  const candidates: { pid: number; ppid: number; command: string }[] = [];
+  for (const line of result.stdout.split('\n')) {
+    const match = /^\s*(\d+)\s+(\d+)\s+(.+)$/.exec(line);
+    if (!match) {
+      if (/(?:elanous\.mjs|harness-queue-child\.ts)/.test(line)) throw new Error('invalid harness process inventory row');
+      continue;
+    }
+    const command = match[3]!;
+    if (!/^(?:\S*\/)?(?:bun|node)(?:\s|$)/.test(command)
+      || !/(?:elanous\.mjs|harness-queue-child\.ts)/.test(command)
+      || !/(?:\bharness\s+(?:say|ask)\b|\bdev\s+.*(?:--say|--ask|--file|--implement)\b|\bself\s+(?:implement|orchestrate)\b|harness-queue-child\.ts)/.test(command)) continue;
+    const pid = Number(match[1]), ppid = Number(match[2]);
+    if (!Number.isSafeInteger(pid) || pid <= 0 || !Number.isSafeInteger(ppid) || ppid < 0) throw new Error('invalid harness pid');
+    candidates.push({ pid, ppid, command });
+  }
+  const byPid = new Map(candidates.map((row) => [row.pid, row]));
+  const children = new Map<number, typeof candidates>();
+  for (const candidate of candidates) {
+    const siblings = children.get(candidate.ppid) ?? [];
+    siblings.push(candidate);
+    children.set(candidate.ppid, siblings);
+  }
+  const trees = probe.seatTrees ?? getUserConfig().loops?.orchestrator?.seatTrees;
+  const ledgerSeats = new Map<number, QueueSeat>();
+  const ambiguousPids = new Set<number>();
+  const candidatePids = new Set(candidates.map(({ pid }) => pid));
+  const births = new Map<number, string | undefined>();
+  for (const record of listSelfDevRuns(probe.runsDir ?? selfDevRunsDir())) {
+    const pid = record.pid;
+    if (!pid || !candidatePids.has(pid) || !record.pidStart || !queueSeatNames.some((name) => name === record.seat)) continue;
+    if (!births.has(pid)) {
+      try { births.set(pid, (probe.birthId ?? ((id: number) => processBirthId(id, platform)))(pid)); }
+      catch { births.set(pid, undefined); }
+    }
+    if (births.get(pid) !== record.pidStart || ambiguousPids.has(pid)) continue;
+    const prior = ledgerSeats.get(pid);
+    if (prior && prior !== record.seat) { ledgerSeats.delete(pid); ambiguousPids.add(pid); }
+    else ledgerSeats.set(pid, record.seat!);
+  }
+  const rows: QueueProcess[] = [];
+  // Attribute a run after grouping: the orchestrator may switch working trees while its launcher stays in HQ.
+  for (const root of candidates.filter((row) => !byPid.has(row.ppid))) {
+    const members: { process: typeof root; depth: number }[] = [];
+    const visit = (process: typeof root, depth: number): void => {
+      members.push({ process, depth });
+      for (const child of children.get(process.pid) ?? []) visit(child, depth + 1);
+    };
+    visit(root, 0);
+    const attributed: { seat: QueueSeat; depth: number; source: 'env' | 'flag' | 'ledger' | 'cwd'; priority: number }[] = [];
+    const worktrees: { seat: QueueSeat; depth: number }[] = [];
+    let launchId: string | undefined;
+    for (const { process: { pid, command }, depth } of members) {
+      let envSeat: string | undefined;
+      if (platform === 'linux') {
+        try {
+          const env = (probe.environ ?? ((id: number) => readFileSync(`/proc/${id}/environ`, 'utf8')))(pid).split('\0');
+          envSeat = env.find((entry) => entry.startsWith('ELANOUS_HARNESS_SEAT='))?.slice('ELANOUS_HARNESS_SEAT='.length);
+          launchId ??= env.find((entry) => entry.startsWith('ELANOUS_HARNESS_QUEUE_LAUNCH='))?.slice('ELANOUS_HARNESS_QUEUE_LAUNCH='.length);
+        } catch { /* Unknown attribution is charged to all seats. */ }
+      }
+      const flag = /--seat[=\s]+(OP|TC|MK|UX)(?:\s|$)/.exec(command)?.[1] as QueueSeat | undefined;
+      const envAssigned = envSeat && /^(OP|TC|MK|UX)$/.test(envSeat) ? envSeat as QueueSeat : undefined;
+      if (envAssigned) attributed.push({ seat: envAssigned, depth, source: 'env', priority: 0 });
+      if (flag) attributed.push({ seat: flag, depth, source: 'flag', priority: 1 });
+      const fromLedger = ledgerSeats.get(pid);
+      if (fromLedger) attributed.push({ seat: fromLedger, depth, source: 'ledger', priority: 2 });
+      if (!trees || !queueSeatNames.some((s) => trees[s]?.length)) continue;
+      try {
+        let cwd: string;
+        if (probe.cwd) cwd = probe.cwd(pid);
+        else if (platform === 'linux') cwd = readlinkSync(`/proc/${pid}/cwd`);
+        else {
+          const lsof = run('lsof', ['-a', '-p', String(pid), '-d', 'cwd', '-Fn'], { encoding: 'utf8', timeout: 5_000 });
+          if (lsof.error || lsof.status !== 0 || typeof lsof.stdout !== 'string') throw new Error('cwd unavailable');
+          cwd = lsof.stdout.split('\n').find((line) => line.startsWith('n/'))?.slice(1) ?? '';
+        }
+        if (!cwd.startsWith('/')) continue;
+        const found = queueSeatForCwd(cwd, trees);
+        if (found) worktrees.push({ seat: found, depth });
+      } catch { /* Unknown attribution is charged to all seats. */ }
+    }
+    const deepest = Math.max(...members.map((entry) => entry.depth));
+    const inferred = new Set(worktrees.filter((entry) => entry.depth === deepest).map((entry) => entry.seat));
+    const complete = worktrees.filter((entry) => entry.depth === deepest).length
+      === members.filter((entry) => entry.depth === deepest).length;
+    if (complete && inferred.size === 1) attributed.push({ seat: [...inferred][0]!, depth: deepest, source: 'cwd', priority: 3 });
+    const priority = Math.min(...attributed.map((entry) => entry.priority));
+    const eligible = attributed.filter((entry) => entry.priority === priority);
+    const depth = Math.max(...eligible.map((entry) => entry.depth));
+    const top = eligible.filter((entry) => entry.depth === depth);
+    const seats = new Set(top.map((entry) => entry.seat));
+    const assigned = seats.size === 1 ? top[0] : undefined;
+    try { debug.log('harness.queue', 'attributed', { pid: root.pid, seat: assigned?.seat ?? null, source: assigned?.source ?? 'all' }); }
+    catch { /* Observation does not block inventory. */ }
+    rows.push({ pid: root.pid, ...(assigned ? { seat: assigned.seat } : {}), ...(launchId ? { launchId } : {}) });
+  }
+  return rows;
+}
+
+export function harnessQueueReceiptPath(root: string, launchId: string): string {
+  if (!/^hq-[0-9a-f-]{36}$/.test(launchId)) throw new Error('invalid queue launch id');
+  return join(root, 'harness', `${launchId}.receipt.json`);
+}
+
+function receipt(root: string, launchId: string): 'started' | 'finished' | 'not-started' | null {
+  try {
+    const record = JSON.parse(readFileSync(harnessQueueReceiptPath(root, launchId), 'utf8')) as { state?: string };
+    return record.state === 'started' || record.state === 'finished' || record.state === 'not-started' ? record.state : null;
+  } catch (error) {
+    if ((error as NodeJS.ErrnoException).code === 'ENOENT') return null;
+    throw error;
+  }
+}
+
+async function launch(item: QueueItem, args: string[], root: string): Promise<number> {
+  const repo = resolve(import.meta.dir, '../..');
+  const logPath = join(root, 'harness', `${item.id}.log`);
+  const fd = openSync(logPath, 'a', 0o600);
+  try {
+    const child = spawn(process.execPath, [join(repo, 'src', 'harness', 'harness-queue-child.ts'),
+      harnessQueueReceiptPath(root, item.launchId!), join(repo, 'bin', 'elanous.mjs'),
+      ...(resolve(root) === prodInstanceRoot() ? [] : [`--test=${root}`]), ...args], {
+      cwd: repo, detached: true, stdio: ['ignore', fd, fd],
+      env: { ...process.env, ELANOUS_STATE_DIR: root, ELANOUS_HARNESS_SEAT: item.seat,
+        ELANOUS_HARNESS_QUEUE_LAUNCH: item.launchId! },
+    });
+    const pid = await new Promise<number>((done, reject) => {
+      child.once('spawn', () => done(child.pid!));
+      child.once('error', (error) => {
+        try { writeHarnessQueueReceipt(harnessQueueReceiptPath(root, item.launchId!), 'not-started'); }
+        catch { /* A failed receipt write leaves the reservation safely indeterminate. */ }
+        reject(error);
+      });
+    });
+    child.unref();
+    return pid;
+  } finally { closeSync(fd); }
+}
+
+/** Only a wrapper receipt proving no active child can clear an uncertain reservation. */
+export async function reconcileHarnessQueue(id: string, deps: HarnessQueueDeps = {}): Promise<'released' | 'running' | 'unknown'> {
+  const root = deps.root ?? effectiveInstanceRoot();
+  return locked(root, async (path) => {
+    const rows = read(path);
+    const item = rows.find((row) => row.id === id);
+    if (!item || !['launching', 'launched'].includes(item.status) || !item.launchId) return 'unknown';
+    const state = (deps.receipt ?? receipt)(root, item.launchId);
+    const processes = (deps.processes ?? readHarnessQueueProcesses)();
+    if (processes.some((row) => row.launchId === item.launchId)
+      || (item.pid !== undefined && (deps.alive ?? alive)(item.pid) && state !== 'finished' && state !== 'not-started')) return 'running';
+    if (state !== 'finished' && state !== 'not-started') return 'unknown';
+    // A finished receipt proves the child exited; a reused wrapper PID alone is not its identity.
+    // Keep completed launches in the ledger, but never charge their seat again.
+    const outcome = state === 'not-started' ? 'retryable' : queueRowOutcome(item, root);
+    if (item.idempotencyKey && (outcome === 'succeeded' || outcome === 'retryable')) recordQueueOutcome(root, id, outcome, item.idempotencyKey);
+    save(path, state === 'finished' && item.status === 'launched'
+      ? rows.map((row) => row.id === id ? { ...row, status: 'finished' } : row)
+      : rows.filter((row) => row.id !== id));
+    observe('skipped', { id, seat: item.seat, reason: `reconciled ${state}`, launchId: item.launchId }, deps);
+    return 'released';
+  });
+}
+
+export async function tickHarnessQueue(deps: HarnessQueueDeps = {}): Promise<QueueTick> {
+  const root = deps.root ?? effectiveInstanceRoot();
+  return locked(root, async (path) => {
+    const items = read(path);
+    const item = items.find((row) => row.status === 'queued');
+    if (!item) { observe('skipped', { reason: 'empty' }, deps); return { outcome: 'skipped', reason: 'empty' }; }
+    const config = deps.configPath === undefined ? getUserConfig() : getUserConfig(deps.configPath);
+    const cap = (deps.cap ?? ((s: QueueSeat) => config.harness?.queue?.seatCap?.[s] ?? 8))(item.seat);
+    if (!Number.isSafeInteger(cap) || cap < 1) throw new Error(`harness queue: invalid seat cap for ${item.seat}`);
+    const caps = config.loops?.orchestrator?.seatCaps;
+    const gate = config.loops?.orchestrator?.releaseGate;
+    const attributed: Record<QueueSeat, number> = { OP: 0, TC: 0, MK: 0, UX: 0 };
+    let unattributed = 0;
+    const wait = (reason: string, active: number): QueueTick => {
+      observe('waiting', { id: item.id, seat: item.seat, reason, active, cap, attributed, unattributed }, deps);
+      return { outcome: 'waiting', item, reason };
+    };
+    let processes: readonly QueueProcess[];
+    try { processes = (deps.processes ?? readHarnessQueueProcesses)(); }
+    catch (error) {
+      const budget = decideSpawn({ seat: item.seat, running: null, caps, gate });
+      return wait(`${budget.reason}: harness process inventory unavailable: ${String(error)}`, 0);
+    }
+    for (const row of processes) {
+      if (row.seat) attributed[row.seat]++;
+      else unattributed++;
+    }
+    const current = items.flatMap((row) => {
+      if ((row.status !== 'launched' && row.status !== 'launching') || !row.launchId) return [row];
+      const state = (deps.receipt ?? receipt)(root, row.launchId);
+      if (state !== 'finished' && state !== 'not-started'
+        || processes.some((process) => process.launchId === row.launchId)) return [row];
+      if (state === 'finished' && row.status === 'launched') return [{ ...row, status: 'finished' as const }];
+      const outcome = state === 'not-started' ? 'retryable' : queueRowOutcome(row, root);
+      if (row.idempotencyKey && (outcome === 'succeeded' || outcome === 'retryable')) recordQueueOutcome(root, row.id, outcome, row.idempotencyKey);
+      return [];
+    });
+    if (current.length !== items.length || current.some((row, index) => row !== items[index])) save(path, current);
+    const charged = processes.filter((row) => !row.seat || row.seat === item.seat);
+    const launchIds = new Set(charged.map((row) => row.launchId));
+    const active = new Set(charged.map((row) => row.launchId ?? `pid:${row.pid}`)).size + current.filter((row) => row.seat === item.seat &&
+      (row.status === 'launching' || row.status === 'launched')
+      && (!row.launchId || !launchIds.has(row.launchId))).length;
+    if (active >= cap) return wait(`seat ${item.seat}: ${active}/${cap}`, active);
+    let pool: QueuePool;
+    try { pool = (deps.pool ?? readHarnessQueuePool)(); }
+    catch (error) { return wait(`pod lease status unavailable: ${String(error)}`, active); }
+    if (![pool.running, pool.pending, pool.reserved, pool.limit].every((n) => Number.isSafeInteger(n) && n >= 0) || pool.limit < 1) {
+      return wait('pod lease status incomplete', active);
+    }
+    const queueLaunching = current.filter((row) => row.status === 'launching' || row.status === 'launched').length;
+    if (pool.running + pool.pending + pool.reserved + queueLaunching >= pool.limit) {
+      return wait(`pool: ${pool.running}+${pool.pending}+${pool.reserved}+${queueLaunching}/${pool.limit}`, active);
+    }
+    const budget = decideSpawn({ seat: item.seat, running: active, caps, gate });
+    if (!budget.allow) return wait(`${budget.reason}: seat ${item.seat}: ${active}/${budget.cap}`, active);
+    const launching: QueueItem = { ...item, status: 'launching', launchId: `hq-${randomUUID()}` };
+    save(path, current.map((row) => row.id === item.id ? launching : row));
+    try {
+      const pid = await (deps.launch ?? launch)(launching, queueLaunchArgs(launching), root);
+      if (!Number.isSafeInteger(pid) || pid < 1) throw new Error('launcher returned no pid');
+      const launched: QueueItem = { ...launching, pid, status: 'launched' };
+      save(path, current.map((row) => row.id === item.id ? launched : row));
+      observe('launched', { id: item.id, seat: item.seat, pid, kind: item.kind, launchId: launched.launchId }, deps);
+      return { outcome: 'launched', item: launched, reason: 'spawned' };
+    } catch (error) {
+      observe('skipped', { id: item.id, seat: item.seat, reason: `launch uncertain: ${String(error)}`, launchId: launching.launchId }, deps);
+      throw error;
+    }
+  });
+}

package/src/harness/helper-scan.ts

@@ -0,0 +1,139 @@
+import { execFileSync } from 'node:child_process';
+import { debug } from '../debug/log.js';
+
+export type StoppedPrCategory = 'review-budget' | 'no-progress' | 'environment-gap' | 'memo-not-reflected' | 'review-oscillation' | 'goal-revision' | 'report-deficit' | 'unknown';
+export type StoppedPrClassification = {
+  category: StoppedPrCategory;
+  signals: string[];
+  firstAction: string;
+  blockers: string[];
+  blockerCount: number;
+  hasPublicExportLeak: boolean;
+};
+
+function section(body: string, title: string): string {
+  const heading = `## ${title}`;
+  const lines = body.split(/\r?\n/);
+  const start = lines.findIndex((line) => line.trim() === heading);
+  if (start < 0) return '';
+  const end = lines.findIndex((line, i) => i > start && /^##\s+/.test(line));
+  return lines.slice(start + 1, end < 0 ? undefined : end).join('\n');
+}
+
+function field(text: string, key: string): string {
+  const line = text.split(/\r?\n/).find((entry) => new RegExp(`^\\s*(?:[-*]\\s*)?(?:\\*\\*)?${key}(?:\\*\\*)?\\s*[:：]\\s*`, 'i').test(entry));
+  return line?.replace(new RegExp(`^\\s*(?:[-*]\\s*)?(?:\\*\\*)?${key}(?:\\*\\*)?\\s*[:：]\\s*`, 'i'), '').trim().replace(/^`|`$/g, '') ?? '';
+}
+
+export function classifyStoppedPr({ body, comments }: { body: string; comments: readonly (string | { body: string })[] }): StoppedPrClassification {
+  const stop = section(body, '중단 사유');
+  const classification = section(body, '중단 원인 분류');
+  const gate = section(body, 'Gate');
+  const reason = field(stop, 'reason');
+  const verdict = field(stop, 'verdict');
+  const label = field(classification, 'classification');
+  const basis = field(classification, 'classificationBasis');
+  const commentText = comments.map((comment) => typeof comment === 'string' ? comment : comment.body);
+  const signals = [reason, verdict, label, basis].filter(Boolean);
+  const allBlockers = gate.split(/\r?\n/).filter((line) => /^\s*(?:✗\s+|\[test\]\s+FAIL\b)/i.test(line));
+  const blockers = allBlockers.slice(0, 5);
+  const publicExportLeak = allBlockers.find((line) => /^\s*✗\s+public-export-leak\b/i.test(line));
+  if (publicExportLeak && !blockers.includes(publicExportLeak)) signals.push(publicExportLeak);
+  const failedTests = allBlockers.filter((line) => /^\s*\[test\]\s+FAIL\b.*[—–-](?:\s*\d+\s+(?:pass|fail|skip|errors?)\b\s*[,|])*\s*[1-9]\d*\s+errors?\b/i.test(line));
+  const failedPwaTest = failedTests.find((line) => /\bapps\/pwa\/[^\s]*\.test\.tsx?\b/i.test(line));
+  if (failedPwaTest) signals.push(failedPwaTest);
+  const hostRetest = /(?:^|;\s*)host=[^;\n]+;\s*cause=(?:missing (?:node_modules|dependency|environment variable)|호스트 의존성 없음)[^;\n]*;\s*retest=PASS bun test (\S+\.test\.tsx?)(?:\s*(?:;|$))/i.exec(basis);
+  // A code error in the Gate outranks any written basis: a basis string alone never turns a code failure into an environment gap.
+  const codeErrorInGate = /\b(?:TypeError|ReferenceError|SyntaxError|RangeError)\b|expect\(received\)/.test(gate);
+  const hostFailureConfirmed = !codeErrorInGate && Boolean(hostRetest && failedTests.some((line) => {
+    const failedPath = /^\s*\[test\]\s+FAIL\s+bun test (\S+\.test\.tsx?)\s+[—–-]/i.exec(line)?.[1];
+    return failedPath === hostRetest?.[1];
+  }));
+  const reviews = commentText.flatMap((text) => {
+    const round = Number(/<!--\s*elanous-pr-comment v1 role=reviewer round=(\d+)\b/.exec(text)?.[1] ?? /\bRound\s+(\d+):\s*reviewer requested\b/i.exec(text)?.[1] ?? -1);
+    const mustFix = /\bRound\s+\d+:\s*reviewer requested\s+(\d+)\s+must-fix change\(s\)/i.exec(text);
+    return mustFix ? [{ round, count: Number(mustFix[1]), evidence: mustFix[0] }] : [];
+  }).sort((a, b) => b.round - a.round);
+  const latest = reviews[0];
+  if (latest) signals.push(latest.evidence);
+
+  const stopEvidence = `${reason}\n${verdict}\n${label}\n${basis}\n${body.match(/정지\((?:needs-human|no-progress)\)/g)?.join('\n') ?? ''}`;
+  let category: StoppedPrCategory = 'unknown';
+  let action = '하지 않음 — 정지 원인 근거 부족';
+  if (hostFailureConfirmed && failedTests.length > 0 && (/(?:자식\s*무관|environment(?:-gap|al)?|환경\s*결손)/i.test(`${label}\n${basis}`) || failedPwaTest)) {
+    category = 'environment-gap';
+    action = '그래프 인스턴스 성형 — 호스트에서 실패 시험만 재측; 원 Gate 보존';
+  } else if (/(?:메모\s*미반영|memo[- ]not[- ]reflected)/i.test(`${label}\n${basis}\n${reason}`)) {
+    category = 'memo-not-reflected';
+    action = '안내 메모 — 동일 칸의 살아 있는 런과 다음 산출 대조; 대상 불명이면 하지 않음';
+  } else if (/(?:리뷰\s*진동|설계\s*역행|review[- ]oscillation)/i.test(`${label}\n${basis}\n${reason}`)) {
+    category = 'review-oscillation';
+    action = latest?.count ? '헬퍼 자식 spawn — 기존 시험 설계 보존 후 한 라운드' : '하지 않음 — 구체적인 must-fix 없음';
+  } else if (label === 'goal-unconvergeable-candidate' || basis === 'supervisor-unconvergeable-goal-candidate') {
+    category = 'goal-revision';
+    signals.push(label === 'goal-unconvergeable-candidate'
+      ? `classification=${label} → goal-revision`
+      : `classificationBasis=${basis} → goal-revision`);
+    action = '골 개정 — 골 문면(판정 신호·경계)을 고쳐 재발사; 구현 자식 붙이지 않음';
+  } else if (label === 'report-deficit') {
+    category = 'report-deficit';
+    signals.push(`classification=${label} → report-deficit`);
+    action = '증거 재측 — 인용된 판정 명령을 호스트에서 다시 돌려 결과를 PR 에 남김; 코드 수정 아님';
+  } else if (/review-budget/i.test(stopEvidence) || basis === 'must-fix-reported') {
+    // Real supervisor bodies (UNCONVERGEABLE · implementation-deficit) carry `classificationBasis: must-fix-reported` instead of the RFC word.
+    if (basis === 'must-fix-reported') signals.push('classificationBasis=must-fix-reported → review-budget');
+    category = 'review-budget';
+    action = latest?.count ? '헬퍼 자식 spawn — must-fix 원문과 기존 시험 설계로 한 라운드' : '하지 않음 — must-fix 0 또는 미관측; 수리 자식 대상 아님';
+  } else if (/no-progress/i.test(stopEvidence) || basis === 'no-must-fix-without-clean-worktree-or-completed-without-changes') {
+    if (basis === 'no-must-fix-without-clean-worktree-or-completed-without-changes') signals.push(`classificationBasis=${basis} → no-progress`);
+    category = 'no-progress';
+    const green = /\[test\]\s+PASS\b/.test(gate) && !/\[test\]\s+FAIL\b/.test(gate);
+    const documentGoal = /(?:문서\s*(?:골|목표)|docs?\/|RFC)/i.test(`${reason}\n${basis}\n${body.split(/\r?\n/)[0] ?? ''}`);
+    action = green && latest?.count === 0 && documentGoal
+      ? '중간 착지 ⊕ 연결 골 — must-fix 0 · 시험 초록 · 문서 골이면 잔여 확인; 잔여 0이면 연결 골 하지 않음'
+      : '중간 착지 ⊕ 연결 골 — 녹색 조각만 확인; 불가분 골·실패 Gate 착지 하지 않음';
+  }
+  return {
+    category, signals,
+    firstAction: allBlockers.length
+      ? `Gate 차단 먼저${publicExportLeak ? ' (✗ public-export-leak 확인)' : category === 'environment-gap' ? ' (그래프 인스턴스 성형으로 호스트 실패 시험 재측)' : ''}`
+      : action,
+    blockers, blockerCount: allBlockers.length, hasPublicExportLeak: Boolean(publicExportLeak),
+  };
+}
+
+/** Stopped means the supervisor wrote a stop verdict or reason into the `## 중단 사유` section — a quoted heading or stop wording elsewhere in the body (goal text, templates) does not count. */
+export function isStoppedPrBody(body: string): boolean {
+  const stop = section(body, '중단 사유');
+  return Boolean(field(stop, 'verdict') || field(stop, 'reason'));
+}
+
+export type ScannedStoppedPr = StoppedPrClassification & { pr: number; headRefName: string; isDraft: boolean };
+export type RunGh = (args: readonly string[]) => string | Promise<string>;
+
+export async function scanStoppedPrs({ runGh }: { runGh: RunGh }): Promise<ScannedStoppedPr[]> {
+  const prs = JSON.parse(await runGh(['pr', 'list', '--state', 'open', '--limit', '100000', '--json', 'number,headRefName,isDraft,body'])) as Array<{ number: number; headRefName: string; isDraft: boolean; body: string }>;
+  const results: ScannedStoppedPr[] = [];
+  for (const pr of prs) {
+    if (!pr.headRefName.startsWith('self-impl/') || !isStoppedPrBody(pr.body)) continue;
+    const data = JSON.parse(await runGh(['pr', 'view', String(pr.number), '--json', 'comments'])) as { comments: Array<{ body: string }> };
+    const result = classifyStoppedPr({ body: pr.body, comments: data.comments });
+    debug.log('harness.helper', 'classified', { pr: pr.number, category: result.category, firstAction: result.firstAction, blockers: result.blockers, blockerCount: result.blockerCount, hasPublicExportLeak: result.hasPublicExportLeak });
+    results.push({ pr: pr.number, headRefName: pr.headRefName, isDraft: pr.isDraft, ...result });
+  }
+  return results;
+}
+
+if (import.meta.main) {
+  if (process.argv.slice(2).some((arg) => arg !== '--json')) {
+    console.error('사용법: bun src/harness/helper-scan.ts [--json]');
+    process.exitCode = 1;
+  } else {
+    const rows = await scanStoppedPrs({ runGh: (args) => execFileSync('gh', [...args], { encoding: 'utf8' }) });
+    if (process.argv.includes('--json')) console.log(JSON.stringify(rows, null, 2));
+    else for (const row of rows) {
+      const gate = row.blockerCount ? ` · Gate 차단 ${row.blockerCount}(${row.hasPublicExportLeak ? row.blockers.find((line) => /public-export-leak/i.test(line))?.trim() ?? '✗ public-export-leak (6번째 이후)' : row.blockers[0]!.trim()})` : '';
+      console.log(`#${row.pr} ${row.category}${gate} → ${row.firstAction}`);
+    }
+  }
+}