@welluable/orch 1.2.0 → 1.4.0

package/lib/continue.js

@@ -0,0 +1,150 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { readJob, reconcileJob, jobPaths } from './jobs.js';
+import { readSeq } from './seq.js';
+
+const CONTINUE_ELIGIBLE = new Set(['done']);
+const FAILURE_TERMINALS = new Set(['failed', 'stopped', 'crashed']);
+
+/** Plain Error whose `toString()` is just the message (no `Error:` prefix), so
+ * `assert.throws(fn, /^exact message$/)` contracts match Node's RegExp check. */
+function fail(message) {
+    const err = new Error(message);
+    err.name = '';
+    return err;
+}
+
+/**
+ * Pure eligibility gate for `orch continue`. Reconciles first; never mutates
+ * beyond reconcile's dead-pid → crashed rewrite. Throws plain Error messages
+ * (CLI wraps with `Error: ${err.message}`).
+ *
+ * Continue is follow-up work on primarily `done` jobs. Terminal failures
+ * refuse by default and point at `orch resume` (see `.spec/resume.md`).
+ */
+export function validateContinue(cwd, slug, { task, ask, quick } = {}) {
+    const existing = readJob(cwd, slug);
+    if (!existing) throw fail(`unknown run ${slug}`);
+
+    if (ask) throw fail('orch continue does not support --ask; use the default orch command');
+    if (quick) throw fail('orch continue does not support --quick; use the default orch command');
+
+    if (typeof task !== 'string' || !task.trim()) {
+        throw fail('task cannot be empty');
+    }
+
+    const record = reconcileJob(cwd, slug, existing);
+
+    if (record.role === 'coordinator') {
+        if (readSeq(cwd, slug)) {
+            throw fail(
+                `cannot continue coordinator ${slug}; continue each failed unit slug, then orch --seq-continue ${slug} or orch resume ${slug}`,
+            );
+        }
+        throw fail(
+            `cannot continue coordinator ${slug}; continue each failed worker slug, then orch --integrate ${slug}`,
+        );
+    }
+
+    if (record.role === 'integration') {
+        const parent = record.parent ?? '<parent-slug>';
+        throw fail(
+            `cannot continue integration ${slug}; use orch --integrate ${parent}`,
+        );
+    }
+
+    if (FAILURE_TERMINALS.has(record.state)) {
+        const phase = record.phase ?? record.lastOutcome?.phase ?? '?';
+        const stage = record.stage ?? record.lastOutcome?.stage ?? '?';
+        throw fail(
+            `${slug} is ${record.state} at ${phase}/${stage};\nuse: orch resume ${slug}`,
+        );
+    }
+
+    if (!CONTINUE_ELIGIBLE.has(record.state)) {
+        throw fail(
+            `cannot continue ${slug} while state is ${record.state}; use orch resume / orch stop`,
+        );
+    }
+
+    if (!record.worktree || !record.branch) {
+        throw fail(`${slug} has no worktree; continue only applies to complex runs`);
+    }
+
+    if (!fs.existsSync(record.worktree)) {
+        throw fail(`worktree missing at ${record.worktree}; cannot continue ${slug}`);
+    }
+
+    return record;
+}
+
+/**
+ * Snapshot prior outcome for continue reopen. Prefer `record.lastOutcome`;
+ * otherwise synthesize from terminal fields (+ best-effort status.md scrape).
+ * Never throws.
+ */
+export function snapshotPriorOutcome(cwd, slug, record) {
+    if (record?.lastOutcome) return record.lastOutcome;
+
+    let summary = '';
+    try {
+        const statusPath = path.join(jobPaths(cwd, slug).dir, 'status.md');
+        if (fs.existsSync(statusPath)) {
+            const content = fs.readFileSync(statusPath, 'utf8');
+            const matches = [...content.matchAll(/^- Summary:\s*(.*)$/gm)];
+            if (matches.length > 0) {
+                summary = (matches[matches.length - 1][1] || '').trim();
+            }
+        }
+    } catch {
+        summary = '';
+    }
+
+    return {
+        state: record.state,
+        phase: record.phase ?? null,
+        stage: record.stage ?? null,
+        round: record.round ?? null,
+        exitCode: record.exitCode ?? null,
+        finishedAt: record.finishedAt ?? null,
+        task: record.task ?? null,
+        summary,
+        error: null,
+    };
+}
+
+function displayOrNone(value) {
+    if (value == null || value === '') return '(none recorded)';
+    return String(value);
+}
+
+/**
+ * Build the `[Prior run outcome]…[/Prior run outcome]` block injected into
+ * research/planner prompts on continue.
+ */
+export function buildPriorOutcomeText(prior, {
+    slug,
+    continuation,
+    worktreePath,
+    branch,
+    parentSlug,
+    workerId,
+} = {}) {
+    const p = prior ?? {};
+    const lines = [
+        '[Prior run outcome]',
+        `- Continuation: this is continue ${continuation} on slug ${slug}`,
+        `- Prior state: ${displayOrNone(p.state)}`,
+        `- Prior phase: ${displayOrNone(p.phase)}`,
+        `- Prior stage: ${displayOrNone(p.stage)}`,
+        `- Prior round: ${p.round == null ? 'null' : p.round}`,
+        `- Prior task: ${displayOrNone(p.task)}`,
+        `- Summary: ${displayOrNone(p.summary)}`,
+        `- Error: ${displayOrNone(p.error)}`,
+        `- Worktree: ${worktreePath} (branch ${branch} tip is the starting point; do not assume a clean tree)`,
+    ];
+    if (parentSlug) lines.push(`- Fan-out parent: ${parentSlug}`);
+    if (workerId) lines.push(`- Worker id: ${workerId}`);
+    lines.push('[/Prior run outcome]');
+    return lines.join('\n');
+}

package/lib/failure-log.js

@@ -0,0 +1,72 @@
+import fs from 'node:fs';
+import path from 'node:path';
+
+/**
+ * Pure failure.log section formatting + append. No jobs imports — safe for
+ * both lib/agent.js (in-process buffer flush) and lib/jobs.js (crashed
+ * header-only write) without circular dependencies.
+ */
+
+/** Relative pointer for status / lastOutcome.error (matches resume.md sample). */
+export function failureLogPointer(slug) {
+    return `see .orch/${slug}/failure.log`;
+}
+
+function fmtField(label, value) {
+    const rendered = value == null || value === '' ? '' : String(value);
+    return `${label}:${' '.repeat(Math.max(1, 11 - label.length))}${rendered}`;
+}
+
+/**
+ * Build one `=== orch failure ===` section (header + stage verbose + optional
+ * prior summaries). `stageVerbose` may be empty (crashed / header-only).
+ */
+export function formatFailureSection({
+    slug,
+    state,
+    phase = null,
+    stage = null,
+    round = null,
+    exitCode = null,
+    finishedAt,
+    task = null,
+    error = null,
+    stageVerbose = '',
+    priorStages = [],
+}) {
+    const stageLabel = stage ?? 'unknown';
+    const lines = [
+        '=== orch failure ===',
+        fmtField('slug', slug),
+        fmtField('state', state),
+        fmtField('phase', phase),
+        fmtField('stage', stage),
+        fmtField('round', round),
+        fmtField('exitCode', exitCode),
+        fmtField('finishedAt', finishedAt),
+        fmtField('task', task),
+        fmtField('error', error),
+        '',
+        `=== stage verbose (${stageLabel}) ===`,
+        stageVerbose.endsWith('\n') || stageVerbose === '' ? stageVerbose : `${stageVerbose}\n`,
+    ];
+
+    if (priorStages.length > 0) {
+        lines.push('=== prior stage summaries (best-effort) ===');
+        for (const prior of priorStages) {
+            const label = prior.stage ?? prior.phase ?? 'prior';
+            lines.push(`--- ${label} ---`);
+            const body = prior.verbose ?? '';
+            lines.push(body.endsWith('\n') || body === '' ? body : `${body}\n`);
+        }
+    }
+
+    return lines.join('\n');
+}
+
+/** Append a section to `failureLogPath`, creating the file/dir if needed. */
+export function appendFailureLog(failureLogPath, section) {
+    fs.mkdirSync(path.dirname(failureLogPath), { recursive: true });
+    const prefix = fs.existsSync(failureLogPath) ? '\n' : '';
+    fs.appendFileSync(failureLogPath, prefix + section, 'utf8');
+}

package/lib/file-tracker.js

@@ -1,4 +1,5 @@
 import path from 'node:path';
+import { toolPath } from './tool-status.js';
 
 /** @param {string} name */
 function markerForName(name) {
@@ -9,11 +10,42 @@ function markerForName(name) {
     return null;
 }
 
-/** @param {Record<string, unknown>|null|undefined} args */
-function extractPath(args) {
-    if (!args || typeof args !== 'object') return null;
-    const p = args.path ?? args.file_path;
-    return typeof p === 'string' && p ? p : null;
+/**
+ * Parse OpenCode `apply_patch` marker lines into file-trail entries.
+ * @param {unknown} patchText
+ * @returns {{ marker: string, path: string }[]}
+ */
+export function parseApplyPatchPaths(patchText) {
+    if (typeof patchText !== 'string' || !patchText) return [];
+
+    /** @type {{ marker: string, path: string }[]} */
+    const entries = [];
+    for (const line of patchText.split('\n')) {
+        const add = line.match(/^\*\*\* Add File:\s*(.+?)\s*$/);
+        if (add) {
+            entries.push({ marker: '+', path: add[1] });
+            continue;
+        }
+        const update = line.match(/^\*\*\* Update File:\s*(.+?)\s*$/);
+        if (update) {
+            entries.push({ marker: '~', path: update[1] });
+            continue;
+        }
+        const del = line.match(/^\*\*\* Delete File:\s*(.+?)\s*$/);
+        if (del) {
+            entries.push({ marker: '-', path: del[1] });
+            continue;
+        }
+        const move = line.match(/^\*\*\* Move to:\s*(.+?)\s*$/);
+        if (move) {
+            // Destination only; drop the preceding Update File source entry.
+            if (entries.length > 0 && entries[entries.length - 1].marker === '~') {
+                entries.pop();
+            }
+            entries.push({ marker: '~', path: move[1] });
+        }
+    }
+    return entries;
 }
 
 /**
@@ -43,7 +75,7 @@ export class FileTracker {
             this.pending.set(callId, {
                 name,
                 marker,
-                path: extractPath(args),
+                path: toolPath(args),
             });
             return null;
         }
@@ -57,9 +89,38 @@ export class FileTracker {
         const marker = markerForName(toolName) || prior?.marker || null;
         if (!marker) return null;
 
-        const rawPath = extractPath(args) || prior?.path || null;
+        const rawPath = toolPath(args) || prior?.path || null;
         if (!rawPath) return null;
 
+        return this.#upsert(marker, rawPath);
+    }
+
+    /**
+     * Record paths from a completed OpenCode `apply_patch` into the trail.
+     * @param {unknown} patchText
+     * @returns {{ marker: string, path: string, isNew: boolean }[]}
+     */
+    recordApplyPatch(patchText) {
+        /** @type {{ marker: string, path: string, isNew: boolean }[]} */
+        const out = [];
+        for (const entry of parseApplyPatchPaths(patchText)) {
+            const result = this.#upsert(entry.marker, entry.path);
+            if (result) out.push(result);
+        }
+        return out;
+    }
+
+    /** @returns {{ marker: string, path: string }[]} */
+    getFiles() {
+        return this.files.map((f) => ({ marker: f.marker, path: f.path }));
+    }
+
+    /**
+     * @param {string} marker
+     * @param {string} rawPath
+     * @returns {{ marker: string, path: string, isNew: boolean }}
+     */
+    #upsert(marker, rawPath) {
         const relPath = this.#toRelative(rawPath);
         const existingIdx = this.indexByPath.get(relPath);
         const isNew = existingIdx === undefined;
@@ -74,11 +135,6 @@ export class FileTracker {
         return { marker, path: relPath, isNew };
     }
 
-    /** @returns {{ marker: string, path: string }[]} */
-    getFiles() {
-        return this.files.map((f) => ({ marker: f.marker, path: f.path }));
-    }
-
     /** @param {string} filePath */
     #toRelative(filePath) {
         if (path.isAbsolute(filePath)) {

package/lib/jobs.js

@@ -1,10 +1,73 @@
 import fs from 'node:fs';
 import path from 'node:path';
 import crypto from 'node:crypto';
+import {
+    appendFailureLog,
+    failureLogPointer,
+    formatFailureSection,
+} from './failure-log.js';
+import { getNotifyEnabled, notifyJob as defaultNotifyJob } from './notify.js';
 
 const ACTIVE_LIVE_STATES = ['running', 'pausing', 'paused'];
 const TERMINAL_STATES = ['done', 'failed', 'stopped', 'crashed'];
 
+/** Injectable notify hook for tests; default is `notifyJob` from lib/notify.js. */
+let notifyJobHook = defaultNotifyJob;
+
+/** Override the notify helper used on terminal state transitions (tests). */
+export function setNotifyJob(fn) {
+    notifyJobHook = typeof fn === 'function' ? fn : defaultNotifyJob;
+}
+
+/** Restore the default notify hook. */
+export function resetNotifyHooks() {
+    notifyJobHook = defaultNotifyJob;
+}
+
+function maybeNotifyTerminalTransition(previousState, updated) {
+    if (!getNotifyEnabled()) return;
+    if (!TERMINAL_STATES.includes(updated.state)) return;
+    if (TERMINAL_STATES.includes(previousState)) return;
+    try {
+        notifyJobHook({
+            slug: updated.slug,
+            state: updated.state,
+            task: updated.task,
+            enabled: true,
+        });
+    } catch {
+        // Notification failures must never affect job writes.
+    }
+}
+
+/**
+ * Compact outcome written onto `run.json` with every terminal state transition.
+ * Shared so all call sites (pipelines, shutdown, reconcile) keep one shape.
+ */
+export function buildLastOutcome({
+    state,
+    phase = null,
+    stage = null,
+    round = null,
+    exitCode = null,
+    finishedAt,
+    task = null,
+    summary = '',
+    error = null,
+}) {
+    return {
+        state,
+        phase: phase ?? null,
+        stage: stage ?? null,
+        round: round ?? null,
+        exitCode: exitCode ?? null,
+        finishedAt,
+        task: task ?? null,
+        summary: summary ?? '',
+        error: error ?? null,
+    };
+}
+
 /** Absolute paths for a job's on-disk artifacts under `<cwd>/.orch/<slug>/`. */
 export function jobPaths(cwd, slug) {
     const dir = path.join(path.resolve(cwd), '.orch', slug);
@@ -13,6 +76,7 @@ export function jobPaths(cwd, slug) {
         runJsonPath: path.join(dir, 'run.json'),
         lockPath: path.join(dir, '.run.lock'),
         logPath: path.join(dir, 'orch.log'),
+        failureLogPath: path.join(dir, 'failure.log'),
     };
 }
 
@@ -104,20 +168,57 @@ function acquireLock(lockPath, { timeoutMs = 5000, retryMs = 5 } = {}) {
     }
 }
 
+/**
+ * Minimal valid job record used when `patchJob` rehydrates a missing
+ * `run.json`. Mirrors `allocateJob` defaults and always includes `slug`.
+ */
+function rehydrateJobRecord(cwd, slug) {
+    return {
+        slug,
+        task: null,
+        agent: null,
+        maxRounds: null,
+        cwd: path.resolve(cwd),
+        pauseRequested: false,
+        branch: null,
+        worktree: null,
+        startedAt: new Date().toISOString(),
+        finishedAt: null,
+        exitCode: null,
+        logPath: jobPaths(cwd, slug).logPath,
+        pid: null,
+        state: 'starting',
+        phase: null,
+        stage: null,
+        round: null,
+        parent: null,
+        role: null,
+        workerId: null,
+    };
+}
+
 /**
  * Locks, re-reads, shallow-merges `patchFnOrObject` (an object, or a
  * `(current) => partialPatch` function) over the latest record, atomically
- * writes, unlocks, and returns the updated record.
+ * writes, unlocks, and returns the updated record. When `run.json` is
+ * missing, rehydrates a minimal valid base (always including `slug`) before
+ * merging — never writes a slug-less stub from `{}`.
  */
 export function patchJob(cwd, slug, patchFnOrObject) {
     const { dir, runJsonPath, lockPath } = jobPaths(cwd, slug);
     fs.mkdirSync(dir, { recursive: true });
     acquireLock(lockPath);
     try {
-        const current = readJob(cwd, slug) ?? {};
+        let current = readJob(cwd, slug);
+        if (!current) {
+            console.warn(`patchJob: rehydrating missing run.json for ${slug}`);
+            current = rehydrateJobRecord(cwd, slug);
+        }
+        const previousState = current.state;
         const patch = typeof patchFnOrObject === 'function' ? patchFnOrObject(current) : patchFnOrObject;
-        const updated = { ...current, ...patch };
+        const updated = { ...current, ...patch, slug };
         atomicWriteJson(dir, runJsonPath, updated);
+        maybeNotifyTerminalTransition(previousState, updated);
         return updated;
     } finally {
         try {
@@ -136,10 +237,78 @@ export function patchJob(cwd, slug, patchFnOrObject) {
 export function reconcileJob(cwd, slug, record) {
     if (!ACTIVE_LIVE_STATES.includes(record.state)) return record;
     if (isPidAlive(record.pid)) return record;
+    return patchJob(cwd, slug, (current) => {
+        const finishedAt = new Date().toISOString();
+        const { failureLogPath } = jobPaths(cwd, slug);
+        // Header-only: out-of-process crash has no in-memory stage buffer.
+        appendFailureLog(
+            failureLogPath,
+            formatFailureSection({
+                slug,
+                state: 'crashed',
+                phase: current.phase,
+                stage: current.stage,
+                round: current.round,
+                exitCode: null,
+                finishedAt,
+                task: current.task,
+                error: 'process died',
+                stageVerbose: '',
+                priorStages: [],
+            }),
+        );
+        return {
+            state: 'crashed',
+            finishedAt,
+            exitCode: null,
+            failureLogPath,
+            lastOutcome: buildLastOutcome({
+                state: 'crashed',
+                phase: current.phase,
+                stage: current.stage,
+                round: current.round,
+                exitCode: null,
+                finishedAt,
+                task: current.task,
+                summary: '',
+                error: failureLogPointer(slug),
+            }),
+        };
+    });
+}
+
+/**
+ * Reopen an existing terminal job for `orch continue`. Patches in place —
+ * never allocates a new slug/directory. Bumps `continuation`, appends
+ * `continuations[]`, clears live `lastOutcome`, resets to running/research.
+ */
+export function reopenJob(cwd, slug, { task, agent, maxRounds, pid, prior, startedAt } = {}) {
+    const existing = readJob(cwd, slug);
+    if (!existing) throw new Error(`reopenJob: unknown job ${slug}`);
+
+    const started = startedAt ?? new Date().toISOString();
+    const continuation = (existing.continuation ?? 1) + 1;
+    const continuations = Array.isArray(existing.continuations)
+        ? [...existing.continuations]
+        : [];
+    continuations.push({ n: continuation, task, startedAt: started, prior: prior ?? null });
+
     return patchJob(cwd, slug, {
-        state: 'crashed',
-        finishedAt: new Date().toISOString(),
+        task,
+        agent,
+        maxRounds,
+        pid,
+        startedAt: started,
+        state: 'running',
+        phase: 'research',
+        stage: null,
+        round: null,
+        finishedAt: null,
         exitCode: null,
+        pauseRequested: false,
+        lastOutcome: null,
+        continuation,
+        continuations,
     });
 }
 
@@ -156,7 +325,9 @@ export function listJobs(cwd) {
     for (const slug of slugs) {
         const record = readJob(cwd, slug);
         if (!record) continue;
-        jobs.push(reconcileJob(cwd, slug, record));
+        // Directory name is the authority when run.json omits/corrupts slug.
+        const normalized = { ...record, slug: record.slug ?? slug };
+        jobs.push(reconcileJob(cwd, slug, normalized));
     }
 
     jobs.sort((a, b) => new Date(b.startedAt).getTime() - new Date(a.startedAt).getTime());
@@ -267,15 +438,46 @@ export function cascadeResume(cwd, parentSlug) {
     return { childrenSignaled };
 }
 
+/**
+ * Slugs of jobs in an active live state whose pid is still alive.
+ * Used by `cleanJobs` / `orch jobs clean` to refuse wiping live runs.
+ */
+export function liveSlugsBlockingClean(cwd) {
+    const orchDir = path.join(path.resolve(cwd), '.orch');
+    if (!fs.existsSync(orchDir)) return [];
+
+    const live = [];
+    for (const entry of fs.readdirSync(orchDir, { withFileTypes: true })) {
+        if (!entry.isDirectory()) continue;
+        const record = readJob(cwd, entry.name);
+        if (!record) continue;
+        if (!ACTIVE_LIVE_STATES.includes(record.state)) continue;
+        if (!isPidAlive(record.pid)) continue;
+        live.push(record.slug ?? entry.name);
+    }
+    return live;
+}
+
 /**
  * The pure operation behind `orch jobs clean`: removes every entry under
  * `<cwd>/.orch/`. Returns the names that were deleted (empty if `.orch`
- * is missing or already empty).
+ * is missing or already empty). Refuses (throws) when any job is in an
+ * active live state (`running`/`pausing`/`paused`) with an alive pid —
+ * caller must `orch stop <slug>` first. Dead-pid "live" states may still
+ * be cleaned.
  */
 export function cleanJobs(cwd) {
     const orchDir = path.join(path.resolve(cwd), '.orch');
     if (!fs.existsSync(orchDir)) return [];
 
+    const live = liveSlugsBlockingClean(cwd);
+    if (live.length > 0) {
+        throw new Error(
+            `cannot clean while live jobs exist: ${live.join(', ')}. ` +
+            `Stop them first with: orch stop <slug>`,
+        );
+    }
+
     const names = fs.readdirSync(orchDir);
     for (const name of names) {
         fs.rmSync(path.join(orchDir, name), { recursive: true, force: true });

package/lib/notify.js

@@ -0,0 +1,71 @@
+import { spawn as defaultSpawn } from 'node:child_process';
+
+const TERMINAL_STATES = new Set(['done', 'failed', 'stopped', 'crashed']);
+
+/** Process-level gate; default off so unit tests that patch jobs stay silent. */
+let notifyEnabled = false;
+
+/** Sets whether terminal job transitions should fire desktop notifications. */
+export function setNotifyEnabled(enabled) {
+    notifyEnabled = Boolean(enabled);
+}
+
+/** Current process-level notify gate. */
+export function getNotifyEnabled() {
+    return notifyEnabled;
+}
+
+function shortTask(task) {
+    if (task == null) return '';
+    const text = String(task).trim();
+    if (!text) return '';
+    if (text.length <= 80) return text;
+    return `${text.slice(0, 80)}…`;
+}
+
+function buildBody(state, task) {
+    const short = shortTask(task);
+    return short ? `${state} — ${short}` : state;
+}
+
+/**
+ * Fire-and-forget desktop notification for a terminal job state.
+ * Inject `spawn` / `platform` in tests. Never throws.
+ */
+export function notifyJob({
+    slug,
+    state,
+    task,
+    enabled,
+    spawn: spawnFn = defaultSpawn,
+    platform = process.platform,
+} = {}) {
+    try {
+        if (enabled === false) return;
+        if (!TERMINAL_STATES.has(state)) return;
+
+        const title = `orch · ${slug}`;
+        const body = buildBody(state, task);
+
+        if (platform === 'darwin') {
+            const script = `display notification ${JSON.stringify(body)} with title ${JSON.stringify(title)}`;
+            const child = spawnFn('osascript', ['-e', script], {
+                stdio: 'ignore',
+                detached: true,
+            });
+            child.unref?.();
+            return;
+        }
+
+        if (platform === 'linux') {
+            const child = spawnFn('notify-send', ['--', title, body], {
+                stdio: 'ignore',
+                detached: true,
+            });
+            child.unref?.();
+            return;
+        }
+    } catch {
+        // Best-effort: never fatal.
+    }
+}