@theowlops/boombot 0.0.0-stage → 1.1.1

package/dist/node_modules/@boombot/core/task-runner/runner.js

@@ -0,0 +1,564 @@
+// Background task runner: durable multi-step tasks executed through an
+// injected, transport-agnostic TurnRunner. Depends only on node builtins.
+//
+// Concurrency model:
+// - One AbortController per task (taskId -> controller).
+// - One background loop per task at most (activeLoops guard).
+// - Steps run sequentially inside a task; tasks run concurrently.
+// - HITL: when a turn reports approval_required, the task parks in
+//   awaiting_approval WITHOUT aborting the turn — the turn service holds the
+//   runTurn promise open. resume(id, decision) delivers the verdict.
+import { randomUUID } from 'node:crypto';
+import { TaskError } from './types.js';
+function nowIso() {
+    return new Date().toISOString();
+}
+function isRetryableStatus(status) {
+    return status === 'failed' || status === 'cancelled' || status === 'interrupted';
+}
+export function createTaskRunner(options) {
+    const { store, turnRunner, onEvent } = options;
+    const controllers = new Map();
+    const activeLoops = new Set();
+    /**
+     * Per-task mutex serializing every read-modify-save section.
+     *
+     * The store hands out copies (`structuredClone`) and `save()` is a blind
+     * upsert, so two interleaved read-modify-save sequences silently lose one
+     * update (last writer wins): a `pause()` racing a completing turn could be
+     * silently undone, or a fresh checkpoint wiped, depending on which save
+     * landed last. Sections never hold the lock across `runTurn` or other
+     * I/O — only across the synchronous read-modify plus the `save()` itself
+     * — so control-plane calls stay responsive, and the lock is never nested
+     * (no deadlock).
+     */
+    const taskLocks = new Map();
+    async function withTaskLock(taskId, fn) {
+        const prev = taskLocks.get(taskId) ?? Promise.resolve();
+        let release;
+        const gate = new Promise((resolve) => {
+            release = resolve;
+        });
+        const chained = prev.then(() => gate);
+        taskLocks.set(taskId, chained);
+        await prev;
+        try {
+            return await fn();
+        }
+        finally {
+            release();
+            if (taskLocks.get(taskId) === chained) {
+                taskLocks.delete(taskId);
+            }
+        }
+    }
+    /**
+     * Turn results that settled while their task was in awaiting_approval.
+     * resume(id, { approved: true }) applies the stashed result; this covers
+     * transports that resolve runTurn at the same time as (or just before)
+     * reporting approval_required.
+     */
+    const pendingTurnResults = new Map();
+    function emit(event) {
+        try {
+            onEvent?.(event);
+        }
+        catch {
+            // A listener must never break the runner.
+        }
+    }
+    function requireTask(id) {
+        const task = store.get(id);
+        if (task === null) {
+            throw new TaskError('NOT_FOUND', `Task ${id} not found`);
+        }
+        return task;
+    }
+    async function saveAndSnapshot(task) {
+        task.updatedAt = nowIso();
+        await store.save(task);
+        const snapshot = store.get(task.id);
+        if (snapshot === null) {
+            throw new TaskError('STORAGE', `Task ${task.id} vanished after save`);
+        }
+        return snapshot;
+    }
+    async function failTask(task, error) {
+        task.status = 'failed';
+        task.error = error;
+        const snapshot = await saveAndSnapshot(task);
+        emit({ type: 'task.failed', taskId: task.id, error });
+        return snapshot;
+    }
+    async function failStep(task, stepIndex, error) {
+        const step = task.steps[stepIndex];
+        if (step !== undefined) {
+            step.status = 'failed';
+            step.error = error;
+            step.endedAt = nowIso();
+        }
+        return failTask(task, error);
+    }
+    /**
+     * Record a finished turn against its step. Safe to call from both the
+     * background loop and resume(): it no-ops unless the task is running and
+     * the step is still the in-flight one.
+     */
+    async function applyTurnResult(taskId, stepIndex, result) {
+        await withTaskLock(taskId, async () => {
+            const task = store.get(taskId);
+            if (task === null || task.status !== 'running')
+                return;
+            const step = task.steps[stepIndex];
+            if (step === undefined || step.status !== 'running')
+                return;
+            if (result.status === 'completed') {
+                if (result.sessionId !== undefined)
+                    task.sessionId = result.sessionId;
+                step.status = 'done';
+                if (result.output !== undefined)
+                    step.output = result.output;
+                step.endedAt = nowIso();
+                task.messageQueue = []; // redirects are consumed by this successful step
+                task.checkpoints.push({ stepIndex, at: nowIso() });
+                task.currentStepIndex = stepIndex + 1;
+                await saveAndSnapshot(task);
+                emit({ type: 'task.checkpoint', taskId, stepIndex });
+            }
+            else {
+                await failStep(task, stepIndex, result.error ?? 'turn failed');
+            }
+        });
+    }
+    async function handleApprovalRequired(taskId, event) {
+        await withTaskLock(taskId, async () => {
+            const task = store.get(taskId);
+            // Ignore stale events (pause/cancel won the race).
+            if (task === null || task.status !== 'running')
+                return;
+            // NOTE: the turn is NOT aborted — the turn service keeps the runTurn
+            // promise open and waits for resume(id, decision).
+            task.status = 'awaiting_approval';
+            task.approval = {
+                runId: event.runId,
+                toolName: event.toolName,
+                args: event.args,
+                reason: event.reason,
+            };
+            await saveAndSnapshot(task);
+            emit({
+                type: 'task.awaiting_approval',
+                taskId,
+                runId: event.runId,
+                toolName: event.toolName,
+                reason: event.reason,
+            });
+        });
+    }
+    async function runStep(taskId, stepIndex) {
+        // The read-modify-save is locked; the turn itself runs outside the lock.
+        const started = await withTaskLock(taskId, async () => {
+            const task = store.get(taskId);
+            if (task === null || task.status !== 'running')
+                return undefined;
+            const step = task.steps[stepIndex];
+            if (step === undefined || step.status !== 'pending')
+                return undefined;
+            // Drain redirect queue into this step's prompt. The queue is cleared only
+            // when the step succeeds, so a pause/failure never loses a redirect.
+            let prompt = step.prompt;
+            if (task.messageQueue.length > 0) {
+                const redirects = task.messageQueue.map((m) => `User redirect: ${m}`).join('\n');
+                prompt = `${redirects}\n\n${prompt}`;
+            }
+            step.status = 'running';
+            step.attempts += 1;
+            step.startedAt = nowIso();
+            delete step.error;
+            task.currentStepIndex = stepIndex;
+            await saveAndSnapshot(task);
+            emit({ type: 'task.progress', taskId, stepIndex, totalSteps: task.steps.length });
+            return { prompt, sessionId: task.sessionId };
+        });
+        if (started === undefined)
+            return;
+        const controller = new AbortController();
+        controllers.set(taskId, controller);
+        let result;
+        try {
+            result = await turnRunner.runTurn({
+                prompt: started.prompt,
+                ...(started.sessionId !== undefined ? { sessionId: started.sessionId } : {}),
+                signal: controller.signal,
+                onEvent: (e) => {
+                    if (e.type === 'approval_required') {
+                        void handleApprovalRequired(taskId, e).catch(() => undefined);
+                    }
+                },
+            });
+        }
+        catch (err) {
+            controllers.delete(taskId);
+            const aborted = controller.signal.aborted;
+            const message = err instanceof Error ? err.message : String(err);
+            await withTaskLock(taskId, async () => {
+                const fresh = store.get(taskId);
+                // pause/cancel/reject already transitioned the task; nothing to do.
+                if (fresh === null || fresh.status !== 'running')
+                    return;
+                if (aborted)
+                    return;
+                await failStep(fresh, stepIndex, message);
+            });
+            return;
+        }
+        controllers.delete(taskId);
+        const fresh = store.get(taskId);
+        if (fresh === null)
+            return;
+        if (fresh.status === 'awaiting_approval') {
+            // The turn settled while (or just as) approval was requested.
+            // Stash it; an approving resume() will apply it.
+            pendingTurnResults.set(taskId, { stepIndex, result });
+            return;
+        }
+        if (fresh.status !== 'running')
+            return; // paused/cancelled meanwhile
+        await applyTurnResult(taskId, stepIndex, result);
+    }
+    async function runLoop(taskId) {
+        if (activeLoops.has(taskId))
+            return; // guard double-start
+        activeLoops.add(taskId);
+        try {
+            for (;;) {
+                const task = store.get(taskId);
+                if (task === null || task.status !== 'running')
+                    return;
+                // Never start a second step while one is still in flight
+                // (e.g. a turn parked on approval whose loop was restarted).
+                const inFlight = task.steps.some((s) => s.status === 'running');
+                if (inFlight)
+                    return;
+                const nextIndex = task.steps.findIndex((s) => s.status === 'pending');
+                if (nextIndex === -1) {
+                    await completeTask(taskId);
+                    return;
+                }
+                await runStep(taskId, nextIndex);
+            }
+        }
+        catch (err) {
+            // Defensive: an unexpected loop error fails the task instead of
+            // silently abandoning it.
+            try {
+                const message = err instanceof Error ? err.message : String(err);
+                await withTaskLock(taskId, async () => {
+                    const task = store.get(taskId);
+                    if (task !== null && task.status === 'running') {
+                        await failTask(task, message);
+                    }
+                });
+            }
+            catch {
+                // Nothing left to do.
+            }
+        }
+        finally {
+            activeLoops.delete(taskId);
+            controllers.delete(taskId);
+            // NOTE: pendingTurnResults is intentionally NOT cleared here — a loop
+            // can exit while parked on approval_required, and resume() still needs
+            // the stashed turn result. cancel()/retry()/recover() clear it.
+        }
+    }
+    async function completeTask(taskId) {
+        await withTaskLock(taskId, async () => {
+            const task = store.get(taskId);
+            if (task === null || task.status !== 'running')
+                return;
+            task.status = 'completed';
+            delete task.error;
+            await saveAndSnapshot(task);
+            emit({ type: 'task.completed', taskId });
+        });
+    }
+    async function createAndStart(input) {
+        const title = input.title.trim();
+        if (title === '') {
+            throw new TaskError('INVALID_STATE', 'Task title must not be empty');
+        }
+        if (input.steps.length === 0) {
+            throw new TaskError('INVALID_STATE', 'Task must have at least one step');
+        }
+        input.steps.forEach((s, i) => {
+            if (s.prompt.trim() === '') {
+                throw new TaskError('INVALID_STATE', `Step ${i} prompt must not be empty`);
+            }
+        });
+        const at = nowIso();
+        const steps = input.steps.map((s, index) => ({
+            id: randomUUID(),
+            index,
+            kind: 'turn',
+            prompt: s.prompt,
+            status: 'pending',
+            attempts: 0,
+        }));
+        const task = {
+            id: randomUUID(),
+            title,
+            status: 'queued',
+            steps,
+            currentStepIndex: 0,
+            checkpoints: [],
+            messageQueue: [],
+            createdAt: at,
+            updatedAt: at,
+        };
+        await store.create(task);
+        emit({ type: 'task.created', taskId: task.id });
+        task.status = 'running';
+        const snapshot = await saveAndSnapshot(task);
+        // Fire-and-forget: the caller never blocks. runLoop funnels every error
+        // into task.failed + persist, so this cannot reject.
+        void runLoop(task.id);
+        return snapshot;
+    }
+    async function get(id) {
+        return store.get(id);
+    }
+    async function list() {
+        return store.list();
+    }
+    async function pause(id) {
+        const snapshot = await withTaskLock(id, async () => {
+            const task = requireTask(id);
+            if (task.status !== 'running') {
+                throw new TaskError('INVALID_STATE', `Cannot pause task in status '${task.status}'`);
+            }
+            task.status = 'paused';
+            const current = task.steps[task.currentStepIndex];
+            if (current !== undefined && current.status === 'running') {
+                current.status = 'pending';
+                current.endedAt = nowIso();
+            }
+            const snapshot = await saveAndSnapshot(task);
+            emit({ type: 'task.paused', taskId: id });
+            return snapshot;
+        });
+        // Abort outside the lock: the in-flight turn's rejection handler takes
+        // the same lock, so aborting while holding it would only add latency.
+        controllers.get(id)?.abort();
+        return snapshot;
+    }
+    async function resume(id, decision) {
+        const task = requireTask(id);
+        if (task.status === 'awaiting_approval') {
+            if (!decision) {
+                throw new TaskError('INVALID_STATE', 'A decision is required to resume a task awaiting approval');
+            }
+            const approval = task.approval;
+            if (!approval) {
+                throw new TaskError('INVALID_STATE', 'Task is awaiting approval but has no approval info');
+            }
+            if (decision.approved) {
+                if (turnRunner.approveTurn) {
+                    const ok = await turnRunner
+                        .approveTurn(approval.runId, true, decision.editedArgs)
+                        .catch(() => false);
+                    if (!ok) {
+                        return withTaskLock(id, async () => failTask(requireTask(id), `Turn service refused to resume run '${approval.runId}'`));
+                    }
+                }
+                // Re-read under the lock: a cancel() may have landed while the
+                // approval I/O was in flight — never resurrect a task that left
+                // awaiting_approval meanwhile.
+                await withTaskLock(id, async () => {
+                    const fresh = requireTask(id);
+                    if (fresh.status !== 'awaiting_approval') {
+                        throw new TaskError('INVALID_STATE', `Cannot resume task in status '${fresh.status}'`);
+                    }
+                    delete fresh.approval;
+                    fresh.status = 'running';
+                    await saveAndSnapshot(fresh);
+                });
+                // Apply a turn result that settled while approval was pending.
+                const pending = pendingTurnResults.get(id);
+                if (pending) {
+                    pendingTurnResults.delete(id);
+                    await applyTurnResult(id, pending.stepIndex, pending.result);
+                }
+                emit({ type: 'task.resumed', taskId: id });
+                // Safe either way: if the old loop is still blocked inside runTurn,
+                // the double-start guard makes this a no-op.
+                void runLoop(id);
+                const snapshot = store.get(id);
+                if (snapshot === null)
+                    throw new TaskError('STORAGE', `Task ${id} vanished after save`);
+                return snapshot;
+            }
+            // Rejected: release the in-flight turn, then fail the step and task.
+            controllers.get(id)?.abort();
+            pendingTurnResults.delete(id);
+            if (turnRunner.approveTurn) {
+                await turnRunner.approveTurn(approval.runId, false, decision.editedArgs).catch(() => false);
+            }
+            return withTaskLock(id, async () => {
+                const fresh = requireTask(id);
+                delete fresh.approval;
+                const current = fresh.steps[fresh.currentStepIndex];
+                if (current !== undefined && (current.status === 'running' || current.status === 'pending')) {
+                    current.status = 'failed';
+                    current.error = 'approval rejected';
+                    current.endedAt = nowIso();
+                }
+                return failTask(fresh, 'approval rejected');
+            });
+        }
+        if (task.status === 'paused') {
+            const snapshot = await withTaskLock(id, async () => {
+                const fresh = requireTask(id);
+                if (fresh.status !== 'paused') {
+                    throw new TaskError('INVALID_STATE', `Cannot resume task in status '${fresh.status}'`);
+                }
+                fresh.status = 'running';
+                return saveAndSnapshot(fresh);
+            });
+            emit({ type: 'task.resumed', taskId: id });
+            void runLoop(id);
+            return snapshot;
+        }
+        throw new TaskError('INVALID_STATE', `Cannot resume task in status '${task.status}'`);
+    }
+    async function cancel(id) {
+        const snapshot = await withTaskLock(id, async () => {
+            const task = requireTask(id);
+            if (task.status === 'completed' || task.status === 'failed' || task.status === 'cancelled') {
+                throw new TaskError('INVALID_STATE', `Cannot cancel task in status '${task.status}'`);
+            }
+            const current = task.steps[task.currentStepIndex];
+            if (current !== undefined && current.status === 'running') {
+                current.status = 'failed';
+                current.error = 'task cancelled';
+                current.endedAt = nowIso();
+            }
+            task.status = 'cancelled';
+            delete task.approval;
+            pendingTurnResults.delete(id);
+            const snapshot = await saveAndSnapshot(task);
+            emit({ type: 'task.cancelled', taskId: id });
+            return snapshot;
+        });
+        controllers.get(id)?.abort();
+        return snapshot;
+    }
+    async function redirect(id, message) {
+        if (message.trim() === '') {
+            throw new TaskError('INVALID_STATE', 'Redirect message must not be empty');
+        }
+        return withTaskLock(id, async () => {
+            const task = requireTask(id);
+            if (task.status !== 'queued' && task.status !== 'running' && task.status !== 'paused') {
+                throw new TaskError('INVALID_STATE', `Cannot redirect task in status '${task.status}'`);
+            }
+            task.messageQueue.push(message);
+            return saveAndSnapshot(task);
+        });
+    }
+    async function retry(id, fromStep) {
+        const snapshot = await withTaskLock(id, async () => {
+            const task = requireTask(id);
+            if (!isRetryableStatus(task.status)) {
+                throw new TaskError('INVALID_STATE', `Cannot retry task in status '${task.status}'`);
+            }
+            let start;
+            if (fromStep !== undefined) {
+                if (!Number.isInteger(fromStep) || fromStep < 0 || fromStep >= task.steps.length) {
+                    throw new TaskError('INVALID_STATE', `fromStep ${fromStep} is out of range`);
+                }
+                start = fromStep;
+            }
+            else {
+                const firstFailed = task.steps.findIndex((s) => s.status === 'failed');
+                if (firstFailed !== -1) {
+                    start = firstFailed;
+                }
+                else {
+                    const firstNonDone = task.steps.findIndex((s) => s.status !== 'done');
+                    if (firstNonDone === -1) {
+                        throw new TaskError('INVALID_STATE', 'Nothing to retry: all steps are done');
+                    }
+                    start = firstNonDone;
+                }
+            }
+            for (const s of task.steps) {
+                if (s.index >= start) {
+                    s.status = 'pending';
+                    delete s.error;
+                    delete s.output;
+                    delete s.startedAt;
+                    delete s.endedAt;
+                }
+            }
+            // Attempts are preserved; checkpoints at/after the retry point are dropped.
+            task.checkpoints = task.checkpoints.filter((c) => c.stepIndex < start);
+            task.currentStepIndex = start;
+            task.status = 'running';
+            delete task.error;
+            delete task.approval;
+            pendingTurnResults.delete(id);
+            const snapshot = await saveAndSnapshot(task);
+            emit({ type: 'task.resumed', taskId: id });
+            return snapshot;
+        });
+        void runLoop(id);
+        return snapshot;
+    }
+    async function recover() {
+        const tasks = store.list();
+        for (const task of tasks) {
+            if (task.status === 'running' ||
+                task.status === 'paused' ||
+                task.status === 'awaiting_approval') {
+                // Re-read under the lock: the boot-time list() snapshot may be
+                // stale if a loop is still settling in the same process.
+                await withTaskLock(task.id, async () => {
+                    const fresh = store.get(task.id);
+                    if (fresh === null)
+                        return;
+                    if (fresh.status !== 'running' &&
+                        fresh.status !== 'paused' &&
+                        fresh.status !== 'awaiting_approval') {
+                        return;
+                    }
+                    fresh.status = 'interrupted';
+                    delete fresh.approval;
+                    pendingTurnResults.delete(fresh.id);
+                    const current = fresh.steps[fresh.currentStepIndex];
+                    if (current !== undefined && current.status === 'running') {
+                        current.status = 'pending';
+                        delete current.startedAt;
+                    }
+                    await saveAndSnapshot(fresh);
+                });
+                // Release a same-process loop still blocked inside runTurn, if any.
+                // After a real restart there is no controller, so this is a no-op.
+                controllers.get(task.id)?.abort();
+                emit({ type: 'task.interrupted', taskId: task.id });
+            }
+        }
+    }
+    const runner = {
+        createAndStart,
+        get,
+        list,
+        pause,
+        resume,
+        cancel,
+        redirect,
+        retry,
+        recover,
+    };
+    return runner;
+}

package/dist/node_modules/@boombot/core/task-runner/store.d.ts

@@ -0,0 +1,28 @@
+import type { Task } from './types.js';
+/**
+ * TaskStore persists tasks to a single JSON file with atomic writes.
+ * `load()` MUST be called before any other method.
+ */
+export declare class TaskStore {
+    private readonly filePath;
+    private tasks;
+    private loadedFlag;
+    private persistQueue;
+    constructor(filePath: string);
+    /** Read the store file into memory. Must be called before any other method. */
+    load(): Promise<void>;
+    private requireLoaded;
+    /** Atomically persist the whole document: write tmp file, then rename. */
+    private persist;
+    /** Insert a new task. */
+    create(task: Task): Promise<void>;
+    /** Upsert a task (insert or replace by id). */
+    save(task: Task): Promise<void>;
+    get(id: string): Task | null;
+    list(): Task[];
+    remove(id: string): Promise<void>;
+}
+/** Factory for {@link TaskStore} (options-object style, matches gateway wiring). */
+export declare function createTaskStore(options: {
+    storagePath: string;
+}): TaskStore;