@mingchuno/agent-workflows 0.1.0

package/dist/src/runner.js

@@ -0,0 +1,370 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, realpath } from "node:fs/promises";
+import { resolve } from "node:path";
+import { DBOS } from "@dbos-inc/dbos-sdk";
+import { configSchema } from "./config.js";
+import { BlockedError, isBlockedError, } from "./domain.js";
+import { defaultWorkflow, Operations } from "./operations.js";
+import { createQueuedRun } from "./run-record.js";
+import { assertProcessesStopped, CheckoutOwnership, } from "./runtime/ownership.js";
+import { createRedactor, runtimeLogger } from "./runtime/redaction.js";
+import { Store } from "./store.js";
+import { ExistingCheckout } from "./workspace.js";
+export class Runner {
+    options;
+    store;
+    config;
+    controllers = new Map();
+    active = new Map();
+    hosting = new Map();
+    workflow;
+    ownership = new CheckoutOwnership();
+    stopping = false;
+    ownsRuntime = false;
+    tickBusy = false;
+    timer;
+    lastPoll = new Map();
+    polling = new Map();
+    constructor(options) {
+        this.options = options;
+        this.config = configSchema.parse(options.config);
+        this.store = new Store(options.databaseUrl, this.config.id, this.redact);
+    }
+    queue(id) {
+        return `${this.config.id}:${id}`;
+    }
+    async start() {
+        if (DBOS.isInitialized())
+            throw new Error("Only one Runner can own the DBOS runtime in a process");
+        if (process.platform === "win32")
+            throw new Error("Phase 1 requires macOS or Linux process-group ownership");
+        const canonical = new Set();
+        const ids = new Set();
+        for (const project of this.config.projects) {
+            project.checkout = await realpath(project.checkout);
+            if (canonical.has(project.checkout) || ids.has(project.id))
+                throw new Error("Duplicate project identity or canonical checkout");
+            canonical.add(project.checkout);
+            ids.add(project.id);
+            this.hosting.set(project.id, this.options.hosting(project));
+        }
+        for (const hosting of this.hosting.values())
+            await hosting.preflight?.();
+        await this.store.initialize();
+        await this.store.acquire([...canonical], () => {
+            this.stopping = true;
+            for (const controller of this.controllers.values())
+                controller.abort();
+        });
+        await assertProcessesStopped(resolve(this.config.stateDirectory));
+        for (const project of this.config.projects) {
+            await this.ownership.acquire(project.checkout, resolve(this.config.stateDirectory));
+            await this.store.registerProject(project.id);
+        }
+        await mkdir(resolve(this.config.stateDirectory), {
+            recursive: true,
+            mode: 0o700,
+        });
+        this.workflow = DBOS.registerWorkflow(async (runId) => this.execute(runId), { name: `${this.config.id}-issue-workflow` });
+        DBOS.setConfig({
+            name: `agent-workflows-${this.config.id}`,
+            systemDatabaseUrl: this.options.databaseUrl,
+            applicationVersion: `${this.config.id}-${this.options.workflowVersion ?? "phase1-v1"}`,
+            executorID: this.config.id,
+            listenQueues: this.config.projects.map((p) => this.queue(p.id)),
+            logger: runtimeLogger(this.redact),
+        });
+        this.ownsRuntime = true;
+        await DBOS.launch();
+        for (const project of this.config.projects)
+            await DBOS.registerQueue(this.queue(project.id), {
+                globalConcurrency: 1,
+                workerConcurrency: 1,
+                minPollingIntervalMs: 100,
+            });
+        this.timer = setInterval(() => {
+            void this.tick().catch((error) => this.store.emit(null, "runner-error", {
+                error: this.redact(String(error)),
+            }));
+        }, 100);
+        await this.tick();
+    }
+    redact = (text) => createRedactor([
+        this.options.databaseUrl,
+        ...this.config.projects.map((project) => process.env[project.hosting.tokenEnv] ?? ""),
+    ])(text);
+    async poll(projectId) {
+        await Promise.all(this.config.projects
+            .filter((p) => !projectId || p.id === projectId)
+            .map((project) => {
+            const pending = this.polling.get(project.id);
+            if (pending)
+                return pending;
+            if (this.stopping)
+                return Promise.resolve();
+            const polling = this.pollProject(project)
+                .catch(async (error) => {
+                await this.store.emit(null, "poll-error", {
+                    projectId: project.id,
+                    error: this.redact(String(error)),
+                });
+                throw error;
+            })
+                .finally(() => this.polling.delete(project.id));
+            this.polling.set(project.id, polling);
+            return polling;
+        }));
+    }
+    async pollProject(project) {
+        const state = await this.store.project(project.id);
+        if (state.paused || state.blocked || this.stopping)
+            return;
+        const hosting = this.hosting.get(project.id);
+        for (const issue of (await hosting.listIssues(project.labels)).sort((a, b) => a.number - b.number)) {
+            if (this.stopping)
+                return;
+            const now = new Date().toISOString();
+            const id = randomUUID();
+            const run = createQueuedRun({
+                id,
+                projectId: project.id,
+                checkout: project.checkout,
+                taskKey: `${hosting.identity}:${issue.id}`,
+                attempt: 1,
+                issue,
+                now,
+                branchTemplate: project.branchTemplate,
+            });
+            await this.store.insertRun(run);
+        }
+    }
+    async tick() {
+        if (this.tickBusy || this.stopping)
+            return;
+        this.tickBusy = true;
+        try {
+            await this.processCommands();
+            this.pollDueProjects();
+            await this.dispatchRuns();
+        }
+        finally {
+            this.tickBusy = false;
+        }
+    }
+    async processCommands() {
+        for (const request of (await this.store.commands()).filter((c) => c.status === "pending")) {
+            try {
+                if (request.kind === "pause")
+                    await this.pause(request.target);
+                else if (request.kind === "resume")
+                    await this.resume(request.target);
+                else if (request.kind === "stop")
+                    await this.stop(request.target);
+                else if (request.kind === "retry")
+                    await this.retry(request.target, request.id);
+                else
+                    throw new Error("Unknown command");
+                await this.store.finishCommand(request.id);
+            }
+            catch (error) {
+                await this.store.finishCommand(request.id, this.redact(String(error)));
+            }
+        }
+    }
+    pollDueProjects() {
+        if (this.stopping)
+            return;
+        for (const project of this.config.projects) {
+            if (!this.polling.has(project.id) &&
+                Date.now() - (this.lastPoll.get(project.id) ?? 0) >=
+                    project.pollIntervalMs) {
+                this.lastPoll.set(project.id, Date.now());
+                void this.poll(project.id).catch((error) => runtimeLogger(this.redact).error(String(error)));
+            }
+        }
+    }
+    async dispatchRuns() {
+        if (this.stopping)
+            return;
+        const runs = await this.store.runs();
+        for (const project of this.config.projects) {
+            const state = await this.store.project(project.id);
+            if (this.stopping)
+                return;
+            if (state.paused || state.blocked)
+                continue;
+            const run = runs.find((r) => r.projectId === project.id &&
+                (r.outcome === "queued" || r.outcome === "running"));
+            if (!run || this.active.has(run.id))
+                continue;
+            const handle = await DBOS.startWorkflow(this.workflow, {
+                workflowID: run.id,
+                queueName: this.queue(project.id),
+            })(run.id);
+            const result = handle
+                .getResult()
+                .catch(async (error) => {
+                await this.recordWorkflowFailure(run, error);
+            })
+                .finally(() => this.active.delete(run.id));
+            this.active.set(run.id, result);
+        }
+    }
+    async recordWorkflowFailure(run, error) {
+        const message = this.redact(String(error));
+        await this.store.emit(run.id, "workflow-error", { error: message });
+        const current = await this.store.run(run.id);
+        if (["queued", "running"].includes(current.outcome)) {
+            await this.store.patchRun(run.id, {
+                outcome: "blocked",
+                error: message,
+            });
+            await this.store.setProject(run.projectId, {
+                blocked: `DBOS execution failed for ${run.id}; inspect recovery evidence`,
+            });
+        }
+    }
+    async execute(runId) {
+        const controller = new AbortController();
+        if (this.stopping)
+            controller.abort();
+        this.controllers.set(runId, controller);
+        try {
+            await this.executeOwned(runId, controller);
+        }
+        finally {
+            this.controllers.delete(runId);
+        }
+    }
+    async executeOwned(runId, controller) {
+        const run = await DBOS.runStep(() => this.store.run(runId), {
+            name: "load-run",
+        });
+        const project = this.config.projects.find((p) => p.id === run.projectId);
+        if (!project)
+            throw new Error("Project removed from configuration");
+        const workspace = this.options.workspace ?? new ExistingCheckout();
+        const operations = new Operations(runId, {
+            store: this.store,
+            project,
+            workspace,
+            hosting: this.hosting.get(project.id),
+            agents: this.options.agents,
+            artifacts: resolve(this.config.stateDirectory),
+            signal: controller.signal,
+            redact: this.redact,
+        });
+        try {
+            while (true) {
+                const state = await DBOS.runStep(() => this.store.project(project.id), {
+                    name: "start-gate",
+                });
+                if (state.blocked)
+                    throw new BlockedError(state.blocked);
+                if (!state.paused)
+                    break;
+                controller.signal.throwIfAborted();
+                await DBOS.sleepms(200);
+            }
+            controller.signal.throwIfAborted();
+            await (this.options.workflow ?? defaultWorkflow)(operations);
+            await DBOS.runStep(async () => {
+                const completed = await this.store.run(runId);
+                if (["queued", "running"].includes(completed.outcome))
+                    throw new BlockedError("Custom workflow returned without a terminal outcome");
+            }, { name: "terminal-check" });
+        }
+        catch (error) {
+            await DBOS.runStep(async () => {
+                let unsafe = false;
+                try {
+                    await workspace.check(project);
+                }
+                catch {
+                    unsafe = true;
+                }
+                const outcome = controller.signal.aborted
+                    ? "cancelled"
+                    : isBlockedError(error)
+                        ? "blocked"
+                        : "failed";
+                await this.store.patchRun(runId, {
+                    outcome,
+                    error: this.redact(String(error)),
+                });
+                if (unsafe || isBlockedError(error))
+                    await this.store.setProject(project.id, {
+                        blocked: `Run ${runId} requires recovery: ${this.redact(String(error))}`,
+                    });
+            }, { name: "record-failure", retriesAllowed: false });
+        }
+        finally {
+            this.controllers.delete(runId);
+        }
+    }
+    async pause(projectId) {
+        await this.store.project(projectId);
+        await this.store.setProject(projectId, { paused: true });
+    }
+    async resume(projectId) {
+        const state = await this.store.project(projectId);
+        if (state.blocked)
+            throw new Error(state.blocked);
+        await this.store.setProject(projectId, { paused: false });
+    }
+    async stop(runId) {
+        const run = await this.store.run(runId);
+        const controller = this.controllers.get(runId);
+        if (controller) {
+            controller.abort();
+            while (this.controllers.has(runId))
+                await new Promise((resolve) => setTimeout(resolve, 20));
+            return;
+        }
+        if (run.outcome === "queued") {
+            await DBOS.cancelWorkflow(runId);
+            await this.store.patchRun(runId, { outcome: "cancelled" });
+            return;
+        }
+        if (run.outcome === "running")
+            throw new BlockedError("No local process ownership for this running workflow");
+    }
+    async retry(runId, commandId) {
+        return this.store.admitRetry(runId, {
+            commandId,
+            checkSafety: async (previous) => {
+                if (this.controllers.has(runId))
+                    throw new Error("Work has not stopped");
+                const project = this.config.projects.find((p) => p.id === previous.projectId);
+                if (!project)
+                    throw new Error("Project removed from configuration");
+                await assertProcessesStopped(resolve(this.config.stateDirectory, runId));
+                await (this.options.workspace ?? new ExistingCheckout()).check(project);
+                return {
+                    checkout: project.checkout,
+                    branchTemplate: project.branchTemplate,
+                };
+            },
+        });
+    }
+    async shutdown() {
+        this.stopping = true;
+        if (this.timer)
+            clearInterval(this.timer);
+        for (const controller of this.controllers.values())
+            controller.abort();
+        while (this.controllers.size || this.tickBusy)
+            await new Promise((resolve) => setTimeout(resolve, 20));
+        await Promise.all(this.active.values());
+        await Promise.allSettled(this.polling.values());
+        if (this.ownsRuntime) {
+            await DBOS.shutdown({
+                deregister: true,
+                workflowCompletionTimeoutMS: 5000,
+            });
+            this.ownsRuntime = false;
+        }
+        await this.ownership.release();
+        await this.store.close();
+    }
+}

package/dist/src/runtime/ownership.d.ts

@@ -0,0 +1,8 @@
+export declare function processExists(pid: number, group?: boolean): boolean;
+export declare function assertProcessesStopped(directory: string): Promise<void>;
+/** Filesystem lease prevents different databases/configurations owning one checkout. */
+export declare class CheckoutOwnership {
+    private readonly paths;
+    acquire(checkout: string, artifacts: string): Promise<void>;
+    release(): Promise<void>;
+}

package/dist/src/runtime/ownership.js

@@ -0,0 +1,84 @@
+import { open, readdir, readFile, unlink } from "node:fs/promises";
+import { join } from "node:path";
+import { BlockedError } from "../domain.js";
+import { command } from "./process.js";
+export function processExists(pid, group = false) {
+    try {
+        process.kill(group && process.platform !== "win32" ? -pid : pid, 0);
+        return true;
+    }
+    catch (error) {
+        return error.code === "EPERM";
+    }
+}
+export async function assertProcessesStopped(directory) {
+    let entries;
+    try {
+        entries = await readdir(directory, { withFileTypes: true });
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return;
+        throw error;
+    }
+    for (const entry of entries) {
+        const path = join(directory, entry.name);
+        if (entry.isDirectory())
+            await assertProcessesStopped(path);
+        else if (entry.name.endsWith(".process.json")) {
+            const record = JSON.parse(await readFile(path, "utf8"));
+            if (processExists(record.pid, true))
+                throw new BlockedError(`Previous process group ${record.pid} is still alive; inspect ${path} before recovery`);
+        }
+    }
+}
+/** Filesystem lease prevents different databases/configurations owning one checkout. */
+export class CheckoutOwnership {
+    paths = [];
+    async acquire(checkout, artifacts) {
+        const gitDirectory = (await command("git", ["rev-parse", "--absolute-git-dir"], {
+            cwd: checkout,
+        })).stdout.trim();
+        const path = join(gitDirectory, "agent-workflows-owner.json");
+        let guard;
+        try {
+            guard = await open(path + ".acquiring", "wx", 0o600);
+        }
+        catch (error) {
+            if (error.code === "EEXIST")
+                throw new BlockedError(`Checkout acquisition is locked: ${path}.acquiring; inspect its owner before removing a stale guard`);
+            throw error;
+        }
+        try {
+            await guard.writeFile(JSON.stringify({ pid: process.pid }));
+            await assertProcessesStopped(join(gitDirectory, "agent-workflows-processes"));
+            try {
+                const previous = JSON.parse(await readFile(path, "utf8"));
+                if (processExists(previous.pid))
+                    throw new BlockedError(`Checkout owned by process ${previous.pid}: ${checkout}`);
+                await assertProcessesStopped(previous.artifacts);
+                await unlink(path);
+            }
+            catch (error) {
+                if (error.code !== "ENOENT")
+                    throw error;
+            }
+            const file = await open(path, "wx", 0o600);
+            try {
+                await file.writeFile(JSON.stringify({ pid: process.pid, artifacts }));
+                this.paths.push(path);
+            }
+            finally {
+                await file.close();
+            }
+        }
+        finally {
+            await guard.close();
+            await unlink(path + ".acquiring");
+        }
+    }
+    async release() {
+        for (const path of this.paths.splice(0))
+            await unlink(path);
+    }
+}

package/dist/src/runtime/process.d.ts

@@ -0,0 +1,18 @@
+export interface CommandOptions {
+    cwd: string;
+    processFile?: string;
+    signal?: AbortSignal;
+    timeoutMs?: number;
+    killGraceMs?: number;
+    env?: NodeJS.ProcessEnv;
+    allowFailure?: boolean;
+    captureOutput?: boolean;
+    onOutput?: (chunk: string) => void;
+}
+export interface CommandResult {
+    stdout: string;
+    stderr: string;
+    exitCode: number;
+}
+/** Resolves only after the child closes. Cancellation terminates its process group. */
+export declare function command(executable: string, args: string[], options: CommandOptions): Promise<CommandResult>;

package/dist/src/runtime/process.js

@@ -0,0 +1,98 @@
+import { spawn } from "node:child_process";
+import { unlinkSync, writeFileSync } from "node:fs";
+/** Resolves only after the child closes. Cancellation terminates its process group. */
+export function command(executable, args, options) {
+    options.signal?.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const child = spawn(executable, args, {
+            cwd: options.cwd,
+            env: options.env ?? process.env,
+            detached: process.platform !== "win32",
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        let stdout = "", stderr = "", cancelled = false, killTimer;
+        let failure;
+        const kill = (signal) => {
+            if (!child.pid)
+                return;
+            try {
+                if (process.platform === "win32")
+                    child.kill(signal);
+                else
+                    process.kill(-child.pid, signal);
+            }
+            catch (error) {
+                if (error.code !== "ESRCH")
+                    failure = error;
+            }
+        };
+        const stop = () => {
+            cancelled = true;
+            kill("SIGTERM");
+            killTimer ??= setTimeout(() => kill("SIGKILL"), options.killGraceMs ?? 2000);
+        };
+        try {
+            if (options.processFile && child.pid)
+                writeFileSync(options.processFile, JSON.stringify({ pid: child.pid }), {
+                    mode: 0o600,
+                });
+        }
+        catch (error) {
+            failure = error;
+            stop();
+        }
+        const timeout = setTimeout(stop, options.timeoutMs ?? 300_000);
+        options.signal?.addEventListener("abort", stop, { once: true });
+        const collect = (target, chunk) => {
+            const value = chunk.toString();
+            try {
+                options.onOutput?.(value);
+            }
+            catch (error) {
+                failure = error;
+                stop();
+            }
+            if (options.captureOutput !== false &&
+                stdout.length + stderr.length <= 32 * 1024 * 1024) {
+                if (target === "stdout")
+                    stdout += value;
+                else
+                    stderr += value;
+                if (stdout.length + stderr.length > 32 * 1024 * 1024)
+                    stop();
+            }
+        };
+        child.stdout.on("data", (chunk) => collect("stdout", chunk));
+        child.stderr.on("data", (chunk) => collect("stderr", chunk));
+        const cleanup = () => {
+            clearTimeout(timeout);
+            if (killTimer)
+                clearTimeout(killTimer);
+            options.signal?.removeEventListener("abort", stop);
+        };
+        child.on("error", (error) => {
+            failure = error;
+        });
+        child.on("close", (code) => {
+            // Kill any descendants left behind by a command that exited before them.
+            kill("SIGKILL");
+            cleanup();
+            if (options.processFile) {
+                try {
+                    unlinkSync(options.processFile);
+                }
+                catch (error) {
+                    if (error.code !== "ENOENT")
+                        return reject(error);
+                }
+            }
+            if (failure)
+                return reject(failure);
+            if (cancelled)
+                return reject(new Error("Command cancelled or timed out; process group terminated"));
+            if (code !== 0 && !options.allowFailure)
+                return reject(new Error(`${executable} failed (${code}): ${stderr.trim()}`));
+            resolve({ stdout, stderr, exitCode: code ?? 1 });
+        });
+    });
+}

package/dist/src/runtime/redaction.d.ts

@@ -0,0 +1,8 @@
+export declare function createRedactor(extra?: string[]): (text: string) => string;
+export declare function redactValue(value: unknown, redact: (text: string) => string): unknown;
+export declare function runtimeLogger(redact: (text: string) => string): {
+    info: (entry: unknown) => void;
+    debug: (entry: unknown) => void;
+    warn: (entry: unknown) => void;
+    error: (entry: unknown) => void;
+};

package/dist/src/runtime/redaction.js

@@ -0,0 +1,33 @@
+import pino from "pino";
+export function createRedactor(extra = []) {
+    const secrets = [
+        ...extra,
+        ...Object.entries(process.env)
+            .filter(([name]) => /TOKEN|PASSWORD|SECRET|API_KEY|DATABASE_URL/.test(name))
+            .map(([, value]) => value),
+    ]
+        .filter((value) => !!value && value.length >= 4)
+        .sort((a, b) => b.length - a.length);
+    return (text) => secrets.reduce((result, secret) => result.split(secret).join("[REDACTED]"), text);
+}
+export function redactValue(value, redact) {
+    if (typeof value === "string")
+        return redact(value);
+    if (Array.isArray(value))
+        return value.map((item) => redactValue(item, redact));
+    if (value && typeof value === "object")
+        return Object.fromEntries(Object.entries(value).map(([key, item]) => [
+            key,
+            redactValue(item, redact),
+        ]));
+    return value;
+}
+export function runtimeLogger(redact) {
+    const logger = pino({ level: "warn" }, pino.destination(2));
+    return {
+        info: (entry) => logger.info(redact(String(entry))),
+        debug: (entry) => logger.debug(redact(String(entry))),
+        warn: (entry) => logger.warn(redact(String(entry))),
+        error: (entry) => logger.error(redact(String(entry))),
+    };
+}