cloving 0.3.20 → 1.1.0

package/dist/src/daemon/issue-queue.js

@@ -0,0 +1,254 @@
+import { randomUUID } from "node:crypto";
+import { readFile, rename, rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { setTimeout as delay } from "node:timers/promises";
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+import { runIdentity } from "../service/submission-handler.js";
+function savedHold(value) {
+    if (value === undefined)
+        return undefined;
+    const held = record(value);
+    if (typeof held?.["mark"] !== "string" ||
+        typeof held["startedAt"] !== "number" ||
+        !Number.isFinite(held["startedAt"]) ||
+        held["startedAt"] <= 0)
+        throw new Error("invalid-queue-commands");
+    return { mark: held["mark"], startedAt: held["startedAt"] };
+}
+function identifiers(value) {
+    if (!Array.isArray(value) ||
+        !value.every((id) => typeof id === "string" && id.length > 0))
+        throw new Error("invalid-queue-commands");
+    return value.map(String);
+}
+async function journal(directory) {
+    const file = join(directory, "queue-commands.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "[]";
+        throw error;
+    });
+    const value = JSON.parse(text);
+    const saved = record(value);
+    const commands = new Set(identifiers(Array.isArray(value) ? value : saved?.["commands"]));
+    const failures = Array.isArray(value) ? [] : identifiers(saved?.["failures"]);
+    if (failures.length > 3)
+        throw new Error("invalid-queue-commands");
+    const pending = {
+        commands,
+        failures,
+        retryCommandId: undefined,
+        selfUpdateHold: savedHold(saved?.["selfUpdateHold"]),
+        save: async () => {
+            await writeFile(`${file}.tmp`, JSON.stringify({
+                commands: [...commands],
+                failures,
+                selfUpdateHold: pending.selfUpdateHold,
+            }) + "\n", { mode: 0o600, flush: true });
+            await rename(`${file}.tmp`, file);
+        },
+    };
+    return pending;
+}
+function terminal(run) {
+    return ["completed", "failed", "cancelled"].includes(String(run["state"]));
+}
+async function runStatus(runtimeDirectory, commandId) {
+    const reply = await send(runtimeDirectory, {
+        operation: "status",
+        parameters: { runId: runIdentity(commandId) },
+    });
+    if (reply.kind === "error" && reply.error.code === "unknown-run")
+        return undefined;
+    if (reply.kind === "error")
+        throw new Error(reply.error.code);
+    const run = record(reply.result["run"]);
+    if (run === undefined)
+        throw new Error("invalid-run-status");
+    return run;
+}
+async function recover(settings, pending) {
+    for (const commandId of pending.commands) {
+        if (await runStatus(settings.runtimeDirectory, commandId))
+            continue;
+        pending.commands.delete(commandId);
+        await pending.save();
+    }
+}
+function handedOff(run) {
+    const outcome = record(run["outcome"]);
+    return (run["state"] === "completed" &&
+        outcome?.["kind"] === "handoff_requested" &&
+        outcome["issueUpdate"] !== "failed");
+}
+class QueueFailureLimit extends Error {
+}
+function checkFailureLimit(pending) {
+    if (pending.failures.length >= 3)
+        throw new QueueFailureLimit(`3 runs in a row ended without handoff: ${pending.failures.join(", ")}`);
+}
+async function completed(settings, pending) {
+    checkFailureLimit(pending);
+    let pendingRun = false;
+    const tracked = [...pending.commands].filter((commandId) => commandId !== pending.retryCommandId);
+    for (const commandId of tracked) {
+        const run = await runStatus(settings.runtimeDirectory, commandId);
+        if (run === undefined)
+            throw new Error("queue-run-missing");
+        if (!terminal(run)) {
+            pendingRun = true;
+            continue;
+        }
+        pending.commands.delete(commandId);
+        if (handedOff(run))
+            pending.failures.length = 0;
+        else
+            pending.failures.push(runIdentity(commandId));
+        await pending.save();
+        checkFailureLimit(pending);
+        if (!handedOff(run)) {
+            const outcome = record(run["outcome"]);
+            process.stderr.write(`Queue run ended ${String(outcome?.["kind"] ?? run["state"])}${outcome?.["issueUpdate"] === "failed" ? "; issueUpdate: failed" : ""}: ${runIdentity(commandId)}; continuing\n`);
+        }
+    }
+    return pendingRun;
+}
+async function active(runtimeDirectory) {
+    let after = "";
+    do {
+        const reply = await send(runtimeDirectory, {
+            operation: "status",
+            parameters: { after, limit: 500 },
+        });
+        if (reply.kind === "error")
+            throw new Error(reply.error.code);
+        const runs = reply.result["runs"];
+        if (!Array.isArray(runs))
+            throw new Error("invalid-run-list");
+        if (runs.some((value) => {
+            const run = record(value);
+            return run?.["workflowId"] === "develop-issue" && !terminal(run);
+        }))
+            return true;
+        const next = reply.result["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-run-page");
+        after = next;
+    } while (after !== null);
+    return false;
+}
+async function submit(settings, pending) {
+    const commandId = pending.retryCommandId ?? randomUUID();
+    pending.commands.add(commandId);
+    await pending.save();
+    const reply = await send(settings.runtimeDirectory, {
+        operation: "run",
+        parameters: {
+            workflowId: "develop-issue",
+            source: "ticket",
+            submission: { ticket: "next" },
+            commandId,
+        },
+    });
+    pending.retryCommandId = undefined;
+    if (reply.kind === "result")
+        return;
+    if ([
+        "resolver-timeout",
+        "next-ticket-not-available",
+        "ticket-not-available",
+    ].includes(reply.error.code)) {
+        pending.retryCommandId = commandId;
+        process.stderr.write(`Queue ${reply.error.code}: command ${commandId}; will retry\n`);
+        return;
+    }
+    if (!["no-ready-ticket", "workflow-active"].includes(reply.error.code))
+        throw new Error(reply.error.code);
+    pending.commands.delete(commandId);
+    await pending.save();
+}
+function updateHold(directory, pending) {
+    let notice;
+    return async () => {
+        const mark = await readFile(join(directory, "pending-self-update"), "utf8").catch((error) => {
+            if (error instanceof Error && "code" in error && error.code === "ENOENT")
+                return undefined;
+            throw error;
+        });
+        if (mark === undefined && pending.selfUpdateHold !== undefined) {
+            pending.selfUpdateHold = undefined;
+            await pending.save();
+        }
+        if (mark === undefined) {
+            notice = undefined;
+            return false;
+        }
+        const timestamp = Number(mark);
+        if (!Number.isFinite(timestamp) || timestamp <= 0)
+            throw new Error("invalid-pending-self-update");
+        if (pending.selfUpdateHold?.mark !== mark) {
+            pending.selfUpdateHold = { mark, startedAt: Date.now() };
+            await pending.save();
+        }
+        const holding = Date.now() - pending.selfUpdateHold.startedAt < 30 * 60 * 1000;
+        const line = holding
+            ? "Queue holding for pending self-update\n"
+            : "Queue continuing after pending self-update time limit\n";
+        const key = mark + line;
+        if (notice !== key)
+            process.stderr.write(line);
+        notice = key;
+        return holding;
+    };
+}
+async function admission(settings, attempt) {
+    const file = join(settings.directory, "queue-submission");
+    await writeFile(file, "", { mode: 0o600 });
+    try {
+        if (!(await attempt.holding()) && !attempt.signal.aborted)
+            await submit(settings, attempt.pending);
+    }
+    finally {
+        await rm(file, { force: true });
+    }
+}
+async function cycle(settings, signal) {
+    const pending = await journal(settings.directory);
+    if (settings.resetQueue === true) {
+        pending.failures.length = 0;
+        await pending.save();
+    }
+    const holding = updateHold(settings.directory, pending);
+    await rm(join(settings.directory, "queue-submission"), { force: true });
+    await recover(settings, pending);
+    const stopped = () => signal.aborted;
+    while (!stopped()) {
+        await delay(settings.intervalMilliseconds, undefined, { signal }).catch(() => undefined);
+        if (stopped())
+            return;
+        if (await completed(settings, pending))
+            continue;
+        if (await active(settings.runtimeDirectory))
+            continue;
+        if (await holding())
+            continue;
+        if (stopped())
+            return;
+        await admission(settings, { pending, holding, signal });
+    }
+}
+export function issueQueue(settings) {
+    const stopping = new AbortController();
+    const running = cycle(settings, stopping.signal).catch((error) => {
+        const reason = error instanceof Error ? error.message : String(error);
+        const diagnostic = error instanceof QueueFailureLimit ? reason : JSON.stringify(reason);
+        process.stderr.write(`Queue stopped: ${diagnostic}\n`);
+    });
+    return {
+        stop: async () => {
+            stopping.abort();
+            await running;
+        },
+    };
+}

package/dist/src/daemon/notify-message.d.ts

@@ -0,0 +1,14 @@
+import { type Fields } from "../protocol.ts";
+export interface Notification {
+    readonly event: "decision" | "blocked" | "failed";
+    readonly runId: string;
+    readonly workflowId: string;
+    readonly subject: string;
+    readonly body: string;
+}
+export interface NotifyEvent {
+    readonly key: string;
+    readonly message: Notification;
+}
+export declare function decisionNotification(run: Fields, decision: Fields): NotifyEvent;
+export declare function outcomeNotification(run: Fields): NotifyEvent | undefined;

package/dist/src/daemon/notify-message.js

@@ -0,0 +1,48 @@
+import { decisionLines, runStatus } from "../cli/render.js";
+import { record } from "../protocol.js";
+function body(text, commands) {
+    const tail = commands.join("\n");
+    const available = 4000 - tail.length - 1;
+    const shortened = text.length <= available ? text : `${text.slice(0, available - 1)}…`;
+    return `${shortened}\n${tail}`;
+}
+function message(run, content) {
+    const workflowId = String(run["workflowId"]);
+    const action = content.event === "decision" ? "waits for your decision" : content.event;
+    return {
+        ...content,
+        runId: String(run["runId"]),
+        workflowId,
+        subject: `Cloving: ${workflowId} ${action}`.replace(/[\r\n]/g, " "),
+    };
+}
+export function decisionNotification(run, decision) {
+    const lines = decisionLines(decision);
+    const allowed = decision["allowed"];
+    const count = Array.isArray(allowed) ? allowed.length : 0;
+    const split = lines.length - count;
+    return {
+        key: `decision:${String(decision["decisionId"])}`,
+        message: message(run, {
+            event: "decision",
+            body: body(lines.slice(0, split).join("\n"), lines.slice(split)),
+        }),
+    };
+}
+export function outcomeNotification(run) {
+    const outcome = record(run["outcome"]);
+    const blocked = run["state"] === "completed" && outcome?.["kind"] === "blocked";
+    if (!blocked && run["state"] !== "failed")
+        return undefined;
+    const event = blocked ? "blocked" : "failed";
+    const lines = runStatus({ run });
+    return {
+        key: `${event}:${String(run["runId"])}`,
+        message: message(run, {
+            event,
+            body: body(String(lines[blocked ? 3 : 5]), [
+                `cloving status ${String(run["runId"])}`,
+            ]),
+        }),
+    };
+}

package/dist/src/daemon/notify.d.ts

@@ -0,0 +1,12 @@
+import { type Notification } from "./notify-message.ts";
+interface NotifySettings {
+    readonly directory: string;
+    readonly runtimeDirectory: string;
+    readonly command: readonly string[];
+    readonly intervalMilliseconds?: number;
+}
+export declare function notifyCommand(command: readonly string[], notification: Notification): Promise<void>;
+export declare function notifyRuns(settings: NotifySettings): Promise<{
+    readonly stop: () => Promise<void>;
+}>;
+export {};

package/dist/src/daemon/notify.js

@@ -0,0 +1,161 @@
+import { spawn } from "node:child_process";
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+import { decisionNotification, outcomeNotification, } from "./notify-message.js";
+async function request(runtimeDirectory, request) {
+    const reply = await send(runtimeDirectory, request);
+    if (reply.kind === "error")
+        throw new Error(reply.error.code);
+    return reply.result;
+}
+function rows(value) {
+    if (!Array.isArray(value))
+        throw new Error("invalid-notify-list");
+    return value.map((item) => {
+        const row = record(item);
+        if (row === undefined)
+            throw new Error("invalid-notify-row");
+        return row;
+    });
+}
+async function observedOutcome(summary, source) {
+    const { runtimeDirectory, cache } = source;
+    const latestSequence = summary["latestSequence"];
+    if (typeof latestSequence !== "number")
+        throw new Error("invalid-notify-sequence");
+    const runId = String(summary["runId"]);
+    const previous = cache.get(runId);
+    if (previous !== undefined && previous.latestSequence === latestSequence)
+        return previous.event;
+    const status = await request(runtimeDirectory, {
+        operation: "status",
+        parameters: { runId },
+    });
+    const run = record(status["run"]);
+    if (run === undefined)
+        throw new Error("invalid-notify-status");
+    const event = outcomeNotification(run);
+    cache.set(runId, { latestSequence, event });
+    return event;
+}
+async function events(runtimeDirectory, cache) {
+    const found = [];
+    const runs = new Map();
+    let after = "";
+    do {
+        const page = await request(runtimeDirectory, {
+            operation: "status",
+            parameters: { after, limit: 500 },
+        });
+        for (const summary of rows(page["runs"])) {
+            runs.set(String(summary["runId"]), summary);
+            const outcome = await observedOutcome(summary, {
+                runtimeDirectory,
+                cache,
+            });
+            if (outcome !== undefined)
+                found.push(outcome);
+        }
+        const next = page["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-notify-page");
+        after = next;
+    } while (after !== null);
+    const questions = await request(runtimeDirectory, {
+        operation: "questions",
+        parameters: {},
+    });
+    for (const decision of rows(questions["decisions"])) {
+        const run = runs.get(String(decision["runId"]));
+        if (run !== undefined)
+            found.push(decisionNotification(run, decision));
+    }
+    return found;
+}
+async function recorded(file) {
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+    if (text === undefined)
+        return undefined;
+    const value = JSON.parse(text);
+    if (!Array.isArray(value) || !value.every((key) => typeof key === "string"))
+        throw new Error("invalid-notify-record");
+    return new Set(value.map(String));
+}
+async function save(file, keys) {
+    await writeFile(`${file}.tmp`, JSON.stringify([...keys]) + "\n", {
+        mode: 0o600,
+        flush: true,
+    });
+    await rename(`${file}.tmp`, file);
+}
+export function notifyCommand(command, notification) {
+    return new Promise((resolve, reject) => {
+        const [executable = "", ...argumentsList] = command;
+        const child = spawn(executable, argumentsList, {
+            stdio: ["pipe", "ignore", "ignore"],
+            timeout: 30_000,
+            killSignal: "SIGKILL",
+        });
+        child.once("error", reject);
+        child.stdin.on("error", reject);
+        child.once("close", (code, signal) => {
+            if (code === 0)
+                resolve();
+            else
+                reject(new Error(signal === null ? `exit code ${String(code)}` : `signal ${signal}`));
+        });
+        child.stdin.end(JSON.stringify(notification) + "\n");
+    });
+}
+function report(label, error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    process.stderr.write(`${label}: ${JSON.stringify(reason)}\n`);
+}
+async function check(settings, journal) {
+    for (const event of await events(settings.runtimeDirectory, journal.cache)) {
+        if (journal.keys.has(event.key))
+            continue;
+        await save(journal.file, new Set([...journal.keys, event.key]));
+        journal.keys.add(event.key);
+        await notifyCommand(settings.command, event.message).catch((error) => {
+            report("Notify command failed", error);
+        });
+    }
+}
+export async function notifyRuns(settings) {
+    const file = join(settings.directory, "notify-events.json");
+    const keys = await recorded(file);
+    const cache = new Map();
+    const journal = { file, keys: keys ?? new Set(), cache };
+    if (keys === undefined) {
+        for (const event of await events(settings.runtimeDirectory, cache))
+            journal.keys.add(event.key);
+        await save(file, journal.keys);
+    }
+    let busy = false;
+    let running = Promise.resolve();
+    const timer = setInterval(() => {
+        if (busy)
+            return;
+        busy = true;
+        running = check(settings, journal)
+            .catch((error) => {
+            report("Notify check failed", error);
+        })
+            .finally(() => {
+            busy = false;
+        });
+    }, settings.intervalMilliseconds ?? 15_000);
+    return {
+        stop: async () => {
+            clearInterval(timer);
+            await running;
+        },
+    };
+}

package/dist/src/daemon/operation-journal.d.ts

@@ -0,0 +1,43 @@
+import type { AdapterOutcome, ProtectedIntent } from "../broker/actor/delivery-intents.ts";
+export interface WorkerReport {
+    readonly runId: string;
+    readonly commandId: string;
+    readonly operationId: string;
+    readonly inputDigest: string;
+    readonly leaseGeneration: number;
+    readonly worker: string;
+    readonly outcome: "completed" | "failed" | "stopped";
+    readonly result?: string;
+    readonly reason?: string;
+}
+export interface Recorded {
+    readonly outcome: WorkerReport["outcome"];
+    readonly result?: string;
+    readonly reason?: string;
+    readonly detail?: string;
+    readonly endedAt: string;
+}
+export interface Located {
+    readonly key: string;
+    readonly folder: string;
+}
+export type Reported = Pick<ProtectedIntent, "runId" | "operationId" | "inputDigest" | "leaseGeneration">;
+export interface Identity {
+    readonly receipt: string;
+    readonly worker: string;
+}
+export declare function operationKey(text: string): string;
+export declare function located(journal: string, operationId: string): Located;
+export declare function readRecord(folder: string, name: string): Promise<unknown>;
+export declare function writeRecord(folder: string, record: {
+    readonly name: string;
+    readonly value: unknown;
+    readonly permissions?: "private";
+}): Promise<void>;
+export declare function replied(identity: Identity, recorded: unknown): AdapterOutcome;
+export declare function interrupted(reason: string): Recorded;
+export declare function reportOf(operation: {
+    readonly intent: Reported;
+    readonly commandId: string;
+    readonly worker: string;
+}, recorded: Recorded): WorkerReport;

package/dist/src/daemon/operation-journal.js

@@ -0,0 +1,55 @@
+import { createHash } from "node:crypto";
+import { chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+export function operationKey(text) {
+    return createHash("sha256").update(text).digest("hex").slice(0, 32);
+}
+export function located(journal, operationId) {
+    const key = operationKey(operationId);
+    return { key, folder: join(journal, key) };
+}
+export async function readRecord(folder, name) {
+    const text = await readFile(join(folder, name), "utf8").catch(() => undefined);
+    const value = text === undefined ? undefined : JSON.parse(text);
+    return value;
+}
+export async function writeRecord(folder, record) {
+    const path = join(folder, record.name);
+    await mkdir(folder, { recursive: true });
+    if (record.permissions === "private")
+        await chmod(folder, 0o700);
+    await writeFile(`${path}.staged`, JSON.stringify(record.value, null, 2), record.permissions === "private" ? { mode: 0o600 } : {});
+    await rename(`${path}.staged`, path);
+}
+export function replied(identity, recorded) {
+    const field = (name) => Reflect.get(Object(recorded), name);
+    if (field("outcome") === "completed")
+        return { kind: "completed", ...identity, result: String(field("result")) };
+    if (field("outcome") === "stopped")
+        return { kind: "stopped", ...identity };
+    const result = field("result");
+    return {
+        kind: "failed",
+        ...identity,
+        reason: String(field("reason")),
+        ...(typeof result === "string" ? { result } : {}),
+    };
+}
+export function interrupted(reason) {
+    return { outcome: "failed", reason, endedAt: new Date().toISOString() };
+}
+export function reportOf(operation, recorded) {
+    const { intent, commandId, worker } = operation;
+    const { outcome, result, reason } = recorded;
+    return {
+        runId: intent.runId,
+        commandId,
+        operationId: intent.operationId,
+        inputDigest: intent.inputDigest,
+        leaseGeneration: intent.leaseGeneration,
+        worker,
+        outcome,
+        ...(result === undefined ? {} : { result }),
+        ...(reason === undefined ? {} : { reason }),
+    };
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -0,0 +1,9 @@
+import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
+export interface RetrySettings {
+    readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
+}
+interface RetryOptions extends RetrySettings {
+    readonly metadata: Readonly<Record<string, unknown>>;
+}
+export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;
+export {};