cloving 1.1.0 → 1.3.0

package/dist/src/daemon/notify.js

@@ -1,9 +1,11 @@
+import { recordedNotifications, saveNotifications, } from "./notify-journal.js";
 import { spawn } from "node:child_process";
-import { readFile, rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { send } from "../cli/client.js";
+import { deliveryBackoffMilliseconds } from "../delivery-backoff.js";
 import { record } from "../protocol.js";
 import { decisionNotification, outcomeNotification, } from "./notify-message.js";
+const retentionMilliseconds = 7 * 24 * 60 * 60_000;
 async function request(runtimeDirectory, request) {
     const reply = await send(runtimeDirectory, request);
     if (reply.kind === "error")
@@ -20,6 +22,20 @@ function rows(value) {
         return row;
     });
 }
+async function flushNotifications(journal) {
+    if (journal.unsaved.size === 0)
+        return;
+    await saveNotifications(journal.file, journal);
+    for (const entry of journal.unsaved.values())
+        if (entry.state === "failed" && entry.attempts === 5)
+            process.stderr.write(`Notify delivery failed: ${entry.key}\n`);
+    journal.unsaved.clear();
+}
+async function finishDelivery(journal, entry) {
+    journal.entries.set(entry.key, entry);
+    journal.unsaved.set(entry.key, entry);
+    await flushNotifications(journal);
+}
 async function observedOutcome(summary, source) {
     const { runtimeDirectory, cache } = source;
     const latestSequence = summary["latestSequence"];
@@ -74,26 +90,6 @@ async function events(runtimeDirectory, cache) {
     }
     return found;
 }
-async function recorded(file) {
-    const text = await readFile(file, "utf8").catch((error) => {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return undefined;
-        throw error;
-    });
-    if (text === undefined)
-        return undefined;
-    const value = JSON.parse(text);
-    if (!Array.isArray(value) || !value.every((key) => typeof key === "string"))
-        throw new Error("invalid-notify-record");
-    return new Set(value.map(String));
-}
-async function save(file, keys) {
-    await writeFile(`${file}.tmp`, JSON.stringify([...keys]) + "\n", {
-        mode: 0o600,
-        flush: true,
-    });
-    await rename(`${file}.tmp`, file);
-}
 export function notifyCommand(command, notification) {
     return new Promise((resolve, reject) => {
         const [executable = "", ...argumentsList] = command;
@@ -117,45 +113,189 @@ function report(label, error) {
     const reason = error instanceof Error ? error.message : String(error);
     process.stderr.write(`${label}: ${JSON.stringify(reason)}\n`);
 }
+function failureReason(error) {
+    if (!(error instanceof Error))
+        return "command error";
+    if ("code" in error && (error.code === "ENOENT" || error.code === "EACCES"))
+        return `spawn ${error.code}`;
+    return /^(?:exit code (?:\d+|null)|signal SIG[A-Z0-9]+)$/.test(error.message)
+        ? error.message
+        : "command error";
+}
+async function prepareDelivery(journal, entry) {
+    if (entry.message === null || entry.state === "sent")
+        return undefined;
+    if (entry.attempts === 5 && entry.state === "pending")
+        await finishDelivery(journal, {
+            ...entry,
+            state: "failed",
+            lastFailure: "attempt interrupted",
+        });
+    if (entry.attempts === 5)
+        return undefined;
+    const backoff = deliveryBackoffMilliseconds[entry.attempts] ?? 0;
+    if (entry.lastAttemptAt !== null &&
+        Date.now() - entry.lastAttemptAt < backoff)
+        return undefined;
+    const pending = {
+        ...entry,
+        message: entry.message,
+        state: "pending",
+        attempts: entry.attempts + 1,
+        lastAttemptAt: Date.now(),
+    };
+    const next = new Map(journal.entries).set(entry.key, pending);
+    await saveNotifications(journal.file, { ...journal, entries: next });
+    journal.entries.set(entry.key, pending);
+    return pending;
+}
+async function deliver(settings, entry) {
+    try {
+        await notifyCommand(settings.command, entry.message);
+        return { ...entry, state: "sent" };
+    }
+    catch (error) {
+        return {
+            ...entry,
+            state: "failed",
+            lastFailure: failureReason(error),
+        };
+    }
+}
+async function settleDelivery(journal, progress) {
+    if (progress.completed !== undefined) {
+        const entry = progress.completed;
+        progress.completed = undefined;
+        await finishDelivery(journal, entry);
+    }
+    await flushNotifications(journal);
+}
+async function scheduleDelivery(settings, delivery) {
+    const { journal, progress } = delivery;
+    const stopped = () => progress.stopping.aborted;
+    if (stopped() ||
+        progress.active !== undefined ||
+        progress.completed !== undefined ||
+        journal.unsaved.size > 0)
+        return;
+    for (const entry of journal.entries.values()) {
+        if (stopped())
+            return;
+        const pending = await prepareDelivery(journal, entry);
+        if (pending === undefined)
+            continue;
+        if (stopped()) {
+            await finishDelivery(journal, entry);
+            return;
+        }
+        progress.active = deliver(settings, pending).then((completed) => {
+            progress.completed = completed;
+            progress.active = undefined;
+            delivery.ready();
+        });
+        return;
+    }
+}
 async function check(settings, journal) {
+    const retained = new Map([...journal.entries].filter(([, entry]) => entry.state === "pending" ||
+        (entry.state === "failed" && entry.attempts < 5) ||
+        entry.lastAttemptAt === null ||
+        entry.lastAttemptAt >= Date.now() - retentionMilliseconds));
+    const removed = [...journal.entries.keys()].filter((key) => !retained.has(key));
+    if (removed.length > 0) {
+        const retiredKeys = new Set([...journal.retiredKeys, ...removed]);
+        await saveNotifications(journal.file, { entries: retained, retiredKeys });
+        for (const key of removed) {
+            journal.entries.delete(key);
+            journal.retiredKeys.add(key);
+        }
+    }
     for (const event of await events(settings.runtimeDirectory, journal.cache)) {
-        if (journal.keys.has(event.key))
+        if (journal.entries.has(event.key) || journal.retiredKeys.has(event.key))
             continue;
-        await save(journal.file, new Set([...journal.keys, event.key]));
-        journal.keys.add(event.key);
-        await notifyCommand(settings.command, event.message).catch((error) => {
-            report("Notify command failed", error);
-        });
+        const entry = {
+            ...event,
+            state: "pending",
+            attempts: 0,
+            lastAttemptAt: null,
+            lastFailure: null,
+        };
+        journal.entries.set(event.key, entry);
+        journal.unsaved.set(event.key, entry);
     }
+    await flushNotifications(journal);
 }
 export async function notifyRuns(settings) {
     const file = join(settings.directory, "notify-events.json");
-    const keys = await recorded(file);
+    const recorded = await recordedNotifications(file);
     const cache = new Map();
-    const journal = { file, keys: keys ?? new Set(), cache };
-    if (keys === undefined) {
+    const journal = {
+        file,
+        entries: recorded?.entries ?? new Map(),
+        retiredKeys: recorded?.retiredKeys ?? new Set(),
+        cache,
+        unsaved: new Map(),
+    };
+    if (recorded === undefined) {
         for (const event of await events(settings.runtimeDirectory, cache))
-            journal.keys.add(event.key);
-        await save(file, journal.keys);
+            journal.entries.set(event.key, {
+                ...event,
+                state: "sent",
+                attempts: 0,
+                lastAttemptAt: Date.now(),
+                lastFailure: null,
+            });
     }
+    await saveNotifications(file, journal);
+    return notificationLoop(settings, journal);
+}
+function notificationLoop(settings, journal) {
     let busy = false;
     let running = Promise.resolve();
+    const stopping = new AbortController();
+    const progress = {
+        stopping: stopping.signal,
+        active: undefined,
+        completed: undefined,
+    };
+    const ready = () => {
+        if (stopping.signal.aborted)
+            return;
+        running = running
+            .then(() => settleDelivery(journal, progress))
+            .then(() => scheduleDelivery(settings, { journal, progress, ready }))
+            .catch((error) => {
+            report("Notify delivery failed", error);
+        });
+    };
     const timer = setInterval(() => {
         if (busy)
             return;
         busy = true;
-        running = check(settings, journal)
+        running = running
+            .then(() => settleDelivery(journal, progress))
+            .then(() => check(settings, journal))
             .catch((error) => {
             report("Notify check failed", error);
-        })
-            .finally(() => {
+        });
+        ready();
+        running = running.finally(() => {
             busy = false;
         });
     }, settings.intervalMilliseconds ?? 15_000);
     return {
-        stop: async () => {
+        stop: () => {
+            stopping.abort();
             clearInterval(timer);
-            await running;
+            running = running
+                .then(async () => {
+                await progress.active;
+                await settleDelivery(journal, progress);
+            })
+                .catch((error) => {
+                report("Notify delivery failed", error);
+            });
+            return running;
         },
     };
 }

package/dist/src/daemon/opencode-stream.d.ts

@@ -0,0 +1,5 @@
+export declare function opencodeStream(): {
+    line: (line: string) => readonly string[];
+    stderr: (chunk: string) => void;
+    result: (exitCode: number | null) => string;
+};

package/dist/src/daemon/opencode-stream.js

@@ -0,0 +1,87 @@
+import { record } from "../protocol.js";
+import { sumUsage, usageOf, tokenCount, } from "../worker-usage.js";
+function parsed(line) {
+    try {
+        return record(JSON.parse(line)) ?? {};
+    }
+    catch {
+        return {};
+    }
+}
+function tokens(value) {
+    const counts = record(value) ?? {};
+    const cache = record(counts["cache"]) ?? {};
+    const output = tokenCount(counts["output"]);
+    const reasoning = tokenCount(counts["reasoning"]);
+    return usageOf({
+        input_tokens: counts["input"],
+        cache_read_input_tokens: cache["read"],
+        output_tokens: output === null || reasoning === null ? null : output + reasoning,
+    });
+}
+function errorMessage(value) {
+    const error = record(value) ?? {};
+    const data = record(error["data"]) ?? {};
+    const message = data["message"] ?? error["name"];
+    return typeof message === "string" ? message : "opencode error";
+}
+function translated(line, state) {
+    const event = parsed(line);
+    const part = record(event["part"]) ?? {};
+    if (typeof event["sessionID"] === "string")
+        state.sessionId = event["sessionID"];
+    if (typeof event["model"] === "string")
+        state.model = event["model"];
+    if (event["type"] === "error")
+        state.error = errorMessage(event["error"]);
+    if (event["type"] === "step_finish") {
+        state.steps.push(tokens(part["tokens"]));
+        state.reason = part["reason"];
+    }
+    if (event["type"] !== "text" || typeof part["text"] !== "string")
+        return [];
+    state.message = part["text"];
+    return [
+        JSON.stringify({
+            type: "assistant",
+            message: { content: [{ type: "text", text: state.message }] },
+        }),
+    ];
+}
+function result(exitCode, state) {
+    const success = exitCode === 0 && state.error === null && state.reason === "stop";
+    const failure = exitCode === 0
+        ? "opencode ended without a final stop step"
+        : `opencode ended with the code ${String(exitCode)}`;
+    return JSON.stringify({
+        type: "result",
+        subtype: success ? "success" : "error_during_execution",
+        is_error: !success,
+        result: success
+            ? state.message
+            : state.error || state.stderr.trim() || failure,
+        total_cost_usd: null,
+        model: state.model,
+        usage: sumUsage(state.steps),
+        session_id: state.sessionId,
+        num_turns: state.steps.length,
+    });
+}
+export function opencodeStream() {
+    const state = {
+        steps: [],
+        message: "",
+        sessionId: null,
+        model: null,
+        reason: null,
+        error: null,
+        stderr: "",
+    };
+    return {
+        line: (line) => translated(line, state),
+        stderr: (chunk) => {
+            state.stderr = (state.stderr + chunk).slice(-16_000);
+        },
+        result: (exitCode) => result(exitCode, state),
+    };
+}

package/dist/src/daemon/queue-status.d.ts

@@ -0,0 +1,4 @@
+export declare function activeQueueRuns(settings: {
+    readonly runtimeDirectory: string;
+    readonly workflowId?: string;
+}): Promise<number>;

package/dist/src/daemon/queue-status.js

@@ -0,0 +1,38 @@
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+function queueBusy(value) {
+    const busy = record(value)?.["busy"];
+    if (typeof busy !== "number")
+        return undefined;
+    if (!Number.isSafeInteger(busy) || busy < 0)
+        throw new Error("invalid-queue-status");
+    return busy;
+}
+export async function activeQueueRuns(settings) {
+    let count = 0;
+    let after = "";
+    do {
+        const reply = await send(settings.runtimeDirectory, {
+            operation: "status",
+            parameters: { after, limit: 500 },
+        });
+        if (reply.kind === "error")
+            throw new Error(reply.error.code);
+        const busy = queueBusy(reply.result["queue"]);
+        if (busy !== undefined)
+            return busy;
+        const runs = reply.result["runs"];
+        if (!Array.isArray(runs))
+            throw new Error("invalid-run-list");
+        count += runs.filter((value) => {
+            const run = record(value);
+            return (run?.["workflowId"] === (settings.workflowId ?? "develop-issue") &&
+                !["completed", "failed", "cancelled"].includes(String(run["state"])));
+        }).length;
+        const next = reply.result["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-run-page");
+        after = next;
+    } while (after !== null);
+    return count;
+}

package/dist/src/daemon/retry-worktree.d.ts

@@ -0,0 +1,18 @@
+export interface RetryTree {
+    readonly directory: string;
+    readonly signal: AbortSignal;
+}
+interface GitResult {
+    readonly text: string;
+    readonly code: number | null;
+}
+export declare function retryGit(tree: RetryTree, parts: readonly string[]): Promise<GitResult>;
+export declare function turnHead(tree: RetryTree): Promise<string | null>;
+export declare function retryEvidence(tree: RetryTree, startCommit: string | null): Promise<string>;
+export interface IndexFailure {
+    readonly reason: "worktree-index-unknown" | "worktree-index-locked";
+    readonly lockPath?: string;
+    readonly fix: string;
+}
+export declare function indexFailure(tree: RetryTree): Promise<IndexFailure | null>;
+export {};

package/dist/src/daemon/retry-worktree.js

@@ -0,0 +1,139 @@
+import { spawn } from "node:child_process";
+import { lstat } from "node:fs/promises";
+import { resolve } from "node:path";
+function kill(child) {
+    if (child.pid === undefined)
+        return;
+    try {
+        process.kill(-child.pid, "SIGKILL");
+    }
+    catch {
+        child.kill("SIGKILL");
+    }
+}
+export async function retryGit(tree, parts) {
+    if (tree.signal.aborted)
+        return { text: "", code: null };
+    const child = spawn("git", ["-C", tree.directory, ...parts], {
+        stdio: ["ignore", "pipe", "ignore"],
+        detached: true,
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+    });
+    const closed = Promise.withResolvers();
+    let text = "";
+    const stop = () => {
+        kill(child);
+        child.stdout.destroy();
+        closed.resolve(null);
+    };
+    const timer = setTimeout(stop, 30_000);
+    tree.signal.addEventListener("abort", stop, { once: true });
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        text += chunk;
+        if (Buffer.byteLength(text) > 1_048_576)
+            stop();
+    });
+    child.once("error", () => {
+        closed.resolve(null);
+    });
+    child.once("close", closed.resolve);
+    const code = await closed.promise;
+    clearTimeout(timer);
+    tree.signal.removeEventListener("abort", stop);
+    return { text: text.trim(), code };
+}
+export async function turnHead(tree) {
+    const head = await retryGit(tree, ["rev-parse", "--verify", "HEAD"]);
+    return head.code === 0 && /^[a-f0-9]{40,64}$/.test(head.text)
+        ? head.text
+        : null;
+}
+async function remoteState(tree) {
+    const branch = await retryGit(tree, ["symbolic-ref", "--short", "HEAD"]);
+    if (branch.code !== 0 || tree.signal.aborted)
+        return "unknown";
+    const reference = `refs/heads/${branch.text}`;
+    const remote = await retryGit(tree, [
+        "ls-remote",
+        "--exit-code",
+        "--heads",
+        "origin",
+        reference,
+    ]);
+    if (remote.code === 2)
+        return "not pushed";
+    if (remote.code !== 0)
+        return "unknown";
+    const tracking = `refs/remotes/origin/${branch.text}`;
+    const fetched = await retryGit(tree, [
+        "fetch",
+        "--no-tags",
+        "origin",
+        `+${reference}:${tracking}`,
+    ]);
+    return fetched.code === 0 ? tracking : "unknown";
+}
+async function pushState(tree, evidence) {
+    const { commit, remote } = evidence;
+    if (!remote.startsWith("refs/"))
+        return remote;
+    const ancestor = await retryGit(tree, [
+        "merge-base",
+        "--is-ancestor",
+        commit,
+        remote,
+    ]);
+    if (ancestor.code === 1)
+        return "not pushed";
+    return ancestor.code === 0 ? "pushed" : "unknown";
+}
+export async function retryEvidence(tree, startCommit) {
+    const remote = await remoteState(tree);
+    if (startCommit === null || tree.signal.aborted)
+        return "unknown: commits since the start of this turn could not be determined.";
+    const commits = await retryGit(tree, [
+        "log",
+        "--format=%H %s",
+        `${startCommit}..HEAD`,
+        "--",
+    ]);
+    if (commits.code !== 0)
+        return "unknown: commits since the start of this turn could not be listed.";
+    const lines = [];
+    for (const line of commits.text.split("\n").filter(Boolean)) {
+        const commit = line.split(" ")[0] ?? "";
+        const state = await pushState(tree, { commit, remote });
+        lines.push(`${commit} ${state}${line.slice(commit.length)}`);
+    }
+    if (lines.length > 0)
+        return lines.join("\n");
+    return remote === "unknown"
+        ? "unknown: remote push state could not be determined; no local commits since the start of this turn."
+        : "No commits since the start of this turn.";
+}
+export async function indexFailure(tree) {
+    const lookup = await retryGit(tree, [
+        "rev-parse",
+        "--git-path",
+        "index.lock",
+    ]);
+    const unknown = {
+        reason: "worktree-index-unknown",
+        fix: "Check that the work tree and its git index path are accessible before starting another turn.",
+    };
+    if (lookup.code !== 0 || lookup.text === "")
+        return unknown;
+    const lockPath = resolve(tree.directory, lookup.text);
+    try {
+        await lstat(lockPath);
+        return {
+            reason: "worktree-index-locked",
+            lockPath,
+            fix: "Remove the index lock when no git process uses the work tree, then start another turn.",
+        };
+    }
+    catch (error) {
+        return Reflect.get(Object(error), "code") === "ENOENT" ? null : unknown;
+    }
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -1,8 +1,13 @@
+import { type TokenPrices } from "../token-prices.ts";
 import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
 export interface RetrySettings {
+    readonly tokenPrices?: TokenPrices;
+    readonly fallbackModels?: readonly string[];
     readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
 }
 interface RetryOptions extends RetrySettings {
+    readonly stopping?: AbortController;
+    readonly stopMilliseconds?: number;
     readonly metadata: Readonly<Record<string, unknown>>;
 }
 export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;