cloving 1.2.0 → 1.4.0

package/dist/src/daemon/notify.js

@@ -1,26 +1,9 @@
 import { recordedNotifications, saveNotifications, } from "./notify-journal.js";
 import { spawn } from "node:child_process";
 import { join } from "node:path";
-import { send } from "../cli/client.js";
-import { record } from "../protocol.js";
-import { decisionNotification, outcomeNotification, } from "./notify-message.js";
+import { deliveryBackoffMilliseconds } from "../delivery-backoff.js";
+import { notificationEvents } from "./notify-events.js";
 const retentionMilliseconds = 7 * 24 * 60 * 60_000;
-async function request(runtimeDirectory, request) {
-    const reply = await send(runtimeDirectory, request);
-    if (reply.kind === "error")
-        throw new Error(reply.error.code);
-    return reply.result;
-}
-function rows(value) {
-    if (!Array.isArray(value))
-        throw new Error("invalid-notify-list");
-    return value.map((item) => {
-        const row = record(item);
-        if (row === undefined)
-            throw new Error("invalid-notify-row");
-        return row;
-    });
-}
 async function flushNotifications(journal) {
     if (journal.unsaved.size === 0)
         return;
@@ -35,60 +18,6 @@ async function finishDelivery(journal, entry) {
     journal.unsaved.set(entry.key, entry);
     await flushNotifications(journal);
 }
-async function observedOutcome(summary, source) {
-    const { runtimeDirectory, cache } = source;
-    const latestSequence = summary["latestSequence"];
-    if (typeof latestSequence !== "number")
-        throw new Error("invalid-notify-sequence");
-    const runId = String(summary["runId"]);
-    const previous = cache.get(runId);
-    if (previous !== undefined && previous.latestSequence === latestSequence)
-        return previous.event;
-    const status = await request(runtimeDirectory, {
-        operation: "status",
-        parameters: { runId },
-    });
-    const run = record(status["run"]);
-    if (run === undefined)
-        throw new Error("invalid-notify-status");
-    const event = outcomeNotification(run);
-    cache.set(runId, { latestSequence, event });
-    return event;
-}
-async function events(runtimeDirectory, cache) {
-    const found = [];
-    const runs = new Map();
-    let after = "";
-    do {
-        const page = await request(runtimeDirectory, {
-            operation: "status",
-            parameters: { after, limit: 500 },
-        });
-        for (const summary of rows(page["runs"])) {
-            runs.set(String(summary["runId"]), summary);
-            const outcome = await observedOutcome(summary, {
-                runtimeDirectory,
-                cache,
-            });
-            if (outcome !== undefined)
-                found.push(outcome);
-        }
-        const next = page["next"];
-        if (next !== null && (typeof next !== "string" || next <= after))
-            throw new Error("invalid-notify-page");
-        after = next;
-    } while (after !== null);
-    const questions = await request(runtimeDirectory, {
-        operation: "questions",
-        parameters: {},
-    });
-    for (const decision of rows(questions["decisions"])) {
-        const run = runs.get(String(decision["runId"]));
-        if (run !== undefined)
-            found.push(decisionNotification(run, decision));
-    }
-    return found;
-}
 export function notifyCommand(command, notification) {
     return new Promise((resolve, reject) => {
         const [executable = "", ...argumentsList] = command;
@@ -132,7 +61,7 @@ async function prepareDelivery(journal, entry) {
         });
     if (entry.attempts === 5)
         return undefined;
-    const backoff = [0, 60_000, 300_000, 900_000, 3_600_000][entry.attempts] ?? 0;
+    const backoff = deliveryBackoffMilliseconds[entry.attempts] ?? 0;
     if (entry.lastAttemptAt !== null &&
         Date.now() - entry.lastAttemptAt < backoff)
         return undefined;
@@ -209,7 +138,7 @@ async function check(settings, journal) {
             journal.retiredKeys.add(key);
         }
     }
-    for (const event of await events(settings.runtimeDirectory, journal.cache)) {
+    for (const event of await notificationEvents(settings.runtimeDirectory, journal.cache)) {
         if (journal.entries.has(event.key) || journal.retiredKeys.has(event.key))
             continue;
         const entry = {
@@ -236,12 +165,12 @@ export async function notifyRuns(settings) {
         unsaved: new Map(),
     };
     if (recorded === undefined) {
-        for (const event of await events(settings.runtimeDirectory, cache))
+        for (const event of await notificationEvents(settings.runtimeDirectory, cache))
             journal.entries.set(event.key, {
                 ...event,
-                state: "sent",
+                state: event.message.event === "schedule-stopped" ? "pending" : "sent",
                 attempts: 0,
-                lastAttemptAt: Date.now(),
+                lastAttemptAt: event.message.event === "schedule-stopped" ? null : Date.now(),
                 lastFailure: null,
             });
     }

package/dist/src/daemon/opencode-stream.d.ts

@@ -0,0 +1,5 @@
+export declare function opencodeStream(): {
+    line: (line: string) => readonly string[];
+    stderr: (chunk: string) => void;
+    result: (exitCode: number | null) => string;
+};

package/dist/src/daemon/opencode-stream.js

@@ -0,0 +1,87 @@
+import { record } from "../protocol.js";
+import { sumUsage, usageOf, tokenCount, } from "../worker-usage.js";
+function parsed(line) {
+    try {
+        return record(JSON.parse(line)) ?? {};
+    }
+    catch {
+        return {};
+    }
+}
+function tokens(value) {
+    const counts = record(value) ?? {};
+    const cache = record(counts["cache"]) ?? {};
+    const output = tokenCount(counts["output"]);
+    const reasoning = tokenCount(counts["reasoning"]);
+    return usageOf({
+        input_tokens: counts["input"],
+        cache_read_input_tokens: cache["read"],
+        output_tokens: output === null || reasoning === null ? null : output + reasoning,
+    });
+}
+function errorMessage(value) {
+    const error = record(value) ?? {};
+    const data = record(error["data"]) ?? {};
+    const message = data["message"] ?? error["name"];
+    return typeof message === "string" ? message : "opencode error";
+}
+function translated(line, state) {
+    const event = parsed(line);
+    const part = record(event["part"]) ?? {};
+    if (typeof event["sessionID"] === "string")
+        state.sessionId = event["sessionID"];
+    if (typeof event["model"] === "string")
+        state.model = event["model"];
+    if (event["type"] === "error")
+        state.error = errorMessage(event["error"]);
+    if (event["type"] === "step_finish") {
+        state.steps.push(tokens(part["tokens"]));
+        state.reason = part["reason"];
+    }
+    if (event["type"] !== "text" || typeof part["text"] !== "string")
+        return [];
+    state.message = part["text"];
+    return [
+        JSON.stringify({
+            type: "assistant",
+            message: { content: [{ type: "text", text: state.message }] },
+        }),
+    ];
+}
+function result(exitCode, state) {
+    const success = exitCode === 0 && state.error === null && state.reason === "stop";
+    const failure = exitCode === 0
+        ? "opencode ended without a final stop step"
+        : `opencode ended with the code ${String(exitCode)}`;
+    return JSON.stringify({
+        type: "result",
+        subtype: success ? "success" : "error_during_execution",
+        is_error: !success,
+        result: success
+            ? state.message
+            : state.error || state.stderr.trim() || failure,
+        total_cost_usd: null,
+        model: state.model,
+        usage: sumUsage(state.steps),
+        session_id: state.sessionId,
+        num_turns: state.steps.length,
+    });
+}
+export function opencodeStream() {
+    const state = {
+        steps: [],
+        message: "",
+        sessionId: null,
+        model: null,
+        reason: null,
+        error: null,
+        stderr: "",
+    };
+    return {
+        line: (line) => translated(line, state),
+        stderr: (chunk) => {
+            state.stderr = (state.stderr + chunk).slice(-16_000);
+        },
+        result: (exitCode) => result(exitCode, state),
+    };
+}

package/dist/src/daemon/operation-journal.d.ts

@@ -11,6 +11,7 @@ export interface WorkerReport {
     readonly reason?: string;
 }
 export interface Recorded {
+    readonly stoppedBy?: "daemon-shutdown";
     readonly outcome: WorkerReport["outcome"];
     readonly result?: string;
     readonly reason?: string;

package/dist/src/daemon/queue-status.d.ts

@@ -0,0 +1,10 @@
+export declare function queueStatus(settings: {
+    readonly runtimeDirectory: string;
+    readonly workflowId: string;
+}): Promise<{
+    readonly busy: number;
+    readonly holding: boolean;
+}>;
+export declare function spendNotice(): (status: {
+    readonly holding: boolean;
+}) => boolean;

package/dist/src/daemon/queue-status.js

@@ -0,0 +1,51 @@
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+function queueBusy(value) {
+    const busy = record(value)?.["busy"];
+    if (typeof busy !== "number")
+        return undefined;
+    if (!Number.isSafeInteger(busy) || busy < 0)
+        throw new Error("invalid-queue-status");
+    return busy;
+}
+export async function queueStatus(settings) {
+    let count = 0;
+    let after = "";
+    do {
+        const reply = await send(settings.runtimeDirectory, {
+            operation: "status",
+            parameters: { after, limit: 500 },
+        });
+        if (reply.kind === "error")
+            throw new Error(reply.error.code);
+        const busy = queueBusy(reply.result["queue"]);
+        if (busy !== undefined)
+            return {
+                busy,
+                holding: record(record(reply.result["queue"])?.["spend"])?.["holding"] ===
+                    true,
+            };
+        const runs = reply.result["runs"];
+        if (!Array.isArray(runs))
+            throw new Error("invalid-run-list");
+        count += runs.filter((value) => {
+            const run = record(value);
+            return (run?.["workflowId"] === settings.workflowId &&
+                !["completed", "failed", "cancelled"].includes(String(run["state"])));
+        }).length;
+        const next = reply.result["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-run-page");
+        after = next;
+    } while (after !== null);
+    return { busy: count, holding: false };
+}
+export function spendNotice() {
+    let held = false;
+    return ({ holding }) => {
+        if (holding && !held)
+            process.stderr.write("Queue holding for spend limit\n");
+        held = holding;
+        return holding;
+    };
+}

package/dist/src/daemon/resume-index-lock.d.ts

@@ -0,0 +1,13 @@
+interface LockIdentity {
+    readonly path: string;
+    readonly dev: number;
+    readonly ino: number;
+    readonly ctimeMs: number;
+    readonly mtimeMs: number;
+}
+export interface OwnedIndexLock extends LockIdentity {
+    readonly processIds: readonly number[];
+}
+export declare function ownedIndexLock(path: string): Promise<OwnedIndexLock | undefined>;
+export declare function removeStoppedIndexLock(lock: OwnedIndexLock, signal: AbortSignal): Promise<boolean>;
+export {};

package/dist/src/daemon/resume-index-lock.js

@@ -0,0 +1,160 @@
+import { execFile } from "node:child_process";
+import { lstat, readdir, readFile, stat, unlink } from "node:fs/promises";
+import { promisify } from "node:util";
+const execute = promisify(execFile);
+function disappeared(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        (error.code === "ENOENT" || error.code === "ESRCH"));
+}
+async function descriptorUsesLock(path, lock) {
+    try {
+        const opened = await stat(path);
+        if (opened.dev !== lock.dev || opened.ino !== lock.ino)
+            return false;
+        const information = await readFile(path.replace("/fd/", "/fdinfo/"), "utf8");
+        const flags = /^flags:\s+([0-7]+)$/m.exec(information)?.[1];
+        const inode = /^ino:\s+(\d+)$/m.exec(information)?.[1];
+        if (flags === undefined || Number(inode) !== lock.ino)
+            return undefined;
+        return (Number.parseInt(flags, 8) & 3) !== 0;
+    }
+    catch (error) {
+        return disappeared(error) ? false : undefined;
+    }
+}
+async function processUsesLock(directory, lock) {
+    try {
+        for (const descriptor of await readdir(`${directory}/fd`)) {
+            const used = await descriptorUsesLock(`${directory}/fd/${descriptor}`, lock);
+            if (used !== false)
+                return used;
+        }
+        return false;
+    }
+    catch (error) {
+        if (disappeared(error))
+            return false;
+        if (!(error instanceof Error) ||
+            !("code" in error) ||
+            (error.code !== "EACCES" && error.code !== "EPERM"))
+            return undefined;
+        return (await unrelatedProcess(directory, lock)) ? false : undefined;
+    }
+}
+async function unrelatedProcess(directory, lock) {
+    if (lock.uid !== process.geteuid?.() || (lock.mode & 0o022) !== 0)
+        return false;
+    try {
+        const status = await readFile(`${directory}/status`, "utf8");
+        const users = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m.exec(status);
+        return (users !== null &&
+            users
+                .slice(1)
+                .map(Number)
+                .every((user) => Number.isSafeInteger(user) && user !== lock.uid));
+    }
+    catch {
+        return false;
+    }
+}
+async function lockOwners(lock) {
+    if (process.platform === "linux") {
+        const processIds = [];
+        for (const entry of await readdir("/proc")) {
+            if (!/^\d+$/.test(entry))
+                continue;
+            const used = await processUsesLock(`/proc/${entry}`, lock);
+            if (used === undefined)
+                return undefined;
+            if (used)
+                processIds.push(Number(entry));
+        }
+        return processIds;
+    }
+    try {
+        const { stdout, stderr } = await execute("lsof", ["-Fpfa", "--", lock.path], {
+            encoding: "utf8",
+            timeout: 5000,
+            maxBuffer: 8 * 1024 * 1024,
+        });
+        return stderr === "" ? writableOwners(stdout) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function writableOwner(record) {
+    const [header, ...descriptors] = record.trim().split(/^f/m);
+    const processId = Number(header?.trim().slice(1));
+    const access = descriptors.map((descriptor) => descriptor.split("\n")[1]);
+    if (!Number.isSafeInteger(processId) ||
+        processId < 1 ||
+        descriptors.length === 0 ||
+        access.some((mode) => mode !== "ar" && mode !== "aw" && mode !== "au"))
+        return undefined;
+    return access.some((mode) => mode === "aw" || mode === "au")
+        ? processId
+        : null;
+}
+function writableOwners(text) {
+    const owners = text
+        .trim()
+        .split(/(?=^p)/m)
+        .map(writableOwner);
+    return owners.includes(undefined)
+        ? undefined
+        : [
+            ...new Set(owners.filter((owner) => typeof owner === "number")),
+        ];
+}
+export async function ownedIndexLock(path) {
+    try {
+        const identity = await lstat(path);
+        if (!identity.isFile())
+            return undefined;
+        const lock = {
+            path,
+            dev: identity.dev,
+            ino: identity.ino,
+            ctimeMs: identity.ctimeMs,
+            mtimeMs: identity.mtimeMs,
+            uid: identity.uid,
+            mode: identity.mode,
+        };
+        const processIds = await lockOwners(lock);
+        return processIds === undefined || processIds.length === 0
+            ? undefined
+            : { ...lock, processIds };
+    }
+    catch {
+        return undefined;
+    }
+}
+function stopped(processId) {
+    try {
+        process.kill(processId, 0);
+        return false;
+    }
+    catch (error) {
+        return error instanceof Error && "code" in error && error.code === "ESRCH";
+    }
+}
+export async function removeStoppedIndexLock(lock, signal) {
+    try {
+        if (!lock.processIds.every(stopped))
+            return false;
+        const after = await lstat(lock.path);
+        if (signal.aborted ||
+            lock.dev !== after.dev ||
+            lock.ino !== after.ino ||
+            lock.ctimeMs !== after.ctimeMs ||
+            lock.mtimeMs !== after.mtimeMs)
+            return false;
+        await unlink(lock.path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/src/daemon/resume-worktree.d.ts

@@ -0,0 +1,7 @@
+import { type IndexFailure } from "./retry-worktree.ts";
+export declare function resumeWorktree(request: {
+    readonly directory: string;
+    readonly folder: string;
+    readonly signal: AbortSignal;
+    readonly stopMilliseconds?: number;
+}): Promise<IndexFailure | null>;

package/dist/src/daemon/resume-worktree.js

@@ -0,0 +1,29 @@
+import { ownedIndexLock, removeStoppedIndexLock } from "./resume-index-lock.js";
+import { stopTreeProcesses } from "./worktree-processes.js";
+import { indexFailure } from "./retry-worktree.js";
+async function inspectLock(failure) {
+    if (failure?.lockPath === undefined)
+        return undefined;
+    return ownedIndexLock(failure.lockPath);
+}
+export async function resumeWorktree(request) {
+    const aborted = () => request.signal.aborted;
+    const before = await indexFailure(request);
+    const lock = await inspectLock(before);
+    if (aborted())
+        return before;
+    const stopped = await stopTreeProcesses(request);
+    const locked = await indexFailure(request);
+    if (!stopped)
+        return (locked ?? {
+            reason: "worktree-index-locked",
+            fix: "Stop the remaining work tree processes before resuming the turn.",
+        });
+    if (locked?.reason !== "worktree-index-locked" || aborted())
+        return locked;
+    if (lock === undefined ||
+        lock.path !== locked.lockPath ||
+        !(await removeStoppedIndexLock(lock, request.signal)))
+        return locked;
+    return indexFailure(request);
+}

package/dist/src/daemon/retry-worktree.d.ts

@@ -0,0 +1,18 @@
+export interface RetryTree {
+    readonly directory: string;
+    readonly signal: AbortSignal;
+}
+interface GitResult {
+    readonly text: string;
+    readonly code: number | null;
+}
+export declare function retryGit(tree: RetryTree, parts: readonly string[]): Promise<GitResult>;
+export declare function turnHead(tree: RetryTree): Promise<string | null>;
+export declare function retryEvidence(tree: RetryTree, startCommit: string | null): Promise<string>;
+export interface IndexFailure {
+    readonly reason: "worktree-index-unknown" | "worktree-index-locked";
+    readonly lockPath?: string;
+    readonly fix: string;
+}
+export declare function indexFailure(tree: RetryTree): Promise<IndexFailure | null>;
+export {};

package/dist/src/daemon/retry-worktree.js

@@ -0,0 +1,139 @@
+import { spawn } from "node:child_process";
+import { lstat } from "node:fs/promises";
+import { resolve } from "node:path";
+function kill(child) {
+    if (child.pid === undefined)
+        return;
+    try {
+        process.kill(-child.pid, "SIGKILL");
+    }
+    catch {
+        child.kill("SIGKILL");
+    }
+}
+export async function retryGit(tree, parts) {
+    if (tree.signal.aborted)
+        return { text: "", code: null };
+    const child = spawn("git", ["-C", tree.directory, ...parts], {
+        stdio: ["ignore", "pipe", "ignore"],
+        detached: true,
+        env: { ...process.env, GIT_TERMINAL_PROMPT: "0" },
+    });
+    const closed = Promise.withResolvers();
+    let text = "";
+    const stop = () => {
+        kill(child);
+        child.stdout.destroy();
+        closed.resolve(null);
+    };
+    const timer = setTimeout(stop, 30_000);
+    tree.signal.addEventListener("abort", stop, { once: true });
+    child.stdout.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        text += chunk;
+        if (Buffer.byteLength(text) > 1_048_576)
+            stop();
+    });
+    child.once("error", () => {
+        closed.resolve(null);
+    });
+    child.once("close", closed.resolve);
+    const code = await closed.promise;
+    clearTimeout(timer);
+    tree.signal.removeEventListener("abort", stop);
+    return { text: text.trim(), code };
+}
+export async function turnHead(tree) {
+    const head = await retryGit(tree, ["rev-parse", "--verify", "HEAD"]);
+    return head.code === 0 && /^[a-f0-9]{40,64}$/.test(head.text)
+        ? head.text
+        : null;
+}
+async function remoteState(tree) {
+    const branch = await retryGit(tree, ["symbolic-ref", "--short", "HEAD"]);
+    if (branch.code !== 0 || tree.signal.aborted)
+        return "unknown";
+    const reference = `refs/heads/${branch.text}`;
+    const remote = await retryGit(tree, [
+        "ls-remote",
+        "--exit-code",
+        "--heads",
+        "origin",
+        reference,
+    ]);
+    if (remote.code === 2)
+        return "not pushed";
+    if (remote.code !== 0)
+        return "unknown";
+    const tracking = `refs/remotes/origin/${branch.text}`;
+    const fetched = await retryGit(tree, [
+        "fetch",
+        "--no-tags",
+        "origin",
+        `+${reference}:${tracking}`,
+    ]);
+    return fetched.code === 0 ? tracking : "unknown";
+}
+async function pushState(tree, evidence) {
+    const { commit, remote } = evidence;
+    if (!remote.startsWith("refs/"))
+        return remote;
+    const ancestor = await retryGit(tree, [
+        "merge-base",
+        "--is-ancestor",
+        commit,
+        remote,
+    ]);
+    if (ancestor.code === 1)
+        return "not pushed";
+    return ancestor.code === 0 ? "pushed" : "unknown";
+}
+export async function retryEvidence(tree, startCommit) {
+    const remote = await remoteState(tree);
+    if (startCommit === null || tree.signal.aborted)
+        return "unknown: commits since the start of this turn could not be determined.";
+    const commits = await retryGit(tree, [
+        "log",
+        "--format=%H %s",
+        `${startCommit}..HEAD`,
+        "--",
+    ]);
+    if (commits.code !== 0)
+        return "unknown: commits since the start of this turn could not be listed.";
+    const lines = [];
+    for (const line of commits.text.split("\n").filter(Boolean)) {
+        const commit = line.split(" ")[0] ?? "";
+        const state = await pushState(tree, { commit, remote });
+        lines.push(`${commit} ${state}${line.slice(commit.length)}`);
+    }
+    if (lines.length > 0)
+        return lines.join("\n");
+    return remote === "unknown"
+        ? "unknown: remote push state could not be determined; no local commits since the start of this turn."
+        : "No commits since the start of this turn.";
+}
+export async function indexFailure(tree) {
+    const lookup = await retryGit(tree, [
+        "rev-parse",
+        "--git-path",
+        "index.lock",
+    ]);
+    const unknown = {
+        reason: "worktree-index-unknown",
+        fix: "Check that the work tree and its git index path are accessible before starting another turn.",
+    };
+    if (lookup.code !== 0 || lookup.text === "")
+        return unknown;
+    const lockPath = resolve(tree.directory, lookup.text);
+    try {
+        await lstat(lockPath);
+        return {
+            reason: "worktree-index-locked",
+            lockPath,
+            fix: "Remove the index lock when no git process uses the work tree, then start another turn.",
+        };
+    }
+    catch (error) {
+        return Reflect.get(Object(error), "code") === "ENOENT" ? null : unknown;
+    }
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -1,12 +1,4 @@
-import { type TokenPrices } from "../token-prices.ts";
 import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
-export interface RetrySettings {
-    readonly tokenPrices?: TokenPrices;
-    readonly fallbackModels?: readonly string[];
-    readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
-}
-interface RetryOptions extends RetrySettings {
-    readonly metadata: Readonly<Record<string, unknown>>;
-}
+import { type RetryOptions } from "./turn-attempts.ts";
+export type { RetrySettings } from "./turn-attempts.ts";
 export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;
-export {};