cloving 1.0.1 → 1.2.0

package/dist/src/daemon/notify.js

@@ -1,9 +1,10 @@
+import { recordedNotifications, saveNotifications, } from "./notify-journal.js";
 import { spawn } from "node:child_process";
-import { readFile, rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { send } from "../cli/client.js";
 import { record } from "../protocol.js";
 import { decisionNotification, outcomeNotification, } from "./notify-message.js";
+const retentionMilliseconds = 7 * 24 * 60 * 60_000;
 async function request(runtimeDirectory, request) {
     const reply = await send(runtimeDirectory, request);
     if (reply.kind === "error")
@@ -20,6 +21,20 @@ function rows(value) {
         return row;
     });
 }
+async function flushNotifications(journal) {
+    if (journal.unsaved.size === 0)
+        return;
+    await saveNotifications(journal.file, journal);
+    for (const entry of journal.unsaved.values())
+        if (entry.state === "failed" && entry.attempts === 5)
+            process.stderr.write(`Notify delivery failed: ${entry.key}\n`);
+    journal.unsaved.clear();
+}
+async function finishDelivery(journal, entry) {
+    journal.entries.set(entry.key, entry);
+    journal.unsaved.set(entry.key, entry);
+    await flushNotifications(journal);
+}
 async function observedOutcome(summary, source) {
     const { runtimeDirectory, cache } = source;
     const latestSequence = summary["latestSequence"];
@@ -74,26 +89,6 @@ async function events(runtimeDirectory, cache) {
     }
     return found;
 }
-async function recorded(file) {
-    const text = await readFile(file, "utf8").catch((error) => {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return undefined;
-        throw error;
-    });
-    if (text === undefined)
-        return undefined;
-    const value = JSON.parse(text);
-    if (!Array.isArray(value) || !value.every((key) => typeof key === "string"))
-        throw new Error("invalid-notify-record");
-    return new Set(value.map(String));
-}
-async function save(file, keys) {
-    await writeFile(`${file}.tmp`, JSON.stringify([...keys]) + "\n", {
-        mode: 0o600,
-        flush: true,
-    });
-    await rename(`${file}.tmp`, file);
-}
 export function notifyCommand(command, notification) {
     return new Promise((resolve, reject) => {
         const [executable = "", ...argumentsList] = command;
@@ -117,45 +112,189 @@ function report(label, error) {
     const reason = error instanceof Error ? error.message : String(error);
     process.stderr.write(`${label}: ${JSON.stringify(reason)}\n`);
 }
+function failureReason(error) {
+    if (!(error instanceof Error))
+        return "command error";
+    if ("code" in error && (error.code === "ENOENT" || error.code === "EACCES"))
+        return `spawn ${error.code}`;
+    return /^(?:exit code (?:\d+|null)|signal SIG[A-Z0-9]+)$/.test(error.message)
+        ? error.message
+        : "command error";
+}
+async function prepareDelivery(journal, entry) {
+    if (entry.message === null || entry.state === "sent")
+        return undefined;
+    if (entry.attempts === 5 && entry.state === "pending")
+        await finishDelivery(journal, {
+            ...entry,
+            state: "failed",
+            lastFailure: "attempt interrupted",
+        });
+    if (entry.attempts === 5)
+        return undefined;
+    const backoff = [0, 60_000, 300_000, 900_000, 3_600_000][entry.attempts] ?? 0;
+    if (entry.lastAttemptAt !== null &&
+        Date.now() - entry.lastAttemptAt < backoff)
+        return undefined;
+    const pending = {
+        ...entry,
+        message: entry.message,
+        state: "pending",
+        attempts: entry.attempts + 1,
+        lastAttemptAt: Date.now(),
+    };
+    const next = new Map(journal.entries).set(entry.key, pending);
+    await saveNotifications(journal.file, { ...journal, entries: next });
+    journal.entries.set(entry.key, pending);
+    return pending;
+}
+async function deliver(settings, entry) {
+    try {
+        await notifyCommand(settings.command, entry.message);
+        return { ...entry, state: "sent" };
+    }
+    catch (error) {
+        return {
+            ...entry,
+            state: "failed",
+            lastFailure: failureReason(error),
+        };
+    }
+}
+async function settleDelivery(journal, progress) {
+    if (progress.completed !== undefined) {
+        const entry = progress.completed;
+        progress.completed = undefined;
+        await finishDelivery(journal, entry);
+    }
+    await flushNotifications(journal);
+}
+async function scheduleDelivery(settings, delivery) {
+    const { journal, progress } = delivery;
+    const stopped = () => progress.stopping.aborted;
+    if (stopped() ||
+        progress.active !== undefined ||
+        progress.completed !== undefined ||
+        journal.unsaved.size > 0)
+        return;
+    for (const entry of journal.entries.values()) {
+        if (stopped())
+            return;
+        const pending = await prepareDelivery(journal, entry);
+        if (pending === undefined)
+            continue;
+        if (stopped()) {
+            await finishDelivery(journal, entry);
+            return;
+        }
+        progress.active = deliver(settings, pending).then((completed) => {
+            progress.completed = completed;
+            progress.active = undefined;
+            delivery.ready();
+        });
+        return;
+    }
+}
 async function check(settings, journal) {
+    const retained = new Map([...journal.entries].filter(([, entry]) => entry.state === "pending" ||
+        (entry.state === "failed" && entry.attempts < 5) ||
+        entry.lastAttemptAt === null ||
+        entry.lastAttemptAt >= Date.now() - retentionMilliseconds));
+    const removed = [...journal.entries.keys()].filter((key) => !retained.has(key));
+    if (removed.length > 0) {
+        const retiredKeys = new Set([...journal.retiredKeys, ...removed]);
+        await saveNotifications(journal.file, { entries: retained, retiredKeys });
+        for (const key of removed) {
+            journal.entries.delete(key);
+            journal.retiredKeys.add(key);
+        }
+    }
     for (const event of await events(settings.runtimeDirectory, journal.cache)) {
-        if (journal.keys.has(event.key))
+        if (journal.entries.has(event.key) || journal.retiredKeys.has(event.key))
             continue;
-        await save(journal.file, new Set([...journal.keys, event.key]));
-        journal.keys.add(event.key);
-        await notifyCommand(settings.command, event.message).catch((error) => {
-            report("Notify command failed", error);
-        });
+        const entry = {
+            ...event,
+            state: "pending",
+            attempts: 0,
+            lastAttemptAt: null,
+            lastFailure: null,
+        };
+        journal.entries.set(event.key, entry);
+        journal.unsaved.set(event.key, entry);
     }
+    await flushNotifications(journal);
 }
 export async function notifyRuns(settings) {
     const file = join(settings.directory, "notify-events.json");
-    const keys = await recorded(file);
+    const recorded = await recordedNotifications(file);
     const cache = new Map();
-    const journal = { file, keys: keys ?? new Set(), cache };
-    if (keys === undefined) {
+    const journal = {
+        file,
+        entries: recorded?.entries ?? new Map(),
+        retiredKeys: recorded?.retiredKeys ?? new Set(),
+        cache,
+        unsaved: new Map(),
+    };
+    if (recorded === undefined) {
         for (const event of await events(settings.runtimeDirectory, cache))
-            journal.keys.add(event.key);
-        await save(file, journal.keys);
+            journal.entries.set(event.key, {
+                ...event,
+                state: "sent",
+                attempts: 0,
+                lastAttemptAt: Date.now(),
+                lastFailure: null,
+            });
     }
+    await saveNotifications(file, journal);
+    return notificationLoop(settings, journal);
+}
+function notificationLoop(settings, journal) {
     let busy = false;
     let running = Promise.resolve();
+    const stopping = new AbortController();
+    const progress = {
+        stopping: stopping.signal,
+        active: undefined,
+        completed: undefined,
+    };
+    const ready = () => {
+        if (stopping.signal.aborted)
+            return;
+        running = running
+            .then(() => settleDelivery(journal, progress))
+            .then(() => scheduleDelivery(settings, { journal, progress, ready }))
+            .catch((error) => {
+            report("Notify delivery failed", error);
+        });
+    };
     const timer = setInterval(() => {
         if (busy)
             return;
         busy = true;
-        running = check(settings, journal)
+        running = running
+            .then(() => settleDelivery(journal, progress))
+            .then(() => check(settings, journal))
             .catch((error) => {
             report("Notify check failed", error);
-        })
-            .finally(() => {
+        });
+        ready();
+        running = running.finally(() => {
             busy = false;
         });
     }, settings.intervalMilliseconds ?? 15_000);
     return {
-        stop: async () => {
+        stop: () => {
+            stopping.abort();
             clearInterval(timer);
-            await running;
+            running = running
+                .then(async () => {
+                await progress.active;
+                await settleDelivery(journal, progress);
+            })
+                .catch((error) => {
+                report("Notify delivery failed", error);
+            });
+            return running;
         },
     };
 }

package/dist/src/daemon/operation-journal.d.ts

@@ -32,6 +32,7 @@ export declare function readRecord(folder: string, name: string): Promise<unknow
 export declare function writeRecord(folder: string, record: {
     readonly name: string;
     readonly value: unknown;
+    readonly permissions?: "private";
 }): Promise<void>;
 export declare function replied(identity: Identity, recorded: unknown): AdapterOutcome;
 export declare function interrupted(reason: string): Recorded;

package/dist/src/daemon/operation-journal.js

@@ -1,5 +1,5 @@
 import { createHash } from "node:crypto";
-import { mkdir, readFile, rename, writeFile } from "node:fs/promises";
+import { chmod, mkdir, readFile, rename, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 export function operationKey(text) {
     return createHash("sha256").update(text).digest("hex").slice(0, 32);
@@ -16,7 +16,9 @@ export async function readRecord(folder, name) {
 export async function writeRecord(folder, record) {
     const path = join(folder, record.name);
     await mkdir(folder, { recursive: true });
-    await writeFile(`${path}.staged`, JSON.stringify(record.value, null, 2));
+    if (record.permissions === "private")
+        await chmod(folder, 0o700);
+    await writeFile(`${path}.staged`, JSON.stringify(record.value, null, 2), record.permissions === "private" ? { mode: 0o600 } : {});
     await rename(`${path}.staged`, path);
 }
 export function replied(identity, recorded) {

package/dist/src/daemon/retrying-turn.d.ts

@@ -0,0 +1,12 @@
+import { type TokenPrices } from "../token-prices.ts";
+import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
+export interface RetrySettings {
+    readonly tokenPrices?: TokenPrices;
+    readonly fallbackModels?: readonly string[];
+    readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
+}
+interface RetryOptions extends RetrySettings {
+    readonly metadata: Readonly<Record<string, unknown>>;
+}
+export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;
+export {};

package/dist/src/daemon/retrying-turn.js

@@ -0,0 +1,231 @@
+import { join } from "node:path";
+import { mkdir } from "node:fs/promises";
+import { setTimeout as delay } from "node:timers/promises";
+import { priceOf, sumUsage, usageOf, } from "../worker-usage.js";
+import { commandModel, replaceCommandModel, estimatedCostUsd, priceForModel, } from "../token-prices.js";
+import { analysisCommand } from "./analysis-launch.js";
+import { capacityRefusal, startTurn, } from "./claude-turn.js";
+import { readRecord, writeRecord } from "./operation-journal.js";
+const waits = [60_000, 180_000];
+const continuation = "An earlier attempt stopped because the model was at capacity; continue from the current state of the work tree.\n\n";
+function written(folder, entry) {
+    return writeRecord(folder, { ...entry, permissions: "private" });
+}
+function total(values) {
+    if (values.includes(null))
+        return null;
+    return priceOf(values.reduce((sum, value) => sum + (value ?? 0), 0));
+}
+function combined(attempts) {
+    const { ends } = attempts;
+    const last = ends.at(-1) ?? { exitCode: null, result: undefined };
+    const result = last.result ?? ends.find((end) => end.result)?.result;
+    if (result === undefined)
+        return last;
+    return {
+        ...last,
+        result: {
+            ...result,
+            attempts: ends.map((end, index) => attemptUsage(attempts, { end, attempt: index + 1 })),
+            ...(last.result === undefined ? { failed: true, text: "" } : {}),
+            usage: sumUsage(ends.map((end) => end.result?.usage ?? usageOf(null))),
+            costUsd: total(ends.map((end) => end.result?.costUsd ?? null)),
+            turns: total(ends.map((end) => end.result?.turns ?? null)),
+        },
+    };
+}
+function controlled(attempts, start) {
+    const turn = startTurn(start);
+    const stop = () => {
+        turn.stop(attempts.grace.milliseconds);
+    };
+    attempts.stopping.signal.addEventListener("abort", stop, { once: true });
+    return {
+        ...turn,
+        ended: turn.ended.finally(() => {
+            attempts.stopping.signal.removeEventListener("abort", stop);
+        }),
+    };
+}
+function attemptCommand(attempts, attempt) {
+    const { start, options } = attempts;
+    const original = start.mode === "analysis" ? analysisCommand() : start.command;
+    const fallback = start.mode === "analysis"
+        ? undefined
+        : options.fallbackModels?.[attempt - 3];
+    const command = fallback === undefined ? original : replaceCommandModel(original, fallback);
+    const model = commandModel(command);
+    return {
+        command,
+        model,
+        ...(fallback !== undefined && model === null
+            ? { fallbackReason: "fallback-model-unavailable" }
+            : {}),
+    };
+}
+function attemptUsage(attempts, completion) {
+    const { end, attempt } = completion;
+    const model = end.result?.model ?? attemptCommand(attempts, attempt).model;
+    const usage = end.result?.usage ?? usageOf(null);
+    return {
+        attempt,
+        model,
+        usage,
+        costUsd: end.result?.costUsd ?? null,
+        estimatedCostUsd: estimatedCostUsd(usage, {
+            worker: attempts.start.mode === "analysis"
+                ? "claude"
+                : (attempts.start.worker ?? "claude"),
+            price: priceForModel(attempts.options.tokenPrices ?? {}, model),
+        }),
+    };
+}
+async function launched(attempts) {
+    const { start, options, ends } = attempts;
+    const attempt = ends.length + 1;
+    const selected = attemptCommand(attempts, attempt);
+    const folder = start.mode === "analysis" || attempt === 1
+        ? start.folder
+        : join(start.folder, `attempt-${String(attempt)}`);
+    const entry = {
+        ...options.metadata,
+        model: selected.model,
+        ...(selected.fallbackReason === undefined
+            ? {}
+            : { fallbackReason: selected.fallbackReason }),
+        attempt,
+        startedAt: attempt === 1 ? attempts.startedAt : new Date().toISOString(),
+    };
+    await written(start.folder, {
+        name: "started.json",
+        value: {
+            ...entry,
+            startedAt: attempts.startedAt,
+            phase: "starting",
+            processId: null,
+        },
+    });
+    await mkdir(folder, { recursive: true, mode: 0o700 });
+    if (attempts.stopping.signal.aborted)
+        return undefined;
+    const turn = controlled(attempts, {
+        ...start,
+        command: selected.command,
+        folder,
+        ...(attempt === 1 ? {} : { activityFolder: start.folder }),
+        prompt: start.mode === "analysis" || attempt === 1
+            ? start.prompt
+            : continuation + start.prompt,
+    });
+    try {
+        const value = { ...entry, phase: "running", processId: turn.processId };
+        await written(start.folder, {
+            name: "started.json",
+            value: { ...value, startedAt: attempts.startedAt },
+        });
+        await written(start.folder, {
+            name: `attempt-${String(attempt)}.json`,
+            value,
+        });
+        return turn;
+    }
+    catch (error) {
+        turn.stop(0);
+        await turn.ended;
+        throw error;
+    }
+}
+async function recordedEnd(attempts, completion) {
+    const { start, ends, options } = attempts;
+    const { processId, end } = completion;
+    const attempt = ends.length;
+    const selected = attemptCommand(attempts, attempt);
+    const name = `attempt-${String(attempt)}.json`;
+    const started = await readRecord(start.folder, name);
+    const startedAt = Reflect.get(Object(started), "startedAt");
+    if (typeof startedAt !== "string")
+        throw new Error("attempt-start-time-missing");
+    await written(start.folder, {
+        name,
+        value: {
+            ...options.metadata,
+            ...attemptUsage(attempts, { end, attempt }),
+            ...(selected.fallbackReason === undefined
+                ? {}
+                : { fallbackReason: selected.fallbackReason }),
+            attempt,
+            processId,
+            startedAt,
+            endedAt: new Date().toISOString(),
+            exitCode: end.exitCode,
+            capacity: capacityRefusal(end.result),
+        },
+    });
+}
+async function waited(attempts, milliseconds) {
+    const { start, options, stopping, ends } = attempts;
+    await written(start.folder, {
+        name: "started.json",
+        value: {
+            ...options.metadata,
+            startedAt: attempts.startedAt,
+            attempt: ends.length,
+            phase: "waiting",
+            processId: null,
+            waitMilliseconds: milliseconds,
+            waitingAt: new Date().toISOString(),
+        },
+    });
+    const wait = options.retryDelay ??
+        ((duration, signal) => delay(duration, undefined, { signal }));
+    await wait(milliseconds, stopping.signal).catch((error) => {
+        if (!stopping.signal.aborted)
+            throw error;
+    });
+}
+async function finished(attempts, first) {
+    const { ends } = attempts;
+    let turn = first;
+    while (turn !== undefined) {
+        const end = await turn.ended;
+        ends.push(end);
+        await recordedEnd(attempts, { processId: turn.processId, end });
+        const milliseconds = waits[ends.length - 1];
+        if (cancelled(attempts) ||
+            !capacityRefusal(end.result) ||
+            milliseconds === undefined)
+            break;
+        await waited(attempts, milliseconds);
+        if (cancelled(attempts))
+            return { ...combined(attempts), stopped: true };
+        turn = await launched(attempts);
+    }
+    return {
+        ...combined(attempts),
+        ...(cancelled(attempts) || turn === undefined ? { stopped: true } : {}),
+    };
+}
+function cancelled(attempts) {
+    return attempts.stopping.signal.aborted;
+}
+export async function startRetriedTurn(start, options) {
+    const attempts = {
+        start,
+        startedAt: new Date().toISOString(),
+        options,
+        stopping: new AbortController(),
+        ends: [],
+        grace: { milliseconds: 2_000 },
+    };
+    const turn = await launched(attempts);
+    if (turn === undefined)
+        throw new Error("turn-start-stopped");
+    return {
+        processId: turn.processId,
+        ended: finished(attempts, turn),
+        stop: (graceMilliseconds) => {
+            attempts.grace.milliseconds = graceMilliseconds;
+            attempts.stopping.abort();
+        },
+    };
+}

package/dist/src/daemon/ticket-dependency-waits.d.ts

@@ -0,0 +1,18 @@
+import { type Fields } from "../protocol.ts";
+interface Source {
+    readonly repository: string;
+    readonly command: readonly string[] | undefined;
+    readonly githubCommand?: readonly string[] | undefined;
+}
+export declare class TicketDependencyReadError extends Error {
+}
+export declare function ticketDependencyWaits(source: Source): {
+    readonly check: (candidate: {
+        readonly ticketId: string;
+        readonly ticket: string;
+        readonly signal: AbortSignal;
+    }) => Promise<readonly string[]>;
+    readonly refresh: (signal: AbortSignal) => Promise<readonly string[]>;
+    readonly skipped: () => readonly Fields[];
+};
+export {};

package/dist/src/daemon/ticket-dependency-waits.js

@@ -0,0 +1,60 @@
+import { hardDependencies } from "../ticket-dependencies.js";
+import { issueIdentity } from "../issue-identity.js";
+import { record } from "../protocol.js";
+import { calls } from "./github-calls.js";
+import { ticketText } from "./ticket-source.js";
+export class TicketDependencyReadError extends Error {
+}
+async function openDependencies(source, candidate) {
+    const github = calls({ repository: source.repository, gh: source.githubCommand ?? ["gh"] }, candidate.signal);
+    const issue = issueIdentity(candidate.ticket);
+    const repository = issue === null ? "{owner}/{repo}" : `${issue.owner}/${issue.repository}`;
+    const open = [];
+    for (const id of hardDependencies(candidate.ticket)) {
+        const response = JSON.parse(await github.gh(["api", `repos/${repository}/issues/${id}`]).catch(() => {
+            throw new TicketDependencyReadError("github-call-failed");
+        }));
+        const state = record(response)?.["state"];
+        if (state !== "open" && state !== "closed")
+            throw new Error("invalid-dependency-state");
+        if (state === "open")
+            open.push(id);
+    }
+    return open;
+}
+export function ticketDependencyWaits(source) {
+    const waiting = new Map();
+    let nextRefresh;
+    const check = async (candidate) => {
+        const open = await openDependencies(source, candidate);
+        if (open.length === 0)
+            waiting.delete(candidate.ticketId);
+        else
+            waiting.set(candidate.ticketId, open);
+        return open;
+    };
+    return {
+        check,
+        refresh: async (signal) => {
+            const tickets = [...waiting.keys()];
+            const ticketId = nextRefresh !== undefined && waiting.has(nextRefresh)
+                ? nextRefresh
+                : tickets[0];
+            if (ticketId === undefined)
+                return [];
+            nextRefresh = tickets[tickets.indexOf(ticketId) + 1] ?? tickets[0];
+            if (source.command === undefined)
+                throw new Error("ticket-source-not-configured");
+            const read = await ticketText(source.command, { ticketId, signal });
+            if (read.kind === "failed")
+                throw new TicketDependencyReadError("ticket-not-available");
+            await check({ ticketId, ticket: read.text, signal });
+            return [...waiting.keys()];
+        },
+        skipped: () => [...waiting].map(([ticketId, dependencies]) => ({
+            ticketId,
+            reason: "ticket-dependency-open",
+            dependencies,
+        })),
+    };
+}

package/dist/src/daemon/ticket-registration.d.ts

@@ -1,8 +1,8 @@
-import type { DeclaredWorkflow } from "../workflow-declaration.ts";
+import type { DeclaredWorkflow, WorkflowStage } from "../workflow-declaration.ts";
 import type { Resolver, WorkflowRegistration } from "../service/registry.ts";
-export declare const ticketWorkflow = "develop-ticket";
-export declare const promptWorkflow = "develop-prompt";
-export declare const issueWorkflow = "develop-issue";
+export declare const ticketWorkflow: "develop-ticket";
+export declare const promptWorkflow: "develop-prompt";
+export declare const issueWorkflow: "develop-issue";
 export declare const ticketCapabilities: string[];
 export type TicketRegistrations = {
     readonly kind: "admitted";
@@ -15,6 +15,8 @@ export interface TicketSource {
     readonly repository: string;
     readonly command: readonly string[] | undefined;
     readonly nextCommand?: readonly string[] | undefined;
+    readonly githubCommand?: readonly string[] | undefined;
+    readonly skipTickets?: readonly string[];
 }
 export interface TicketRegistration {
     readonly workflowId: string;
@@ -23,6 +25,8 @@ export interface TicketRegistration {
     readonly submission: WorkflowRegistration["submission"];
     readonly effectLimit: number;
     readonly capabilities?: readonly string[];
+    readonly stages?: readonly WorkflowStage[];
+    readonly handoffs?: readonly string[];
 }
 export declare function ticketResolver(source: TicketSource): Resolver;
 export declare function issueResolver(source: TicketSource): Resolver;