cloving 1.3.0 → 1.5.0

package/dist/src/daemon/queue-journal.d.ts

@@ -0,0 +1,20 @@
+import { type QueueCommand, type QueueDrain } from "../queue-state.ts";
+export declare const queueControlLimit = 32;
+export interface QueueControlRecord {
+    readonly commandId: string;
+    readonly command: QueueCommand;
+}
+export interface SelfUpdateHold {
+    readonly mark: string;
+    readonly startedAt: number;
+}
+export interface QueueJournal {
+    readonly commands: Set<string>;
+    readonly failures: string[];
+    readonly controls: QueueControlRecord[];
+    readonly save: () => Promise<void>;
+    retryCommandId: string | undefined;
+    selfUpdateHold: SelfUpdateHold | undefined;
+    drain: QueueDrain | undefined;
+}
+export declare function queueJournal(directory: string): Promise<QueueJournal>;

package/dist/src/daemon/queue-journal.js

@@ -0,0 +1,89 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { queueFailureLimit, } from "../queue-state.js";
+export const queueControlLimit = 32;
+function savedHold(value) {
+    if (value === undefined)
+        return undefined;
+    const held = record(value);
+    if (typeof held?.["mark"] !== "string" ||
+        typeof held["startedAt"] !== "number" ||
+        !Number.isFinite(held["startedAt"]) ||
+        held["startedAt"] <= 0)
+        throw new Error("invalid-queue-commands");
+    return { mark: held["mark"], startedAt: held["startedAt"] };
+}
+function instant(value) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0;
+}
+function savedDrain(value) {
+    if (value === undefined)
+        return undefined;
+    const drain = record(value);
+    const drainedAt = drain?.["drainedAt"];
+    if (!instant(drain?.["startedAt"]) ||
+        !(drainedAt === null || instant(drainedAt)))
+        throw new Error("invalid-queue-commands");
+    return { startedAt: drain["startedAt"], drainedAt };
+}
+function savedControls(value) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value) || value.length > queueControlLimit)
+        throw new Error("invalid-queue-commands");
+    return value.map((entry) => {
+        const control = record(entry);
+        const commandId = control?.["commandId"];
+        const command = control?.["command"];
+        if (typeof commandId !== "string" ||
+            commandId.length === 0 ||
+            (command !== "resume" && command !== "drain"))
+            throw new Error("invalid-queue-commands");
+        return { commandId, command };
+    });
+}
+function identifiers(value) {
+    if (!Array.isArray(value) ||
+        !value.every((id) => typeof id === "string" && id.length > 0))
+        throw new Error("invalid-queue-commands");
+    return value.map(String);
+}
+export async function queueJournal(directory) {
+    const file = join(directory, "queue-commands.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "[]";
+        throw error;
+    });
+    const value = JSON.parse(text);
+    const saved = record(value);
+    const commands = new Set(identifiers(Array.isArray(value) ? value : saved?.["commands"]));
+    const failures = Array.isArray(value) ? [] : identifiers(saved?.["failures"]);
+    if (failures.length > queueFailureLimit)
+        throw new Error("invalid-queue-commands");
+    let saving = Promise.resolve();
+    const write = async () => {
+        await writeFile(`${file}.tmp`, JSON.stringify({
+            commands: [...commands],
+            failures,
+            selfUpdateHold: pending.selfUpdateHold,
+            drain: pending.drain,
+            controls: pending.controls.length === 0 ? undefined : pending.controls,
+        }) + "\n", { mode: 0o600, flush: true });
+        await rename(`${file}.tmp`, file);
+    };
+    const pending = {
+        commands,
+        failures,
+        controls: savedControls(saved?.["controls"]),
+        retryCommandId: undefined,
+        selfUpdateHold: savedHold(saved?.["selfUpdateHold"]),
+        drain: savedDrain(saved?.["drain"]),
+        save: () => {
+            saving = saving.then(write, write);
+            return saving;
+        },
+    };
+    return pending;
+}

package/dist/src/daemon/queue-status.d.ts

@@ -1,4 +1,10 @@
-export declare function activeQueueRuns(settings: {
+export declare function queueStatus(settings: {
     readonly runtimeDirectory: string;
-    readonly workflowId?: string;
-}): Promise<number>;
+    readonly workflowId: string;
+}): Promise<{
+    readonly busy: number;
+    readonly holding: boolean;
+}>;
+export declare function spendNotice(): (status: {
+    readonly holding: boolean;
+}) => boolean;

package/dist/src/daemon/queue-status.js

@@ -8,7 +8,7 @@ function queueBusy(value) {
         throw new Error("invalid-queue-status");
     return busy;
 }
-export async function activeQueueRuns(settings) {
+export async function queueStatus(settings) {
     let count = 0;
     let after = "";
     do {
@@ -20,13 +20,17 @@ export async function activeQueueRuns(settings) {
             throw new Error(reply.error.code);
         const busy = queueBusy(reply.result["queue"]);
         if (busy !== undefined)
-            return busy;
+            return {
+                busy,
+                holding: record(record(reply.result["queue"])?.["spend"])?.["holding"] ===
+                    true,
+            };
         const runs = reply.result["runs"];
         if (!Array.isArray(runs))
             throw new Error("invalid-run-list");
         count += runs.filter((value) => {
             const run = record(value);
-            return (run?.["workflowId"] === (settings.workflowId ?? "develop-issue") &&
+            return (run?.["workflowId"] === settings.workflowId &&
                 !["completed", "failed", "cancelled"].includes(String(run["state"])));
         }).length;
         const next = reply.result["next"];
@@ -34,5 +38,14 @@ export async function activeQueueRuns(settings) {
             throw new Error("invalid-run-page");
         after = next;
     } while (after !== null);
-    return count;
+    return { busy: count, holding: false };
+}
+export function spendNotice() {
+    let held = false;
+    return ({ holding }) => {
+        if (holding && !held)
+            process.stderr.write("Queue holding for spend limit\n");
+        held = holding;
+        return holding;
+    };
 }

package/dist/src/daemon/record-archive.d.ts

@@ -0,0 +1,19 @@
+export interface RecordFiles {
+    readonly move: (source: string, target: string) => Promise<void>;
+    readonly copy: (source: string, target: string) => Promise<void>;
+}
+export interface StoredFile {
+    readonly path: string;
+    readonly size: number;
+    readonly changedAt: number;
+}
+export declare const nodeFiles: RecordFiles;
+export declare function failureReason(error: unknown): string;
+export declare function storedFiles(folder: string): Promise<StoredFile[]>;
+export declare function archiveReady(folder: string, settings: {
+    readonly configured: boolean;
+}): Promise<string | undefined>;
+export declare function archivedFolder(place: {
+    readonly source: string;
+    readonly target: string;
+}, files: RecordFiles): Promise<void>;

package/dist/src/daemon/record-archive.js

@@ -0,0 +1,82 @@
+import { constants } from "node:fs";
+import { access, cp, lstat, mkdir, readdir, rename, rm, stat, } from "node:fs/promises";
+import { dirname, join, relative } from "node:path";
+export const nodeFiles = {
+    move: rename,
+    copy: (source, target) => cp(source, target, {
+        recursive: true,
+        errorOnExist: true,
+        force: false,
+        preserveTimestamps: true,
+    }),
+};
+export function failureReason(error) {
+    if (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        typeof error.code === "string")
+        return error.code;
+    return error instanceof Error ? error.message : String(error);
+}
+export async function storedFiles(folder) {
+    const found = [];
+    for (const entry of await readdir(folder, {
+        withFileTypes: true,
+        recursive: true,
+    })) {
+        if (!entry.isFile())
+            continue;
+        const path = join(entry.parentPath, entry.name);
+        const file = await lstat(path);
+        found.push({
+            path: relative(folder, path),
+            size: file.size,
+            changedAt: file.mtimeMs,
+        });
+    }
+    return found;
+}
+export async function archiveReady(folder, settings) {
+    try {
+        if (!settings.configured)
+            await mkdir(folder, { recursive: true });
+        if (!(await stat(folder)).isDirectory())
+            return "not-a-folder";
+        await access(folder, constants.W_OK);
+        return undefined;
+    }
+    catch (error) {
+        return failureReason(error);
+    }
+}
+async function sameFiles(source, copy) {
+    const listed = async (folder) => JSON.stringify((await storedFiles(folder)).map((file) => [file.path, file.size]).sort());
+    return (await listed(source)) === (await listed(copy));
+}
+async function copied(place, files) {
+    const staged = `${place.target}.staged`;
+    await rm(staged, { recursive: true, force: true });
+    try {
+        await files.copy(place.source, staged);
+        if (!(await sameFiles(place.source, staged)))
+            throw new Error("archive-copy-incomplete");
+        await rename(staged, place.target);
+    }
+    catch (error) {
+        await rm(staged, { recursive: true, force: true });
+        throw error;
+    }
+    await rm(place.source, { recursive: true, force: true });
+}
+export async function archivedFolder(place, files) {
+    await mkdir(dirname(place.target), { recursive: true });
+    try {
+        await files.move(place.source, place.target);
+        return;
+    }
+    catch (error) {
+        if (failureReason(error) !== "EXDEV")
+            throw error;
+    }
+    await copied(place, files);
+}

package/dist/src/daemon/record-sweep.d.ts

@@ -0,0 +1,24 @@
+import { type RecordFiles } from "./record-archive.ts";
+import { type RetentionSettings, type StoredRun } from "../record-retention.ts";
+import type { HostPorts } from "../service/service.ts";
+export interface SweepSettings {
+    readonly directory: string;
+    readonly retention: RetentionSettings;
+    readonly runs: (runIds: readonly string[]) => Promise<Readonly<Record<string, StoredRun>>>;
+    readonly now: () => number;
+    readonly log: (line: string) => void;
+    readonly files?: RecordFiles;
+}
+export interface RecordSweep {
+    readonly pass: () => Promise<void>;
+}
+export interface RetentionLoop {
+    readonly stop: () => Promise<void>;
+}
+export declare const retentionIntervalMilliseconds = 3600000;
+export declare function archiveFolder(settings: Pick<SweepSettings, "directory" | "retention">): string;
+export declare function recordSweep(settings: SweepSettings): RecordSweep;
+export declare function recordRetention(settings: SweepSettings & {
+    readonly intervalMilliseconds?: number;
+}): Promise<RetentionLoop>;
+export declare function authorityRuns(ports: Pick<HostPorts, "runWait">): SweepSettings["runs"];

package/dist/src/daemon/record-sweep.js

@@ -0,0 +1,147 @@
+import { readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { archivedFolder, archiveReady, failureReason, nodeFiles, storedFiles, } from "./record-archive.js";
+import { folderRunId, retentionVerdict, sharedRecordFolders, } from "../record-retention.js";
+export const retentionIntervalMilliseconds = 3_600_000;
+const kinds = ["github", "journal"];
+const batchSize = 200;
+async function folders(directory) {
+    const found = [];
+    for (const kind of kinds)
+        for (const entry of await readdir(join(directory, kind), {
+            withFileTypes: true,
+        }).catch(() => []))
+            if (entry.isDirectory())
+                found.push(`${kind}/${entry.name}`);
+    return found.filter((path) => !sharedRecordFolders.includes(path)).sort();
+}
+async function newestChange(folder) {
+    return Math.max(Number.NEGATIVE_INFINITY, ...(await storedFiles(folder)).map((file) => file.changedAt));
+}
+async function startedRecord(folder) {
+    try {
+        const value = JSON.parse(await readFile(join(folder, "started.json"), "utf8"));
+        return value;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function examined(directory, path) {
+    const folder = join(directory, path);
+    try {
+        return {
+            path,
+            started: await startedRecord(folder),
+            changedAt: await newestChange(folder),
+        };
+    }
+    catch {
+        return { path, started: undefined, changedAt: Number.NaN };
+    }
+}
+export function archiveFolder(settings) {
+    return (settings.retention.archiveFolder ?? join(settings.directory, "archive"));
+}
+async function removed(settings, path) {
+    const source = join(settings.directory, path);
+    if (settings.retention.action === "delete") {
+        await rm(source, { recursive: true, force: true });
+        return;
+    }
+    await archivedFolder({ source, target: join(archiveFolder(settings), path) }, settings.files ?? nodeFiles);
+}
+function countLine(action, tally) {
+    const { counts, failures } = tally;
+    const moved = action === "delete" ? "deleted" : "archived";
+    const failed = failures.length === 0
+        ? ""
+        : `, failed ${String(failures.length)} (${failures[0] ?? ""})`;
+    return `retention: ${moved} ${String(counts.due)}, kept ${String(counts.kept)}, skipped ${String(counts.skipped)} (active runs), unreadable ${String(counts.unreadable)}${failed}\n`;
+}
+async function unavailable(settings) {
+    if (settings.retention.action === "delete")
+        return false;
+    const reason = await archiveReady(archiveFolder(settings), {
+        configured: settings.retention.archiveFolder !== null,
+    });
+    if (reason === undefined)
+        return false;
+    settings.log(`retention: archive folder not available (${reason}), moved nothing\n`);
+    return true;
+}
+async function handled(settings, folder) {
+    try {
+        await removed(settings, folder.path);
+        folder.tally.counts.due += 1;
+    }
+    catch (error) {
+        folder.tally.failures.push(`${folder.path}: ${failureReason(error)}`);
+    }
+}
+async function nextBatch(sweep) {
+    const found = await folders(sweep.settings.directory);
+    const rest = found.filter((path) => path > sweep.cursor);
+    const batch = (rest.length === 0 ? found : rest).slice(0, batchSize);
+    sweep.cursor = batch.at(-1) ?? "";
+    return batch;
+}
+async function swept(sweep) {
+    const { settings } = sweep;
+    const days = settings.retention.days;
+    if (days === null || (await unavailable(settings)))
+        return;
+    const batch = await Promise.all((await nextBatch(sweep)).map((path) => examined(settings.directory, path)));
+    const runIds = [
+        ...new Set(batch.flatMap((folder) => folderRunId(folder.started) ?? [])),
+    ];
+    const runs = await settings.runs(runIds);
+    const now = settings.now();
+    const tally = {
+        counts: { due: 0, kept: 0, skipped: 0, unreadable: 0 },
+        failures: [],
+    };
+    for (const folder of batch) {
+        const verdict = retentionVerdict(folder, { runs, now, days });
+        if (verdict === "due")
+            await handled(settings, { path: folder.path, tally });
+        else
+            tally.counts[verdict] += 1;
+    }
+    settings.log(countLine(settings.retention.action, tally));
+}
+export function recordSweep(settings) {
+    const sweep = { settings, cursor: "" };
+    return {
+        pass: () => swept(sweep).catch((error) => {
+            settings.log(`retention: failed (${failureReason(error)})\n`);
+        }),
+    };
+}
+export async function recordRetention(settings) {
+    const sweep = recordSweep(settings);
+    await sweep.pass();
+    let running = Promise.resolve();
+    const timer = setInterval(() => {
+        running = running.then(sweep.pass);
+    }, settings.intervalMilliseconds ?? retentionIntervalMilliseconds);
+    timer.unref();
+    return {
+        stop: async () => {
+            clearInterval(timer);
+            await running;
+        },
+    };
+}
+export function authorityRuns(ports) {
+    return async (runIds) => {
+        const found = [];
+        for (const runId of runIds) {
+            const target = await ports.runWait?.target(runId);
+            if (target === undefined)
+                continue;
+            found.push([runId, { status: target.status, endedAt: target.endedAt }]);
+        }
+        return Object.fromEntries(found);
+    };
+}

package/dist/src/daemon/resume-index-lock.d.ts

@@ -0,0 +1,13 @@
+interface LockIdentity {
+    readonly path: string;
+    readonly dev: number;
+    readonly ino: number;
+    readonly ctimeMs: number;
+    readonly mtimeMs: number;
+}
+export interface OwnedIndexLock extends LockIdentity {
+    readonly processIds: readonly number[];
+}
+export declare function ownedIndexLock(path: string): Promise<OwnedIndexLock | undefined>;
+export declare function removeStoppedIndexLock(lock: OwnedIndexLock, signal: AbortSignal): Promise<boolean>;
+export {};

package/dist/src/daemon/resume-index-lock.js

@@ -0,0 +1,160 @@
+import { execFile } from "node:child_process";
+import { lstat, readdir, readFile, stat, unlink } from "node:fs/promises";
+import { promisify } from "node:util";
+const execute = promisify(execFile);
+function disappeared(error) {
+    return (error instanceof Error &&
+        "code" in error &&
+        (error.code === "ENOENT" || error.code === "ESRCH"));
+}
+async function descriptorUsesLock(path, lock) {
+    try {
+        const opened = await stat(path);
+        if (opened.dev !== lock.dev || opened.ino !== lock.ino)
+            return false;
+        const information = await readFile(path.replace("/fd/", "/fdinfo/"), "utf8");
+        const flags = /^flags:\s+([0-7]+)$/m.exec(information)?.[1];
+        const inode = /^ino:\s+(\d+)$/m.exec(information)?.[1];
+        if (flags === undefined || Number(inode) !== lock.ino)
+            return undefined;
+        return (Number.parseInt(flags, 8) & 3) !== 0;
+    }
+    catch (error) {
+        return disappeared(error) ? false : undefined;
+    }
+}
+async function processUsesLock(directory, lock) {
+    try {
+        for (const descriptor of await readdir(`${directory}/fd`)) {
+            const used = await descriptorUsesLock(`${directory}/fd/${descriptor}`, lock);
+            if (used !== false)
+                return used;
+        }
+        return false;
+    }
+    catch (error) {
+        if (disappeared(error))
+            return false;
+        if (!(error instanceof Error) ||
+            !("code" in error) ||
+            (error.code !== "EACCES" && error.code !== "EPERM"))
+            return undefined;
+        return (await unrelatedProcess(directory, lock)) ? false : undefined;
+    }
+}
+async function unrelatedProcess(directory, lock) {
+    if (lock.uid !== process.geteuid?.() || (lock.mode & 0o022) !== 0)
+        return false;
+    try {
+        const status = await readFile(`${directory}/status`, "utf8");
+        const users = /^Uid:\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)$/m.exec(status);
+        return (users !== null &&
+            users
+                .slice(1)
+                .map(Number)
+                .every((user) => Number.isSafeInteger(user) && user !== lock.uid));
+    }
+    catch {
+        return false;
+    }
+}
+async function lockOwners(lock) {
+    if (process.platform === "linux") {
+        const processIds = [];
+        for (const entry of await readdir("/proc")) {
+            if (!/^\d+$/.test(entry))
+                continue;
+            const used = await processUsesLock(`/proc/${entry}`, lock);
+            if (used === undefined)
+                return undefined;
+            if (used)
+                processIds.push(Number(entry));
+        }
+        return processIds;
+    }
+    try {
+        const { stdout, stderr } = await execute("lsof", ["-Fpfa", "--", lock.path], {
+            encoding: "utf8",
+            timeout: 5000,
+            maxBuffer: 8 * 1024 * 1024,
+        });
+        return stderr === "" ? writableOwners(stdout) : undefined;
+    }
+    catch {
+        return undefined;
+    }
+}
+function writableOwner(record) {
+    const [header, ...descriptors] = record.trim().split(/^f/m);
+    const processId = Number(header?.trim().slice(1));
+    const access = descriptors.map((descriptor) => descriptor.split("\n")[1]);
+    if (!Number.isSafeInteger(processId) ||
+        processId < 1 ||
+        descriptors.length === 0 ||
+        access.some((mode) => mode !== "ar" && mode !== "aw" && mode !== "au"))
+        return undefined;
+    return access.some((mode) => mode === "aw" || mode === "au")
+        ? processId
+        : null;
+}
+function writableOwners(text) {
+    const owners = text
+        .trim()
+        .split(/(?=^p)/m)
+        .map(writableOwner);
+    return owners.includes(undefined)
+        ? undefined
+        : [
+            ...new Set(owners.filter((owner) => typeof owner === "number")),
+        ];
+}
+export async function ownedIndexLock(path) {
+    try {
+        const identity = await lstat(path);
+        if (!identity.isFile())
+            return undefined;
+        const lock = {
+            path,
+            dev: identity.dev,
+            ino: identity.ino,
+            ctimeMs: identity.ctimeMs,
+            mtimeMs: identity.mtimeMs,
+            uid: identity.uid,
+            mode: identity.mode,
+        };
+        const processIds = await lockOwners(lock);
+        return processIds === undefined || processIds.length === 0
+            ? undefined
+            : { ...lock, processIds };
+    }
+    catch {
+        return undefined;
+    }
+}
+function stopped(processId) {
+    try {
+        process.kill(processId, 0);
+        return false;
+    }
+    catch (error) {
+        return error instanceof Error && "code" in error && error.code === "ESRCH";
+    }
+}
+export async function removeStoppedIndexLock(lock, signal) {
+    try {
+        if (!lock.processIds.every(stopped))
+            return false;
+        const after = await lstat(lock.path);
+        if (signal.aborted ||
+            lock.dev !== after.dev ||
+            lock.ino !== after.ino ||
+            lock.ctimeMs !== after.ctimeMs ||
+            lock.mtimeMs !== after.mtimeMs)
+            return false;
+        await unlink(lock.path);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/src/daemon/resume-worktree.d.ts

@@ -0,0 +1,7 @@
+import { type IndexFailure } from "./retry-worktree.ts";
+export declare function resumeWorktree(request: {
+    readonly directory: string;
+    readonly folder: string;
+    readonly signal: AbortSignal;
+    readonly stopMilliseconds?: number;
+}): Promise<IndexFailure | null>;

package/dist/src/daemon/resume-worktree.js

@@ -0,0 +1,29 @@
+import { ownedIndexLock, removeStoppedIndexLock } from "./resume-index-lock.js";
+import { stopTreeProcesses } from "./worktree-processes.js";
+import { indexFailure } from "./retry-worktree.js";
+async function inspectLock(failure) {
+    if (failure?.lockPath === undefined)
+        return undefined;
+    return ownedIndexLock(failure.lockPath);
+}
+export async function resumeWorktree(request) {
+    const aborted = () => request.signal.aborted;
+    const before = await indexFailure(request);
+    const lock = await inspectLock(before);
+    if (aborted())
+        return before;
+    const stopped = await stopTreeProcesses(request);
+    const locked = await indexFailure(request);
+    if (!stopped)
+        return (locked ?? {
+            reason: "worktree-index-locked",
+            fix: "Stop the remaining work tree processes before resuming the turn.",
+        });
+    if (locked?.reason !== "worktree-index-locked" || aborted())
+        return locked;
+    if (lock === undefined ||
+        lock.path !== locked.lockPath ||
+        !(await removeStoppedIndexLock(lock, request.signal)))
+        return locked;
+    return indexFailure(request);
+}

package/dist/src/daemon/retrying-turn.d.ts

@@ -1,14 +1,4 @@
-import { type TokenPrices } from "../token-prices.ts";
 import { type RunningTurn, type TurnStart } from "./claude-turn.ts";
-export interface RetrySettings {
-    readonly tokenPrices?: TokenPrices;
-    readonly fallbackModels?: readonly string[];
-    readonly retryDelay?: (milliseconds: number, signal: AbortSignal) => Promise<void>;
-}
-interface RetryOptions extends RetrySettings {
-    readonly stopping?: AbortController;
-    readonly stopMilliseconds?: number;
-    readonly metadata: Readonly<Record<string, unknown>>;
-}
+import { type RetryOptions } from "./turn-attempts.ts";
+export type { RetrySettings } from "./turn-attempts.ts";
 export declare function startRetriedTurn(start: TurnStart, options: RetryOptions): Promise<RunningTurn>;
-export {};