cloving 1.4.0 → 1.5.0

package/dist/src/daemon/queue-control.js

@@ -0,0 +1,86 @@
+import { queueControlLimit } from "./queue-journal.js";
+const unavailable = {
+    kind: "unavailable",
+    detail: "The queue is not running.",
+};
+export function queueConnection() {
+    let connected;
+    return {
+        port: {
+            control: () => connected?.control(),
+            change: (request) => connected?.change(request) ?? Promise.resolve(unavailable),
+        },
+        connect: (port) => {
+            connected = port;
+        },
+    };
+}
+export function queueStopped(runtime, error) {
+    const reason = error instanceof Error ? error.message : String(error);
+    runtime.error = reason;
+    runtime.stopping.abort();
+    process.stderr.write(`Queue stopped: ${JSON.stringify(reason)}\n`);
+    return reason;
+}
+function stoppedChange(error) {
+    return {
+        kind: "unavailable",
+        detail: `The queue stopped: ${error}. Restart the service.`,
+    };
+}
+function recorded(pending, request) {
+    const prior = pending.controls.find((control) => control.commandId === request.commandId);
+    if (prior === undefined)
+        return undefined;
+    return {
+        kind: prior.command === request.command ? "replayed" : "conflicting-command",
+    };
+}
+function applied(pending, command) {
+    if (command === "drain") {
+        pending.drain ??= { startedAt: Date.now(), drainedAt: null };
+        return;
+    }
+    pending.failures.length = 0;
+    pending.drain = undefined;
+}
+async function changed(runtime, request) {
+    const { pending, error } = runtime;
+    if (error !== undefined)
+        return stoppedChange(error);
+    if (pending === undefined)
+        return unavailable;
+    const prior = recorded(pending, request);
+    if (prior !== undefined)
+        return prior;
+    applied(pending, request.command);
+    pending.controls.push(request);
+    pending.controls.splice(0, pending.controls.length - queueControlLimit);
+    try {
+        await pending.save();
+        return { kind: "changed" };
+    }
+    catch (failure) {
+        return stoppedChange(queueStopped(runtime, failure));
+    }
+}
+export function queueControlPort(runtime) {
+    let changing = Promise.resolve();
+    return {
+        control: () => {
+            const { pending, error } = runtime;
+            if (pending === undefined && error === undefined)
+                return undefined;
+            return {
+                failedRunIds: [...(pending?.failures ?? [])],
+                drain: pending?.drain,
+                error,
+            };
+        },
+        change: (request) => {
+            const next = changing.then(() => changed(runtime, request));
+            changing = next;
+            return next;
+        },
+    };
+}

package/dist/src/daemon/queue-journal.d.ts

@@ -0,0 +1,20 @@
+import { type QueueCommand, type QueueDrain } from "../queue-state.ts";
+export declare const queueControlLimit = 32;
+export interface QueueControlRecord {
+    readonly commandId: string;
+    readonly command: QueueCommand;
+}
+export interface SelfUpdateHold {
+    readonly mark: string;
+    readonly startedAt: number;
+}
+export interface QueueJournal {
+    readonly commands: Set<string>;
+    readonly failures: string[];
+    readonly controls: QueueControlRecord[];
+    readonly save: () => Promise<void>;
+    retryCommandId: string | undefined;
+    selfUpdateHold: SelfUpdateHold | undefined;
+    drain: QueueDrain | undefined;
+}
+export declare function queueJournal(directory: string): Promise<QueueJournal>;

package/dist/src/daemon/queue-journal.js

@@ -0,0 +1,89 @@
+import { readFile, rename, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { record } from "../protocol.js";
+import { queueFailureLimit, } from "../queue-state.js";
+export const queueControlLimit = 32;
+function savedHold(value) {
+    if (value === undefined)
+        return undefined;
+    const held = record(value);
+    if (typeof held?.["mark"] !== "string" ||
+        typeof held["startedAt"] !== "number" ||
+        !Number.isFinite(held["startedAt"]) ||
+        held["startedAt"] <= 0)
+        throw new Error("invalid-queue-commands");
+    return { mark: held["mark"], startedAt: held["startedAt"] };
+}
+function instant(value) {
+    return typeof value === "number" && Number.isFinite(value) && value > 0;
+}
+function savedDrain(value) {
+    if (value === undefined)
+        return undefined;
+    const drain = record(value);
+    const drainedAt = drain?.["drainedAt"];
+    if (!instant(drain?.["startedAt"]) ||
+        !(drainedAt === null || instant(drainedAt)))
+        throw new Error("invalid-queue-commands");
+    return { startedAt: drain["startedAt"], drainedAt };
+}
+function savedControls(value) {
+    if (value === undefined)
+        return [];
+    if (!Array.isArray(value) || value.length > queueControlLimit)
+        throw new Error("invalid-queue-commands");
+    return value.map((entry) => {
+        const control = record(entry);
+        const commandId = control?.["commandId"];
+        const command = control?.["command"];
+        if (typeof commandId !== "string" ||
+            commandId.length === 0 ||
+            (command !== "resume" && command !== "drain"))
+            throw new Error("invalid-queue-commands");
+        return { commandId, command };
+    });
+}
+function identifiers(value) {
+    if (!Array.isArray(value) ||
+        !value.every((id) => typeof id === "string" && id.length > 0))
+        throw new Error("invalid-queue-commands");
+    return value.map(String);
+}
+export async function queueJournal(directory) {
+    const file = join(directory, "queue-commands.json");
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return "[]";
+        throw error;
+    });
+    const value = JSON.parse(text);
+    const saved = record(value);
+    const commands = new Set(identifiers(Array.isArray(value) ? value : saved?.["commands"]));
+    const failures = Array.isArray(value) ? [] : identifiers(saved?.["failures"]);
+    if (failures.length > queueFailureLimit)
+        throw new Error("invalid-queue-commands");
+    let saving = Promise.resolve();
+    const write = async () => {
+        await writeFile(`${file}.tmp`, JSON.stringify({
+            commands: [...commands],
+            failures,
+            selfUpdateHold: pending.selfUpdateHold,
+            drain: pending.drain,
+            controls: pending.controls.length === 0 ? undefined : pending.controls,
+        }) + "\n", { mode: 0o600, flush: true });
+        await rename(`${file}.tmp`, file);
+    };
+    const pending = {
+        commands,
+        failures,
+        controls: savedControls(saved?.["controls"]),
+        retryCommandId: undefined,
+        selfUpdateHold: savedHold(saved?.["selfUpdateHold"]),
+        drain: savedDrain(saved?.["drain"]),
+        save: () => {
+            saving = saving.then(write, write);
+            return saving;
+        },
+    };
+    return pending;
+}

package/dist/src/daemon/record-archive.d.ts

@@ -0,0 +1,19 @@
+export interface RecordFiles {
+    readonly move: (source: string, target: string) => Promise<void>;
+    readonly copy: (source: string, target: string) => Promise<void>;
+}
+export interface StoredFile {
+    readonly path: string;
+    readonly size: number;
+    readonly changedAt: number;
+}
+export declare const nodeFiles: RecordFiles;
+export declare function failureReason(error: unknown): string;
+export declare function storedFiles(folder: string): Promise<StoredFile[]>;
+export declare function archiveReady(folder: string, settings: {
+    readonly configured: boolean;
+}): Promise<string | undefined>;
+export declare function archivedFolder(place: {
+    readonly source: string;
+    readonly target: string;
+}, files: RecordFiles): Promise<void>;

package/dist/src/daemon/record-archive.js

@@ -0,0 +1,82 @@
+import { constants } from "node:fs";
+import { access, cp, lstat, mkdir, readdir, rename, rm, stat, } from "node:fs/promises";
+import { dirname, join, relative } from "node:path";
+export const nodeFiles = {
+    move: rename,
+    copy: (source, target) => cp(source, target, {
+        recursive: true,
+        errorOnExist: true,
+        force: false,
+        preserveTimestamps: true,
+    }),
+};
+export function failureReason(error) {
+    if (typeof error === "object" &&
+        error !== null &&
+        "code" in error &&
+        typeof error.code === "string")
+        return error.code;
+    return error instanceof Error ? error.message : String(error);
+}
+export async function storedFiles(folder) {
+    const found = [];
+    for (const entry of await readdir(folder, {
+        withFileTypes: true,
+        recursive: true,
+    })) {
+        if (!entry.isFile())
+            continue;
+        const path = join(entry.parentPath, entry.name);
+        const file = await lstat(path);
+        found.push({
+            path: relative(folder, path),
+            size: file.size,
+            changedAt: file.mtimeMs,
+        });
+    }
+    return found;
+}
+export async function archiveReady(folder, settings) {
+    try {
+        if (!settings.configured)
+            await mkdir(folder, { recursive: true });
+        if (!(await stat(folder)).isDirectory())
+            return "not-a-folder";
+        await access(folder, constants.W_OK);
+        return undefined;
+    }
+    catch (error) {
+        return failureReason(error);
+    }
+}
+async function sameFiles(source, copy) {
+    const listed = async (folder) => JSON.stringify((await storedFiles(folder)).map((file) => [file.path, file.size]).sort());
+    return (await listed(source)) === (await listed(copy));
+}
+async function copied(place, files) {
+    const staged = `${place.target}.staged`;
+    await rm(staged, { recursive: true, force: true });
+    try {
+        await files.copy(place.source, staged);
+        if (!(await sameFiles(place.source, staged)))
+            throw new Error("archive-copy-incomplete");
+        await rename(staged, place.target);
+    }
+    catch (error) {
+        await rm(staged, { recursive: true, force: true });
+        throw error;
+    }
+    await rm(place.source, { recursive: true, force: true });
+}
+export async function archivedFolder(place, files) {
+    await mkdir(dirname(place.target), { recursive: true });
+    try {
+        await files.move(place.source, place.target);
+        return;
+    }
+    catch (error) {
+        if (failureReason(error) !== "EXDEV")
+            throw error;
+    }
+    await copied(place, files);
+}

package/dist/src/daemon/record-sweep.d.ts

@@ -0,0 +1,24 @@
+import { type RecordFiles } from "./record-archive.ts";
+import { type RetentionSettings, type StoredRun } from "../record-retention.ts";
+import type { HostPorts } from "../service/service.ts";
+export interface SweepSettings {
+    readonly directory: string;
+    readonly retention: RetentionSettings;
+    readonly runs: (runIds: readonly string[]) => Promise<Readonly<Record<string, StoredRun>>>;
+    readonly now: () => number;
+    readonly log: (line: string) => void;
+    readonly files?: RecordFiles;
+}
+export interface RecordSweep {
+    readonly pass: () => Promise<void>;
+}
+export interface RetentionLoop {
+    readonly stop: () => Promise<void>;
+}
+export declare const retentionIntervalMilliseconds = 3600000;
+export declare function archiveFolder(settings: Pick<SweepSettings, "directory" | "retention">): string;
+export declare function recordSweep(settings: SweepSettings): RecordSweep;
+export declare function recordRetention(settings: SweepSettings & {
+    readonly intervalMilliseconds?: number;
+}): Promise<RetentionLoop>;
+export declare function authorityRuns(ports: Pick<HostPorts, "runWait">): SweepSettings["runs"];

package/dist/src/daemon/record-sweep.js

@@ -0,0 +1,147 @@
+import { readdir, readFile, rm } from "node:fs/promises";
+import { join } from "node:path";
+import { archivedFolder, archiveReady, failureReason, nodeFiles, storedFiles, } from "./record-archive.js";
+import { folderRunId, retentionVerdict, sharedRecordFolders, } from "../record-retention.js";
+export const retentionIntervalMilliseconds = 3_600_000;
+const kinds = ["github", "journal"];
+const batchSize = 200;
+async function folders(directory) {
+    const found = [];
+    for (const kind of kinds)
+        for (const entry of await readdir(join(directory, kind), {
+            withFileTypes: true,
+        }).catch(() => []))
+            if (entry.isDirectory())
+                found.push(`${kind}/${entry.name}`);
+    return found.filter((path) => !sharedRecordFolders.includes(path)).sort();
+}
+async function newestChange(folder) {
+    return Math.max(Number.NEGATIVE_INFINITY, ...(await storedFiles(folder)).map((file) => file.changedAt));
+}
+async function startedRecord(folder) {
+    try {
+        const value = JSON.parse(await readFile(join(folder, "started.json"), "utf8"));
+        return value;
+    }
+    catch {
+        return undefined;
+    }
+}
+async function examined(directory, path) {
+    const folder = join(directory, path);
+    try {
+        return {
+            path,
+            started: await startedRecord(folder),
+            changedAt: await newestChange(folder),
+        };
+    }
+    catch {
+        return { path, started: undefined, changedAt: Number.NaN };
+    }
+}
+export function archiveFolder(settings) {
+    return (settings.retention.archiveFolder ?? join(settings.directory, "archive"));
+}
+async function removed(settings, path) {
+    const source = join(settings.directory, path);
+    if (settings.retention.action === "delete") {
+        await rm(source, { recursive: true, force: true });
+        return;
+    }
+    await archivedFolder({ source, target: join(archiveFolder(settings), path) }, settings.files ?? nodeFiles);
+}
+function countLine(action, tally) {
+    const { counts, failures } = tally;
+    const moved = action === "delete" ? "deleted" : "archived";
+    const failed = failures.length === 0
+        ? ""
+        : `, failed ${String(failures.length)} (${failures[0] ?? ""})`;
+    return `retention: ${moved} ${String(counts.due)}, kept ${String(counts.kept)}, skipped ${String(counts.skipped)} (active runs), unreadable ${String(counts.unreadable)}${failed}\n`;
+}
+async function unavailable(settings) {
+    if (settings.retention.action === "delete")
+        return false;
+    const reason = await archiveReady(archiveFolder(settings), {
+        configured: settings.retention.archiveFolder !== null,
+    });
+    if (reason === undefined)
+        return false;
+    settings.log(`retention: archive folder not available (${reason}), moved nothing\n`);
+    return true;
+}
+async function handled(settings, folder) {
+    try {
+        await removed(settings, folder.path);
+        folder.tally.counts.due += 1;
+    }
+    catch (error) {
+        folder.tally.failures.push(`${folder.path}: ${failureReason(error)}`);
+    }
+}
+async function nextBatch(sweep) {
+    const found = await folders(sweep.settings.directory);
+    const rest = found.filter((path) => path > sweep.cursor);
+    const batch = (rest.length === 0 ? found : rest).slice(0, batchSize);
+    sweep.cursor = batch.at(-1) ?? "";
+    return batch;
+}
+async function swept(sweep) {
+    const { settings } = sweep;
+    const days = settings.retention.days;
+    if (days === null || (await unavailable(settings)))
+        return;
+    const batch = await Promise.all((await nextBatch(sweep)).map((path) => examined(settings.directory, path)));
+    const runIds = [
+        ...new Set(batch.flatMap((folder) => folderRunId(folder.started) ?? [])),
+    ];
+    const runs = await settings.runs(runIds);
+    const now = settings.now();
+    const tally = {
+        counts: { due: 0, kept: 0, skipped: 0, unreadable: 0 },
+        failures: [],
+    };
+    for (const folder of batch) {
+        const verdict = retentionVerdict(folder, { runs, now, days });
+        if (verdict === "due")
+            await handled(settings, { path: folder.path, tally });
+        else
+            tally.counts[verdict] += 1;
+    }
+    settings.log(countLine(settings.retention.action, tally));
+}
+export function recordSweep(settings) {
+    const sweep = { settings, cursor: "" };
+    return {
+        pass: () => swept(sweep).catch((error) => {
+            settings.log(`retention: failed (${failureReason(error)})\n`);
+        }),
+    };
+}
+export async function recordRetention(settings) {
+    const sweep = recordSweep(settings);
+    await sweep.pass();
+    let running = Promise.resolve();
+    const timer = setInterval(() => {
+        running = running.then(sweep.pass);
+    }, settings.intervalMilliseconds ?? retentionIntervalMilliseconds);
+    timer.unref();
+    return {
+        stop: async () => {
+            clearInterval(timer);
+            await running;
+        },
+    };
+}
+export function authorityRuns(ports) {
+    return async (runIds) => {
+        const found = [];
+        for (const runId of runIds) {
+            const target = await ports.runWait?.target(runId);
+            if (target === undefined)
+                continue;
+            found.push([runId, { status: target.status, endedAt: target.endedAt }]);
+        }
+        return Object.fromEntries(found);
+    };
+}

package/dist/src/daemon/retrying-turn.js

@@ -153,10 +153,6 @@ function interruptedChild(attempts, completion) {
         end.result === undefined);
 }
 async function resumedChild(attempts) {
-    const started = await readRecord(attempts.start.folder, "started.json");
-    const resume = await readResume(attempts.start.folder, started);
-    if (resume === undefined)
-        return { ...combined(attempts), reason: "claude-turn-interrupted" };
     const locked = await resumeWorktree({
         directory: attempts.start.directory,
         folder: attempts.start.folder,
@@ -165,6 +161,10 @@ async function resumedChild(attempts) {
     });
     if (locked !== null)
         return { ...combined(attempts), reason: locked.reason };
+    const started = await readRecord(attempts.start.folder, "started.json");
+    const resume = await readResume(attempts.start.folder, started);
+    if (resume === undefined)
+        return { ...combined(attempts), reason: "claude-turn-interrupted" };
     const prepared = await preparedRetry(attempts);
     if (typeof prepared !== "string")
         return prepared;

package/dist/src/daemon/settings-reload.d.ts

@@ -0,0 +1,26 @@
+import { type HarnessSettings } from "../harness-defaults.ts";
+import { type TurnSettings } from "../project-settings.ts";
+import type { SettingChanges } from "../service/workflow-reload.ts";
+import { type TurnProfiles } from "../turn-profile.ts";
+export type SettingDigests = Readonly<Record<string, string>>;
+export interface ReloadedSettings {
+    readonly turnSettings: TurnSettings;
+    readonly settingDigests: SettingDigests;
+}
+export interface TurnSettingsSource {
+    readonly current: () => TurnSettings;
+    readonly replace: (next: TurnSettings) => void;
+}
+export declare function turnSettingsSource(initial: TurnSettings): TurnSettingsSource;
+export declare function reloadedSettings(output: Readonly<Record<string, unknown>> | undefined): ReloadedSettings | undefined;
+export declare function settingChanges(current: SettingDigests, next: SettingDigests): SettingChanges;
+export declare function keptDigests(current: SettingDigests, change: {
+    readonly digests: SettingDigests;
+    readonly reloadedFields: readonly string[];
+}): SettingDigests;
+export declare function profileRefusal(settings: HarnessSettings & {
+    readonly profiles?: TurnProfiles;
+}, turnSettings: TurnSettings): {
+    readonly code: string;
+    readonly detail: string;
+} | undefined;

package/dist/src/daemon/settings-reload.js

@@ -0,0 +1,82 @@
+import { turnHarness } from "../harness-defaults.js";
+import { reloadedSettingFields, } from "../project-settings.js";
+import { record } from "../protocol.js";
+import { parseFallbackModels, parseTokenPrices } from "../token-prices.js";
+import { isProfilesFile } from "../turn-profile.js";
+import { checkedProfiles } from "./worker-profile.js";
+export function turnSettingsSource(initial) {
+    let current = initial;
+    return {
+        current: () => current,
+        replace: (next) => {
+            current = next;
+        },
+    };
+}
+function turnSettingsOf(value) {
+    const fields = record(value) ?? {};
+    const profiles = { profiles: fields["profiles"] };
+    if (!isProfilesFile(profiles))
+        return undefined;
+    try {
+        return {
+            profiles: profiles.profiles,
+            fallbackModels: parseFallbackModels(JSON.stringify(fields["fallbackModels"])),
+            tokenPrices: parseTokenPrices(JSON.stringify(fields["tokenPrices"])),
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+function digestsOf(value) {
+    const fields = record(value);
+    if (fields === undefined)
+        return undefined;
+    const entries = Object.entries(fields);
+    const digests = entries.filter((entry) => typeof entry[1] === "string");
+    return digests.length === entries.length
+        ? Object.fromEntries(digests)
+        : undefined;
+}
+export function reloadedSettings(output) {
+    const turnSettings = turnSettingsOf(output?.["turnSettings"]);
+    const settingDigests = digestsOf(output?.["settingDigests"]);
+    return turnSettings === undefined || settingDigests === undefined
+        ? undefined
+        : { turnSettings, settingDigests };
+}
+export function settingChanges(current, next) {
+    const changed = [...new Set([...Object.keys(current), ...Object.keys(next)])]
+        .filter((field) => current[field] !== next[field])
+        .sort();
+    return {
+        reloadedFields: changed.filter((field) => reloadedSettingFields.includes(field)),
+        restartFields: changed.filter((field) => !reloadedSettingFields.includes(field)),
+    };
+}
+export function keptDigests(current, change) {
+    return {
+        ...current,
+        ...Object.fromEntries(Object.entries(change.digests).filter(([field]) => change.reloadedFields.includes(field))),
+    };
+}
+function newHarness(settings, profiles) {
+    const started = new Set([{}, ...Object.values(settings.profiles ?? {})].map((profile) => turnHarness(settings, profile).worker));
+    return Object.values(profiles)
+        .map((profile) => turnHarness(settings, profile).worker)
+        .find((name) => !started.has(name));
+}
+export function profileRefusal(settings, turnSettings) {
+    try {
+        checkedProfiles(settings, turnSettings.profiles);
+    }
+    catch (error) {
+        const detail = String(Reflect.get(Object(error), "message"));
+        return { code: detail.replace(/:.*$/s, ""), detail };
+    }
+    const added = newHarness(settings, turnSettings.profiles);
+    return added === undefined
+        ? undefined
+        : { code: "harness-no-worker", detail: `harness-no-worker: ${added}` };
+}

package/dist/src/daemon/turn-details.d.ts

@@ -0,0 +1,4 @@
+import { type SweepSettings } from "./record-sweep.ts";
+import { type RunTurns, type TurnDetails } from "../record-retention.ts";
+export type TurnDetailsSettings = Pick<SweepSettings, "directory" | "retention" | "runs" | "now">;
+export declare function turnDetailsPlace(settings: TurnDetailsSettings): (turns: RunTurns) => Promise<TurnDetails | undefined>;

package/dist/src/daemon/turn-details.js

@@ -0,0 +1,33 @@
+import { stat } from "node:fs/promises";
+import { join } from "node:path";
+import { located } from "./operation-journal.js";
+import { archiveFolder } from "./record-sweep.js";
+import { turnDetails, } from "../record-retention.js";
+function absent(error) {
+    return error instanceof Error && "code" in error && error.code === "ENOENT";
+}
+async function present(journal, operationIds) {
+    const found = await Promise.all(operationIds.map((operationId) => stat(located(journal, operationId).folder).then(() => true, (error) => (absent(error) ? false : undefined))));
+    if (found.includes(undefined))
+        return undefined;
+    return found.filter(Boolean).length;
+}
+export function turnDetailsPlace(settings) {
+    return async ({ runId, operationIds }) => {
+        const archive = join(archiveFolder(settings), "journal");
+        const runs = await settings.runs([runId]);
+        const archived = await present(archive, operationIds);
+        const kept = await present(join(settings.directory, "journal"), operationIds);
+        if (archived === undefined || kept === undefined)
+            return undefined;
+        return turnDetails({
+            turns: operationIds.length,
+            kept,
+            archived: archived === 0 ? undefined : archive,
+        }, {
+            run: Object.hasOwn(runs, runId) ? runs[runId] : undefined,
+            retention: settings.retention,
+            now: settings.now(),
+        });
+    };
+}

package/dist/src/daemon/worker-profile.d.ts

@@ -5,4 +5,5 @@ export interface WorkerProfiles extends HarnessSettings {
     readonly profiles?: TurnProfiles;
     readonly workflowId?: (runId: string) => Promise<string | undefined>;
 }
+export declare function checkedProfiles(settings: HarnessSettings, profiles: TurnProfiles): void;
 export declare function selectedProfile(settings: WorkerProfiles, intent: ProtectedIntent): Promise<TurnProfile | string>;

package/dist/src/daemon/worker-profile.js

@@ -20,6 +20,10 @@ function unsupportedProfile(profile, harness) {
         ? "profile-needs-claude"
         : undefined;
 }
+export function checkedProfiles(settings, profiles) {
+    for (const profile of Object.values(profiles))
+        turnHarness(settings, profile);
+}
 export async function selectedProfile(settings, intent) {
     if (intent.method === "run-turn")
         return {};