cloving 1.3.0 → 1.5.0

package/dist/src/daemon/daemon.js

@@ -1,4 +1,4 @@
-import { adapters, recoverCommands, } from "./daemon-adapters.js";
+import { adapters, recoverCommands, recoverClaude, } from "./daemon-adapters.js";
 import { mkdir, realpath } from "node:fs/promises";
 import { join } from "node:path";
 import { registerStepBridge, sdkStepCall, stepBridge, } from "../host/step-bridge-host.js";
@@ -12,8 +12,14 @@ import { commandEffects } from "./command-journal.js";
 import { worldDirectory } from "./host-environment.js";
 import { pinnedWorkflowHandler } from "./workflow-replay.js";
 import { scheduledDeclarations } from "./workflow-schedules.js";
+import { scheduleConnection } from "./schedule-control.js";
+import { queueConnection } from "./queue-control.js";
 import { daemonLoops } from "./daemon-loops.js";
-import { ticketRegistrations, ticketRegistrationTable, issueWorkflow, } from "./ticket-registration.js";
+import { authorityRuns, recordRetention } from "./record-sweep.js";
+import { turnDetailsPlace } from "./turn-details.js";
+import { defaultRetention } from "../record-retention.js";
+import { clovingVersion } from "../cli/toolchain.js";
+import { daemonRegistrations, queueWorkflowId, workflowReload, } from "./workflow-reload.js";
 export const turnLeaseMilliseconds = 4 * 3_600_000;
 export const loopMilliseconds = 250;
 export const bridgeTiming = {
@@ -65,45 +71,6 @@ async function hosted(settings, hosting) {
         },
     };
 }
-async function daemonRegistrations(settings) {
-    const conflict = settings.workflowRefusals?.find((refusal) => refusal.codes.includes("queue-workflow-conflict"));
-    if (conflict !== undefined)
-        return {
-            table: [],
-            admitted: {
-                kind: "rejected",
-                detail: `queue-workflow-conflict: ${conflict.scope} ${conflict.label}`,
-            },
-        };
-    const table = ticketRegistrationTable({
-        repository: settings.repository,
-        command: settings.ticketCommand,
-        nextCommand: settings.nextCommand,
-        githubCommand: settings.githubCommand,
-    }, settings.declarations);
-    const admitted = await ticketRegistrations(settings.manifests, {
-        table,
-        commands: settings.commands ?? {},
-        ...(settings.bundleDigest === undefined
-            ? {}
-            : { bundleDigest: settings.bundleDigest }),
-    });
-    return {
-        table,
-        admitted: admitted.kind === "rejected"
-            ? admitted
-            : {
-                ...admitted,
-                registrations: admitted.registrations.map((registration) => ({
-                    ...registration,
-                    ...(settings.queueMilliseconds !== undefined &&
-                        registration.workflowId === queueWorkflowId(settings)
-                        ? { admission: settings.queueLanes ?? 1, queue: true }
-                        : {}),
-                })),
-            },
-    };
-}
 export async function startDaemon(givenSettings) {
     const repository = await realpath(givenSettings.repository);
     const scheduled = scheduledDeclarations(givenSettings, {
@@ -118,16 +85,56 @@ export async function startDaemon(givenSettings) {
         repository,
     });
 }
+function controlConnections() {
+    return { schedules: scheduleConnection(), queues: queueConnection() };
+}
+function retentionReads(settings, ports) {
+    return turnDetailsPlace({
+        directory: settings.directory,
+        retention: settings.retention ?? defaultRetention,
+        runs: async (runIds) => authorityRuns(await ports)(runIds),
+        now: settings.retentionClock ?? Date.now,
+    });
+}
+function rejectedOpening(detail) {
+    return {
+        kind: "refused",
+        reason: detail.startsWith("queue-workflow-conflict")
+            ? "queue-workflow-conflict"
+            : "workflow-rejected",
+        detail,
+    };
+}
+function workflowOptions(settings, parts) {
+    const { reloadCommand, loadBundle } = settings;
+    const { generations, turnSettings } = parts;
+    return {
+        workflowBundles: {
+            directory: join(settings.directory, "workflow-bundles"),
+            repository: settings.repository,
+        },
+        workflowRefusals: settings.workflowRefusals ?? [],
+        retainedWorkflow: generations.available,
+        ...(settings.bundleDigest === undefined
+            ? {}
+            : { bundleDigest: settings.bundleDigest }),
+        ...(reloadCommand === undefined || loadBundle === undefined
+            ? {}
+            : {
+                reload: workflowReload({
+                    ...settings,
+                    reloadCommand,
+                    loadBundle,
+                    generations,
+                    turnSettings,
+                }),
+            }),
+    };
+}
 async function openDaemon(settings) {
     const { table, admitted } = await daemonRegistrations(settings);
     if (admitted.kind === "rejected")
-        return {
-            kind: "refused",
-            reason: admitted.detail.startsWith("queue-workflow-conflict")
-                ? "queue-workflow-conflict"
-                : "workflow-rejected",
-            detail: admitted.detail,
-        };
+        return rejectedOpening(admitted.detail);
     const given = Promise.withResolvers();
     const generations = workflowGenerations({
         ...settings,
@@ -138,18 +145,21 @@ async function openDaemon(settings) {
     const { worker, github, runs, handoff, commands } = resources;
     await mkdir(join(settings.directory, "state"), { recursive: true });
     await mkdir(settings.runtimeDirectory, { recursive: true, mode: 0o700 });
+    const controls = controlConnections();
     const opening = await startService({
+        version: clovingVersion,
+        schedule: controls.schedules.request,
+        queue: controls.queues.port,
+        ...(settings.queueDailySpendLimitUsd === undefined
+            ? {}
+            : { queueDailySpendLimitUsd: settings.queueDailySpendLimitUsd }),
         workerHarness: worker.harness,
+        turnDetails: retentionReads(settings, given.promise),
         commandEffects: commandEffects(join(settings.directory, "commands")),
         stateDirectory: join(settings.directory, "state"),
         runtimeDirectory: settings.runtimeDirectory,
-        workflowBundles: {
-            directory: join(settings.directory, "workflow-bundles"),
-            repository: settings.repository,
-        },
-        workflowRefusals: settings.workflowRefusals ?? [],
+        ...workflowOptions(settings, { ...resources, generations }),
         workflows: admitted.registrations,
-        retainedWorkflow: generations.available,
         delivery: deliveryRouter({
             runs,
             effects: { claude: worker, github, handoff, command: commands },
@@ -164,16 +174,18 @@ async function openDaemon(settings) {
         await closeWorkers(resources);
         return opening;
     }
-    const running = { ...resources, service: opening.service };
+    const running = { ...resources, ...controls, service: opening.service };
     await recoverWorkers(running, given.promise);
     return {
         kind: "started",
-        daemon: await runningDaemon(settings, running),
+        daemon: await runningDaemon(settings, { ...running, ports: given.promise }),
     };
 }
 async function recoverWorkers(resources, ports) {
     try {
-        await recoverCommands(resources.commands, await ports);
+        const host = await ports;
+        await recoverCommands(resources.commands, host);
+        await recoverClaude(resources.worker, host);
     }
     catch (error) {
         await Promise.allSettled([
@@ -192,12 +204,22 @@ async function closeWorkers(resources) {
 }
 async function runningDaemon(settings, resources) {
     const { service, github } = resources;
+    const retention = await recordRetention({
+        directory: settings.directory,
+        retention: settings.retention ?? defaultRetention,
+        runs: authorityRuns(await resources.ports),
+        now: settings.retentionClock ?? Date.now,
+        log: (line) => process.stderr.write(line),
+    });
     await github.scan();
     const loops = await daemonLoops({
         ...settings,
+        scheduleConnection: resources.schedules,
+        queueConnection: resources.queues,
         submit: service.worker.submit,
         queueWorkflowId: queueWorkflowId(settings),
     }).catch(async (error) => {
+        await retention.stop();
         await closeWorkers(resources);
         await service.close();
         throw error;
@@ -205,13 +227,10 @@ async function runningDaemon(settings, resources) {
     return {
         socket: service.socket,
         close: async () => {
+            await retention.stop();
             await loops.stop();
             await closeWorkers(resources);
             await service.close();
         },
     };
 }
-function queueWorkflowId(settings) {
-    return (settings.declarations?.find((declaration) => declaration.queue === true)
-        ?.workflowId ?? issueWorkflow);
-}

package/dist/src/daemon/github-attach.d.ts

@@ -0,0 +1,2 @@
+import { type Operation } from "./github-operation.ts";
+export declare function attachAsset(operation: Operation): Promise<unknown>;

package/dist/src/daemon/github-attach.js

@@ -0,0 +1,222 @@
+import { rm, writeFile } from "node:fs/promises";
+import { join } from "node:path";
+import { copiedAttachment, openedAttachment } from "./attachment-file.js";
+import { attachmentFolder } from "./attachments.js";
+import { repositoryName } from "./github-issue-methods.js";
+import { issuePrivacy, operatorPrivateTexts } from "./github-issue-privacy.js";
+import { field, repositoryPath, text, } from "./github-operation.js";
+import { githubRead } from "./github-retry.js";
+import { inTurn } from "./merge-turns.js";
+function isList(value) {
+    return Array.isArray(value);
+}
+function validName(name) {
+    return /^[\w-][\w.-]*\.(?:mp4|webm|png)$/.test(name) && !name.includes("..");
+}
+function validRelease(release) {
+    return /^[a-z][a-z0-9-]{0,63}$/.test(release) && !/^v\d/.test(release);
+}
+function placeOf(operation) {
+    const place = operation.payload["place"] ?? "comment";
+    if (place !== "comment" && place !== "description")
+        throw new Error("invalid-payload");
+    return place;
+}
+function attachmentOf(operation) {
+    const number = operation.payload["number"];
+    const name = text(operation, "name");
+    const release = text(operation, "release");
+    const comment = text(operation, "comment");
+    if (typeof number !== "number" || !Number.isSafeInteger(number) || number < 1)
+        throw new Error("invalid-payload");
+    if (!validName(name) || !validRelease(release) || comment.length > 4000)
+        throw new Error("invalid-payload");
+    return { number, name, release, comment, place: placeOf(operation) };
+}
+function attachedFile(operation, name) {
+    return openedAttachment({
+        folder: attachmentFolder({
+            worktrees: operation.settings.worktrees,
+            target: operation.target,
+        }),
+        name,
+    });
+}
+async function pages(operation, path) {
+    const reply = JSON.parse(await githubRead(operation, ["api", "--paginate", "--slurp", path]));
+    if (!isList(reply))
+        throw new Error("github-call-failed");
+    return reply.flatMap((page) => {
+        if (!isList(page))
+            throw new Error("github-call-failed");
+        return page;
+    });
+}
+async function releaseOf(operation, tag) {
+    const releases = await pages(operation, `${repositoryPath}/releases?per_page=100`);
+    return releases.find((item) => field(item, "tag_name") === tag);
+}
+async function ensuredRelease(operation, release) {
+    if ((await releaseOf(operation, release)) !== undefined)
+        return;
+    try {
+        await operation.calls.gh([
+            "release",
+            "create",
+            release,
+            "--prerelease",
+            "--latest=false",
+            "--title",
+            "Pull request demos",
+            "--notes",
+            "Demo videos of pull requests. Cloving uploads them.",
+        ]);
+    }
+    catch (error) {
+        if ((await releaseOf(operation, release)) === undefined)
+            throw error;
+    }
+}
+async function uploaded(operation, attachment) {
+    const { release } = attachment;
+    await ensuredRelease(operation, release);
+    const asset = `pr-${String(attachment.number)}-${attachment.name}`;
+    const copy = join(operation.folder, asset);
+    try {
+        await copiedAttachment(attachment.file, copy);
+        await operation.calls.gh(["release", "upload", release, copy, "--clobber"]);
+    }
+    finally {
+        await rm(copy, { force: true });
+    }
+    const assets = field(await releaseOf(operation, release), "assets");
+    const found = isList(assets)
+        ? assets.find((item) => field(item, "name") === asset)
+        : undefined;
+    const url = field(found, "browser_download_url");
+    if (typeof url !== "string" || !url.startsWith("https://"))
+        throw new Error("github-call-failed");
+    return url;
+}
+async function ownLogin(operation) {
+    const user = JSON.parse(await githubRead(operation, ["api", "user"]));
+    const login = field(user, "login");
+    if (typeof login !== "string" || login === "")
+        throw new Error("github-call-failed");
+    return login;
+}
+async function commented(operation, attachment) {
+    const { number, name, url } = attachment;
+    const marker = `<!-- cloving-attachment:${String(number)}:${name} -->`;
+    const body = `${attachment.comment}\n\n[${name}](${url})\n\n${marker}`;
+    const comments = await pages(operation, `${repositoryPath}/issues/${String(number)}/comments?per_page=100`);
+    const login = await ownLogin(operation);
+    const known = comments.find((item) => {
+        const existing = field(item, "body");
+        return (typeof existing === "string" &&
+            existing.includes(marker) &&
+            field(field(item, "user"), "login") === login);
+    });
+    const id = field(known, "id");
+    if (typeof id === "number") {
+        await operation.calls.gh([
+            "api",
+            "-X",
+            "PATCH",
+            `${repositoryPath}/issues/comments/${String(id)}`,
+            "-f",
+            `body=${body}`,
+        ]);
+        return;
+    }
+    await operation.calls.gh([
+        "api",
+        `${repositoryPath}/issues/${String(number)}/comments`,
+        "-f",
+        `body=${body}`,
+    ]);
+}
+function withSection(body, marks) {
+    const from = body.indexOf(marks.start);
+    const to = from < 0 ? -1 : body.indexOf(marks.end, from);
+    if (to >= 0)
+        return (body.slice(0, from) + marks.section + body.slice(to + marks.end.length));
+    const kept = body.trimEnd();
+    return kept === "" ? marks.section : `${kept}\n\n${marks.section}`;
+}
+async function pullBody(operation, number) {
+    const pull = JSON.parse(await githubRead(operation, [
+        "api",
+        `${repositoryPath}/pulls/${String(number)}`,
+    ]));
+    const body = field(pull, "body");
+    return typeof body === "string" ? body : "";
+}
+function attachedUrl(body, marks) {
+    const from = body.indexOf(marks.start);
+    const to = from < 0 ? -1 : body.indexOf(marks.end, from);
+    const url = to < 0 ? "" : (body.slice(from, to).trimEnd().split("\n").at(-1) ?? "");
+    if (!/^https:\/\/github\.com\/user-attachments\/assets\/[\w-]+$/.test(url))
+        throw new Error("github-call-failed");
+    return url;
+}
+async function described(operation, attachment) {
+    const { number, name } = attachment;
+    const start = `<!-- cloving-attachment:${String(number)}:${name} -->`;
+    const end = `<!-- /cloving-attachment:${String(number)}:${name} -->`;
+    const section = `${start}\n${attachment.comment}\n\n![${name}](./${name})\n${end}`;
+    const copy = join(operation.folder, name);
+    const text = join(operation.folder, "body.md");
+    const turn = {
+        key: `${operation.settings.repository}#pull-${String(number)}`,
+        signal: operation.signal,
+    };
+    try {
+        await copiedAttachment(attachment.file, copy);
+        await inTurn(turn, async () => {
+            const current = await pullBody(operation, number);
+            await writeFile(text, withSection(current, { start, end, section }), "utf8");
+            await operation.calls.gh([
+                "pr",
+                "edit",
+                String(number),
+                "--body-file",
+                text,
+                "--attach",
+                copy,
+            ]);
+        });
+    }
+    finally {
+        await rm(copy, { force: true });
+        await rm(text, { force: true });
+    }
+    return attachedUrl(await pullBody(operation, number), {
+        start,
+        end,
+    });
+}
+async function delivered(operation, attachment) {
+    if (attachment.place === "description")
+        return await described(operation, attachment);
+    const url = await uploaded(operation, attachment);
+    await commented(operation, { ...attachment, url });
+    return url;
+}
+export async function attachAsset(operation) {
+    const attachment = attachmentOf(operation);
+    operatorPrivateTexts(operation.settings);
+    const repository = repositoryName(await operation.calls.gh(["repo", "view", "--json", "nameWithOwner"]));
+    issuePrivacy(attachment.comment, {
+        settings: operation.settings,
+        repository,
+    });
+    const file = await attachedFile(operation, attachment.name);
+    try {
+        const url = await delivered(operation, { ...attachment, file });
+        return { number: attachment.number, url };
+    }
+    finally {
+        await file.close();
+    }
+}

package/dist/src/daemon/github-calls.js

@@ -24,7 +24,7 @@ function failureDetail(error, place) {
     const text = typeof stderr === "string" ? stderr : "";
     return {
         detail: githubText(text, place).slice(-2_000),
-        status: Number(/\(HTTP (\d{3})\)/.exec(text)?.[1]),
+        status: Number(/\bHTTP (\d{3})\b/.exec(text)?.[1]),
     };
 }
 export function githubText(text, settings) {

package/dist/src/daemon/github-ci-filters.d.ts

@@ -0,0 +1,6 @@
+export declare function triggerKind(trigger: unknown): "push" | "event";
+export declare function pullRequestTrigger(source: string): unknown;
+export declare function workflowApplies(trigger: unknown, context: {
+    readonly branch: string;
+    readonly paths: readonly string[];
+}): boolean;

package/dist/src/daemon/github-ci-filters.js

@@ -0,0 +1,138 @@
+import { parse } from "yaml";
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+export function triggerKind(trigger) {
+    if (!isRecord(trigger))
+        return "push";
+    const types = patterns(trigger["types"]);
+    return types === undefined || types.includes("synchronize")
+        ? "push"
+        : "event";
+}
+export function pullRequestTrigger(source) {
+    try {
+        const document = parse(source, { merge: true });
+        if (!isRecord(document) || !("on" in document))
+            return true;
+        const events = document["on"];
+        if (typeof events === "string")
+            return events === "pull_request";
+        if (Array.isArray(events))
+            return events.includes("pull_request");
+        if (isRecord(events))
+            return "pull_request" in events ? events["pull_request"] : false;
+        return true;
+    }
+    catch {
+        return true;
+    }
+}
+function patternToken(token) {
+    if (token === "**/")
+        return { kind: "directory" };
+    if (token === "*" || token === "**")
+        return {
+            kind: "character",
+            matcher: token === "*" ? /[^/]/ : /[\s\S]/,
+            optional: true,
+            repeat: true,
+        };
+    const literal = token.startsWith("\\") ? token.slice(1) : token;
+    const source = token.startsWith("[")
+        ? token
+        : literal.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+    return {
+        kind: "character",
+        matcher: new RegExp(`^${source}$`),
+        optional: false,
+        repeat: false,
+    };
+}
+function patternTokens(source) {
+    const tokens = [];
+    for (const raw of source.match(/\*\*\/|\*\*|\*|\[[a-zA-Z0-9-]+\]|\\[\s\S]|[\s\S]/g) ?? []) {
+        if (raw !== "?" && raw !== "+") {
+            tokens.push(patternToken(raw));
+            continue;
+        }
+        const previous = tokens.pop();
+        if (previous?.kind !== "character" || previous.optional || previous.repeat)
+            throw new Error("invalid-workflow-filter");
+        tokens.push({ ...previous, optional: raw === "?", repeat: raw === "+" });
+    }
+    return tokens;
+}
+function advanceDirectory({ value, previous, }) {
+    let reachable = false;
+    return Array.from({ length: value.length + 1 }, (_, index) => {
+        const matches = previous[index] === true || (reachable && value[index - 1] === "/");
+        reachable ||= previous[index] === true;
+        return matches;
+    });
+}
+function advancePattern(token, input) {
+    const { value, previous } = input;
+    const next = [];
+    for (let index = 0; index <= value.length; index += 1) {
+        const consumes = index > 0 && token.matcher.test(value[index - 1] ?? "");
+        next[index] =
+            (token.optional && previous[index] === true) ||
+                (consumes &&
+                    (previous[index - 1] === true ||
+                        (token.repeat && next[index - 1] === true)));
+    }
+    return next;
+}
+function matchesPattern(value, source) {
+    let previous = [true];
+    for (const token of patternTokens(source))
+        previous =
+            token.kind === "directory"
+                ? advanceDirectory({ value, previous })
+                : advancePattern(token, { value, previous });
+    return previous[value.length] === true;
+}
+function selected(value, patterns) {
+    let included = false;
+    for (const pattern of patterns) {
+        const negative = pattern.startsWith("!");
+        const source = negative ? pattern.slice(1) : pattern;
+        try {
+            if (matchesPattern(value, source))
+                included = !negative;
+        }
+        catch {
+            return undefined;
+        }
+    }
+    return included;
+}
+function patterns(value) {
+    if (!Array.isArray(value) || value.length === 0)
+        return undefined;
+    return value.every((item) => typeof item === "string")
+        ? value
+        : undefined;
+}
+function allows(values, filters) {
+    const include = patterns(filters.include);
+    const ignore = patterns(filters.ignore);
+    if (include !== undefined &&
+        !values.some((value) => selected(value, include) !== false))
+        return false;
+    return (ignore === undefined ||
+        values.some((value) => selected(value, ignore) !== true));
+}
+export function workflowApplies(trigger, context) {
+    if (!isRecord(trigger))
+        return trigger !== false;
+    return (allows([context.branch], {
+        include: trigger["branches"],
+        ignore: trigger["branches-ignore"],
+    }) &&
+        allows(context.paths, {
+            include: trigger["paths"],
+            ignore: trigger["paths-ignore"],
+        }));
+}

package/dist/src/daemon/github-ci-workflows.d.ts

@@ -0,0 +1,10 @@
+import { type Operation } from "./github-operation.ts";
+export interface ExpectedWorkflow {
+    readonly path: string;
+    readonly kind: "push" | "event";
+}
+export declare function pullRequestWorkflows(operation: Operation, commit: string): Promise<readonly ExpectedWorkflow[]>;
+export declare function pendingWorkflows(operation: Operation, input: {
+    readonly commit: string;
+    readonly workflows: readonly ExpectedWorkflow[];
+}): Promise<readonly string[]>;