cloving 0.3.20 → 1.1.0

package/dist/tools/self-host.js

@@ -0,0 +1,261 @@
+import { isProfilesFile } from "../src/turn-profile.js";
+import { parseTokenPrices } from "../src/token-prices.js";
+import { execFileSync, spawn } from "node:child_process";
+import { copyFile, mkdir, readFile, rm, symlink, writeFile, } from "node:fs/promises";
+import { dirname, extname, join, resolve } from "node:path";
+import packageMetadata from "#package" with { type: "json" };
+import { findPackageJSON } from "node:module";
+import { pathToFileURL } from "node:url";
+import { workflowDeclarations, workflowFiles, workflowManifests, } from "./self-host-workflows.js";
+import { isWorkflowManifest } from "./workflow-manifest.js";
+import { selfUpdateMilliseconds } from "./self-update-host.js";
+import { hostedPrograms } from "./self-host-processes.js";
+const product = resolve(import.meta.dirname, "..");
+const flags = process.argv.slice(2);
+function flag(name) {
+    const index = flags.lastIndexOf(name);
+    return index === -1 ? undefined : flags[index + 1];
+}
+function topLevel(given) {
+    try {
+        return execFileSync("git", ["-C", given, "rev-parse", "--show-toplevel"], {
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+        }).trim();
+    }
+    catch {
+        return given;
+    }
+}
+const runtimeArgument = flag("--runtime");
+if (runtimeArgument === undefined)
+    throw new Error("The start program needs --runtime. Use: cloving start");
+const runtime = resolve(runtimeArgument);
+const directory = dirname(runtime);
+const repository = topLevel(resolve(flag("--repository") ?? process.cwd()));
+const updateMilliseconds = await selfUpdateMilliseconds({
+    seconds: process.env["CLOVING_SELF_UPDATE_SECONDS"],
+    repository,
+    product: extname(import.meta.filename) === ".ts" ? topLevel(product) : product,
+});
+const client = flag("--client") ?? "cloving";
+const project = join(directory, "project");
+const concurrency = Number(process.env["CLOVING_HOST_BOUND"] ?? "8");
+const leaseText = process.env["CLOVING_OWNERSHIP_LEASE_SECONDS"];
+const leaseSeconds = leaseText === undefined ? undefined : Number(leaseText);
+const defaultCommand = [
+    "claude",
+    "-p",
+    "--output-format",
+    "stream-json",
+    "--verbose",
+    "--permission-mode",
+    "acceptEdits",
+    "--allowedTools",
+    "Bash,Read,Edit,Write,Glob,Grep,WebSearch,WebFetch",
+    "--max-budget-usd",
+    "5",
+];
+const defaultCodexCommand = [
+    "codex",
+    "exec",
+    "--json",
+    "--dangerously-bypass-approvals-and-sandbox",
+    "--skip-git-repo-check",
+    "-",
+];
+function selectedWorker() {
+    const worker = process.env["CLOVING_WORKER"] ?? "claude";
+    if (worker === "claude" || worker === "codex")
+        return worker;
+    throw new Error(`The daemon did not start: invalid-worker: ${worker}`);
+}
+async function profilesFile() {
+    const path = process.env["CLOVING_PROFILES"];
+    if (path === undefined)
+        return {};
+    try {
+        const value = JSON.parse(await readFile(path, "utf8"));
+        if (isProfilesFile(value))
+            return value.profiles;
+    }
+    catch {
+        throw new Error("The daemon did not start: invalid-profiles");
+    }
+    throw new Error("The daemon did not start: invalid-profiles");
+}
+function queueInterval(commands) {
+    const value = process.env["CLOVING_QUEUE_SECONDS"];
+    if (value === undefined)
+        return undefined;
+    const seconds = Number(value);
+    if (!/^\d+$/.test(value) ||
+        !Number.isInteger(seconds) ||
+        seconds < 10 ||
+        seconds > 3600)
+        throw new Error("The daemon did not start: invalid-queue-seconds");
+    if (!commands.nextCommand?.length)
+        throw new Error("The daemon did not start: queue-needs-next-command");
+    if (!commands.ticketCommand?.length)
+        throw new Error("The daemon did not start: queue-needs-ticket-command");
+    return seconds * 1000;
+}
+function listed(name) {
+    const given = process.env[name];
+    if (given === undefined)
+        return undefined;
+    const list = JSON.parse(given);
+    if (!Array.isArray(list) || !list.every((part) => typeof part === "string"))
+        throw new Error(`${name} is not a JSON list of texts.`);
+    return list.map(String);
+}
+function record(value) {
+    return typeof value === "object" && value !== null ? { ...value } : {};
+}
+function ended(argumentsList, environment) {
+    return new Promise((settle, reject) => {
+        const child = spawn(process.execPath, argumentsList, {
+            cwd: project,
+            env: environment,
+            stdio: "inherit",
+        });
+        for (const signal of ["SIGINT", "SIGTERM"])
+            process.on(signal, () => child.kill(signal));
+        child.once("error", reject);
+        child.once("close", (status) => {
+            settle(status ?? 1);
+        });
+    });
+}
+async function linkedPackages() {
+    const modules = join(project, "node_modules");
+    await mkdir(join(modules, "cloving"), { recursive: true });
+    for (const name of Object.keys(packageMetadata.dependencies)) {
+        const location = findPackageJSON(name, join(product, "package.json"));
+        if (location === undefined)
+            throw new Error(`missing-runtime-dependency: ${name}`);
+        await mkdir(dirname(join(modules, name)), { recursive: true });
+        await symlink(dirname(location), join(modules, name));
+    }
+    const manifest = record(packageMetadata);
+    const exported = Object.keys(record(manifest["exports"])).filter((subpath) => subpath.startsWith("./workflows"));
+    await symlink(join(product, "src"), join(modules, "cloving", "src"));
+    await writeFile(join(modules, "cloving", "package.json"), JSON.stringify({
+        name: manifest["name"],
+        version: manifest["version"],
+        type: "module",
+        exports: Object.fromEntries(exported.map((subpath) => [
+            subpath,
+            `./src/${subpath.slice("./workflows/".length) || "workflows"}${extname(import.meta.filename)}`,
+        ])),
+    }));
+}
+async function registered() {
+    const compiled = JSON.parse(await readFile(join(project, "artifacts/manifest.json"), "utf8"));
+    if (!isWorkflowManifest(compiled))
+        throw new Error("The compiled workflow manifest is invalid.");
+    const manifests = workflowManifests(compiled);
+    const declarations = await workflowDeclarations(join(product, "workflows"), manifests);
+    const paths = new Map();
+    for (const [functionName, manifest] of Object.entries(workflowManifests(compiled, declarations))) {
+        const path = join(project, `${manifest.name}.workflow.json`);
+        await writeFile(path, JSON.stringify(manifest));
+        paths.set(functionName, path);
+    }
+    return { manifests: Object.fromEntries(paths), declarations };
+}
+function source(file) {
+    return JSON.stringify(pathToFileURL(join(product, file.replace(/\.ts$/, extname(import.meta.filename)))).href);
+}
+async function launchers(commands, workflows) {
+    const settings = {
+        ...commands,
+        directory,
+        repository,
+        concurrency,
+        runtimeDirectory: runtime,
+        ...workflows,
+    };
+    await writeFile(join(project, "start.mjs"), `import { hostEnvironment } from ${source("src/daemon/host-environment.ts")};
+Object.assign(process.env, hostEnvironment(${JSON.stringify({ directory, concurrency, leaseSeconds })}));
+await import("./main.mjs");
+`);
+    await writeFile(join(project, "main.mjs"), `import * as workflows from "./artifacts/client.mjs";
+import { POST } from "./artifacts/flow.mjs";
+import { startDaemon } from ${source("src/daemon/daemon.ts")};
+const opening = await startDaemon({ ...JSON.parse(${JSON.stringify(JSON.stringify(settings))}), workflows, handler: POST });
+if (opening.kind !== "started") {
+  process.stderr.write("The daemon did not start: " + opening.reason + ": " + opening.detail + "\\n");
+  process.exit(1);
+}
+process.stdout.write(${JSON.stringify(guide())});
+const stop = async () => { await opening.daemon.close(); process.exit(0); };
+process.once("SIGINT", stop);
+process.once("SIGTERM", stop);
+`);
+}
+function guide() {
+    return [
+        "Cloving runs. Stop it with Control-C.",
+        `Repository: ${repository}`,
+        `Work trees: ${join(directory, "worktrees")}`,
+        "Commands for a second terminal:",
+        `  ${client} run develop-ticket --ticket <ticket-id>`,
+        `  ${client} run develop-issue --ticket <ticket-id>`,
+        `  ${client} run develop-prompt --prompt "<the text of the task>"`,
+        `  ${client} run develop-prd --prompt "<the idea>"`,
+        `  ${client} status`,
+        `  ${client} questions`,
+        `  ${client} watch`,
+        "",
+    ].join("\n");
+}
+async function prepared() {
+    await rm(project, { recursive: true, force: true });
+    await mkdir(join(project, "workflows"), { recursive: true });
+    await writeFile(join(project, "package.json"), '{"name":"cloving-self-host","private":true,"type":"module"}');
+    for (const file of await workflowFiles(join(product, "workflows")))
+        await copyFile(join(product, "workflows", file), join(project, "workflows", file));
+    await linkedPackages();
+    const compiled = await ended([
+        join(product, `tools/compile-workflows${extname(import.meta.filename)}`),
+        "workflows",
+        "artifacts",
+    ], process.env);
+    if (compiled !== 0)
+        throw new Error("The workflows did not compile.");
+    return registered();
+}
+const tokenPrices = parseTokenPrices(process.env["CLOVING_TOKEN_PRICES"]);
+const profiles = await profilesFile();
+const worker = selectedWorker();
+const command = worker === "codex"
+    ? (listed("CLOVING_CODEX_COMMAND") ?? defaultCodexCommand)
+    : (listed("CLOVING_CLAUDE_COMMAND") ?? defaultCommand);
+const nextCommand = listed("CLOVING_NEXT_COMMAND");
+const ticketCommand = listed("CLOVING_TICKET_COMMAND");
+const queueMilliseconds = queueInterval({ nextCommand, ticketCommand });
+const manifests = await prepared();
+await launchers({
+    tokenPrices,
+    worker,
+    profiles,
+    command,
+    ticketCommand,
+    nextCommand,
+    queueMilliseconds,
+    resetQueue: flags.includes("--reset-queue"),
+    notifyCommand: listed("CLOVING_NOTIFY_COMMAND"),
+    githubCommand: listed("CLOVING_GH_COMMAND"),
+    checkCommand: listed("CLOVING_CHECK_COMMAND"),
+    privateTexts: listed("CLOVING_PRIVATE_TEXTS"),
+}, manifests);
+process.exitCode = await hostedPrograms({
+    project,
+    product,
+    runtime,
+    directory,
+    repository,
+    updateMilliseconds,
+    watchPort: process.env["CLOVING_WATCH_PORT"] ?? "3663",
+});

package/dist/tools/self-update-host.d.ts

@@ -0,0 +1,13 @@
+interface ActivitySettings {
+    readonly directory: string;
+    readonly runtimeDirectory: string;
+    readonly signal?: AbortSignal;
+}
+export declare function selfUpdateActive(settings: ActivitySettings): Promise<boolean>;
+export declare function selfUpdateMilliseconds(settings: {
+    readonly seconds: string | undefined;
+    readonly repository: string;
+    readonly product: string;
+}): Promise<number | undefined>;
+export declare function workerTurnActive(directory: string): Promise<boolean>;
+export {};

package/dist/tools/self-update-host.js

@@ -0,0 +1,88 @@
+import { access, readdir, realpath } from "node:fs/promises";
+import { execFile } from "node:child_process";
+import { extname, join } from "node:path";
+import { promisify } from "node:util";
+import { record } from "../src/protocol.js";
+async function runPage(settings, after) {
+    const output = await promisify(execFile)(process.execPath, [
+        join(import.meta.dirname, `../src/cli${extname(import.meta.filename)}`),
+        "status",
+        "--json",
+        "--limit",
+        "500",
+        ...(after === "" ? [] : ["--after", after]),
+    ], {
+        env: { ...process.env, CLOVING_RUNTIME_DIR: settings.runtimeDirectory },
+        encoding: "utf8",
+        timeout: 30000,
+        signal: settings.signal,
+    });
+    const page = record(JSON.parse(output.stdout));
+    if (!page)
+        throw new Error("invalid-run-list");
+    return page;
+}
+export async function selfUpdateActive(settings) {
+    if (await present(join(settings.directory, "queue-submission")))
+        return true;
+    if (await workerTurnActive(settings.directory))
+        return true;
+    let after = "";
+    do {
+        const page = await runPage(settings, after);
+        const runs = page["runs"];
+        if (!Array.isArray(runs))
+            throw new Error("invalid-run-list");
+        if (runs.some((value) => {
+            const run = record(value);
+            if (!run)
+                throw new Error("invalid-run-status");
+            return !["completed", "failed", "cancelled"].includes(String(run["state"]));
+        }))
+            return true;
+        const next = page["next"];
+        if (next !== null && (typeof next !== "string" || next <= after))
+            throw new Error("invalid-run-page");
+        after = next;
+    } while (after !== null);
+    return false;
+}
+export async function selfUpdateMilliseconds(settings) {
+    const { seconds } = settings;
+    if (seconds === undefined)
+        return;
+    const value = Number(seconds);
+    if (!/^\d+$/.test(seconds) ||
+        !Number.isInteger(value) ||
+        value < 30 ||
+        value > 3600)
+        throw new Error("The daemon did not start: invalid-self-update-seconds");
+    if ((await realpath(settings.repository)) !== (await realpath(settings.product)))
+        throw new Error("The daemon did not start: self-update-needs-own-repository");
+    return value * 1000;
+}
+async function present(path) {
+    try {
+        await access(path);
+        return true;
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return false;
+        throw error;
+    }
+}
+export async function workerTurnActive(directory) {
+    const journal = join(directory, "journal");
+    if (!(await present(journal)))
+        return false;
+    for (const entry of await readdir(journal, { withFileTypes: true })) {
+        if (!entry.isDirectory())
+            continue;
+        const turn = join(journal, entry.name);
+        if ((await present(join(turn, "started.json"))) &&
+            !(await present(join(turn, "result.json"))))
+            return true;
+    }
+    return false;
+}

package/dist/tools/self-update.d.ts

@@ -0,0 +1,11 @@
+export interface SelfUpdateSettings {
+    readonly directory: string;
+    readonly git: (argumentsList: readonly string[]) => Promise<string>;
+    readonly active: () => Promise<boolean>;
+    readonly stop: () => Promise<void>;
+    readonly error: (line: string) => void;
+}
+export declare function lockRequiresInstall(before: string, after: string): boolean;
+export declare function selfUpdater(settings: SelfUpdateSettings): {
+    readonly check: () => Promise<number | undefined>;
+};

package/dist/tools/self-update.js

@@ -0,0 +1,177 @@
+export function lockRequiresInstall(before, after) {
+    try {
+        return !isDeepStrictEqual(lockPackages(before), lockPackages(after));
+    }
+    catch {
+        return true;
+    }
+}
+function lockPackages(text) {
+    const lock = JSON.parse(text);
+    if (typeof lock !== "object" || lock === null || !("packages" in lock))
+        throw new Error("invalid package lock");
+    const packages = lock.packages;
+    if (typeof packages !== "object" ||
+        packages === null ||
+        Array.isArray(packages))
+        throw new Error("invalid package lock packages");
+    return Object.fromEntries(Object.entries(packages).filter(([path]) => path !== ""));
+}
+async function clean(git) {
+    const status = await git([
+        "status",
+        "--porcelain",
+        "--untracked-files=all",
+        "--ignore-submodules=none",
+    ]);
+    if (status.length > 0)
+        return false;
+    return !/^[a-zS] /m.test(await git(["ls-files", "-v"]));
+}
+async function installRequired(git, update) {
+    try {
+        await git([
+            "diff",
+            "--exit-code",
+            update.head,
+            update.target,
+            "--",
+            "package-lock.json",
+        ]);
+        return false;
+    }
+    catch {
+        const before = await git(["show", `${update.head}:package-lock.json`]);
+        const after = await git(["show", `${update.target}:package-lock.json`]);
+        return lockRequiresInstall(before, after);
+    }
+}
+async function candidate(settings) {
+    const git = settings.git;
+    await git(["fetch", "origin"]);
+    await git(["remote", "set-head", "origin", "--auto"]);
+    const remoteBranch = (await git(["symbolic-ref", "refs/remotes/origin/HEAD"])).trim();
+    if (!remoteBranch.startsWith("refs/remotes/origin/"))
+        return;
+    const branch = remoteBranch.replace("refs/remotes/origin/", "refs/heads/");
+    if ((await git(["symbolic-ref", "HEAD"])).trim() !== branch)
+        return;
+    const head = (await git(["rev-parse", "HEAD"])).trim();
+    const target = (await git(["rev-parse", remoteBranch])).trim();
+    if (head === target)
+        return;
+    await git(["merge-base", "--is-ancestor", head, target]);
+    if (!(await clean(git)))
+        return;
+    await markPending(settings.directory);
+    if (await settings.active())
+        return "waiting";
+    return { branch, head, target };
+}
+async function markPending(directory, publication = "initial") {
+    if (publication === "renew") {
+        const timestamp = Number(await readFile(join(directory, "pending-self-update"), "utf8"));
+        if (timestamp > 0 && Date.now() - timestamp < 30 * 60 * 1000)
+            return;
+    }
+    await mkdir(directory, { recursive: true });
+    const temporary = join(directory, `pending-self-update-${randomUUID()}.tmp`);
+    try {
+        await writeFile(temporary, `${String(Date.now())}\n`, {
+            flag: "wx",
+            mode: 0o600,
+            flush: true,
+        });
+        if (publication === "renew") {
+            await rename(temporary, join(directory, "pending-self-update"));
+            return;
+        }
+        await link(temporary, join(directory, "pending-self-update")).catch((error) => {
+            if (!(error instanceof Error) ||
+                !("code" in error) ||
+                error.code !== "EEXIST")
+                throw error;
+        });
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+}
+async function failedTargets(directory) {
+    try {
+        return (await readFile(join(directory, "failed-self-updates"), "utf8")).split("\n");
+    }
+    catch (error) {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return [];
+        throw error;
+    }
+}
+async function updated(settings, update) {
+    await settings.stop();
+    try {
+        if (!(await clean(settings.git)))
+            throw new Error("checkout changed after stop");
+        if ((await settings.git(["symbolic-ref", "HEAD"])).trim() !== update.branch ||
+            (await settings.git(["rev-parse", "HEAD"])).trim() !== update.head)
+            throw new Error("checkout changed after stop");
+        await settings.git([
+            "merge",
+            "--ff-only",
+            "--no-overwrite-ignore",
+            update.target,
+        ]);
+    }
+    catch (error) {
+        settings.error(`Self-update failed: ${String(error).replace(/[\r\n]+/g, " ")}\n`);
+        await mkdir(settings.directory, { recursive: true });
+        await appendFile(join(settings.directory, "failed-self-updates"), `${update.target}\n`);
+    }
+    return 75;
+}
+export function selfUpdater(settings) {
+    let disabled = false;
+    let checking = false;
+    return {
+        check: async () => {
+            if (disabled || checking)
+                return;
+            checking = true;
+            let waiting = false;
+            try {
+                const update = await candidate(settings).catch(() => undefined);
+                if (update === "waiting") {
+                    waiting = true;
+                    return;
+                }
+                if (!update)
+                    return;
+                if (await installRequired(settings.git, update).catch(() => true)) {
+                    disabled = true;
+                    settings.error("Self-update disabled: package-lock.json changed; update and install by hand.\n");
+                    return;
+                }
+                const failed = await failedTargets(settings.directory).catch(() => undefined);
+                if (!failed || failed.includes(update.target))
+                    return;
+                await markPending(settings.directory, "renew");
+                waiting = await settings.active().catch(() => true);
+                if (waiting)
+                    return;
+                disabled = true;
+                return await updated(settings, update);
+            }
+            finally {
+                checking = false;
+                if (!waiting)
+                    await rm(join(settings.directory, "pending-self-update"), {
+                        force: true,
+                    });
+            }
+        },
+    };
+}
+import { randomUUID } from "node:crypto";
+import { appendFile, link, mkdir, readFile, rename, rm, writeFile, } from "node:fs/promises";
+import { join } from "node:path";
+import { isDeepStrictEqual } from "node:util";

package/dist/tools/workflow-manifest.d.ts

@@ -0,0 +1,3 @@
+import type { WorkflowManifest } from "@workflow/builders";
+export declare function isWorkflowManifest(value: unknown): value is WorkflowManifest;
+export declare function serializeManifest(manifest: WorkflowManifest): string;

package/dist/tools/workflow-manifest.js

@@ -0,0 +1,34 @@
+export function isWorkflowManifest(value) {
+    if (!isRecord(value))
+        return false;
+    return ((!("workflows" in value) ||
+        hasIdentities(value["workflows"], "workflowId")) &&
+        (!("steps" in value) || hasIdentities(value["steps"], "stepId")) &&
+        (!("classes" in value) || hasIdentities(value["classes"], "classId")));
+}
+function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+function hasIdentities(value, identifier) {
+    if (!isRecord(value))
+        return false;
+    return Object.values(value).every((entries) => isRecord(entries) &&
+        Object.values(entries).every((entry) => isRecord(entry) && typeof entry[identifier] === "string"));
+}
+export function serializeManifest(manifest) {
+    const categories = orderedEntries(manifest).map(([kind, files]) => [
+        kind,
+        Object.fromEntries(orderedEntries(files).map(([file, entries]) => [
+            file,
+            Object.fromEntries(orderedEntries(entries)),
+        ])),
+    ]);
+    return JSON.stringify(Object.fromEntries(categories), null, 2);
+}
+function orderedEntries(value) {
+    return Object.entries(value).sort(([first], [second]) => {
+        if (first === second)
+            return 0;
+        return first < second ? -1 : 1;
+    });
+}

package/dist/workflows/analyze-chats.d.ts

@@ -0,0 +1,10 @@
+import type { CycleOutcome } from "cloving/workflows";
+interface AnalysisInput {
+    readonly cycleId: string;
+    readonly repository: string;
+    readonly input: {
+        readonly importId: string;
+    };
+}
+export declare function analyzeChats(input: AnalysisInput): Promise<CycleOutcome>;
+export {};

package/dist/workflows/analyze-chats.declaration.json

@@ -0,0 +1,9 @@
+{
+  "workflowId": "analyze-chats",
+  "effectLimit": 3,
+  "capabilities": ["chats", "claude"],
+  "input": {
+    "maximumBytes": 4096,
+    "fields": { "importId": { "type": "string" } }
+  }
+}