cloving 1.1.0 → 1.3.0

package/dist/tools/self-host-runtime.js

@@ -0,0 +1,60 @@
+import { randomUUID } from "node:crypto";
+import { link, mkdir, readdir, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { readRetainedWorkflowBundle } from "./workflow-bundles.js";
+export async function selectRuntimeDirectory(input) {
+    const { runtime, repository, legacyRuntime, adoptionRecord } = input;
+    if (legacyRuntime === undefined ||
+        adoptionRecord === undefined ||
+        runtime === legacyRuntime)
+        return { runtime, note: "" };
+    const directory = join(dirname(legacyRuntime), "workflow-bundles");
+    if (!(await belongsToRepository(directory, repository)))
+        return {
+            runtime,
+            note: `Legacy runtime ${legacyRuntime} was not adopted: retained bundles are absent, damaged or belong to another repository.`,
+        };
+    return { runtime: legacyRuntime, note: "", adoptionRecord };
+}
+export async function adoptRuntimeDirectory(input) {
+    const selected = await selectRuntimeDirectory(input);
+    const adoptionRecord = selected.adoptionRecord;
+    if (adoptionRecord === undefined)
+        return { runtime: selected.runtime, note: selected.note };
+    const { repository } = input;
+    const legacyRuntime = selected.runtime;
+    await mkdir(dirname(adoptionRecord), { recursive: true, mode: 0o700 });
+    const contents = JSON.stringify({
+        schemaVersion: 1,
+        repository,
+        runtimeDirectory: legacyRuntime,
+    });
+    const temporary = `${adoptionRecord}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temporary, contents, {
+            flag: "wx",
+            mode: 0o600,
+            flush: true,
+        });
+        await link(temporary, adoptionRecord);
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+    return { runtime: legacyRuntime, note: "" };
+}
+async function belongsToRepository(directory, repository) {
+    try {
+        const ids = (await readdir(directory)).filter((id) => id !== ".history");
+        if (ids.length === 0)
+            return false;
+        for (const id of ids)
+            if ((await readRetainedWorkflowBundle({ directory, repository, id }))
+                .kind !== "available")
+                return false;
+        return true;
+    }
+    catch {
+        return false;
+    }
+}

package/dist/tools/self-host-workflows.d.ts

@@ -12,4 +12,5 @@ interface HostManifest {
 export declare function workflowFiles(directory: string): Promise<string[]>;
 export declare function workflowDeclarations(directory: string, manifests: Readonly<Record<string, HostManifest>>): Promise<readonly DeclaredWorkflow[]>;
 export declare function workflowManifests(compiled: WorkflowManifest, declarations?: readonly DeclaredWorkflow[]): Readonly<Record<string, HostManifest>>;
+export declare function workflowBundleLoader(): string;
 export {};

package/dist/tools/self-host-workflows.js

@@ -6,7 +6,8 @@ export async function workflowFiles(directory) {
     return entries
         .filter((entry) => entry.isFile() &&
         /\.(?:ts|js)$/.test(entry.name) &&
-        !entry.name.endsWith(".d.ts"))
+        !entry.name.endsWith(".d.ts") &&
+        !entry.name.endsWith(".test.ts"))
         .map((entry) => entry.name)
         .sort();
 }
@@ -76,3 +77,37 @@ export function workflowManifests(compiled, declarations = []) {
         });
     }));
 }
+export function workflowBundleLoader() {
+    return `
+import { AsyncLocalStorage } from "node:async_hooks";
+import { pathToFileURL } from "node:url";
+const bundleRegistries = new AsyncLocalStorage();
+const registryKeys = ["@workflow/core//registeredSteps", "workflow-class-registry"].map(Symbol.for);
+for (const key of registryKeys) {
+  let current = globalThis[key] ?? new Map();
+  Object.defineProperty(globalThis, key, {
+    configurable: true,
+    get: () => bundleRegistries.getStore()?.get(key) ?? current,
+    set: (value) => {
+      const scope = bundleRegistries.getStore();
+      if (scope === undefined) current = value;
+      else scope.set(key, value);
+    },
+  });
+}
+const loadedBundles = new Map();
+const loadWorkflowBundle = (directory) => {
+  const known = loadedBundles.get(directory);
+  if (known !== undefined) return known;
+  const scope = new Map(registryKeys.map((key) => [key, new Map()]));
+  const loaded = bundleRegistries.run(scope, async () => {
+    const workflows = await import(pathToFileURL(directory + "/artifacts/client.mjs").href);
+    const { POST } = await import(pathToFileURL(directory + "/artifacts/flow.mjs").href);
+    if (typeof POST !== "function") throw new Error("pinned-workflow-unavailable");
+    return { workflows, handler: (request) => bundleRegistries.run(scope, () => POST(request)) };
+  });
+  loadedBundles.set(directory, loaded);
+  return loaded;
+};
+`;
+}

package/dist/tools/self-host.js

@@ -1,79 +1,74 @@
+import { turnHarness, harnessDefaults, nativeHarness, } from "../src/harness-defaults.js";
+import { absoluteWorkflowFolder } from "../src/project-settings.js";
+import { projectConfiguration, projectRoot, topLevel, } from "./project-configuration.js";
 import { isProfilesFile } from "../src/turn-profile.js";
-import { parseTokenPrices } from "../src/token-prices.js";
-import { execFileSync, spawn } from "node:child_process";
-import { copyFile, mkdir, readFile, rm, symlink, writeFile, } from "node:fs/promises";
+import { parseCommands, parseCommandSeconds, } from "../src/command-settings.js";
+import { parseFallbackModels, parseTokenPrices, } from "../src/token-prices.js";
+import { readFile, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
 import { dirname, extname, join, resolve } from "node:path";
-import packageMetadata from "#package" with { type: "json" };
-import { findPackageJSON } from "node:module";
 import { pathToFileURL } from "node:url";
-import { workflowDeclarations, workflowFiles, workflowManifests, } from "./self-host-workflows.js";
-import { isWorkflowManifest } from "./workflow-manifest.js";
+import { preparedProject, } from "./self-host-project.js";
 import { selfUpdateMilliseconds } from "./self-update-host.js";
 import { hostedPrograms } from "./self-host-processes.js";
+import { workflowBundleLoader } from "./self-host-workflows.js";
+import { adoptRuntimeDirectory } from "./self-host-runtime.js";
+import { parseSchedulesSetting } from "../src/workflow-schedule.js";
 const product = resolve(import.meta.dirname, "..");
 const flags = process.argv.slice(2);
 function flag(name) {
-    const index = flags.lastIndexOf(name);
-    return index === -1 ? undefined : flags[index + 1];
-}
-function topLevel(given) {
-    try {
-        return execFileSync("git", ["-C", given, "rev-parse", "--show-toplevel"], {
-            encoding: "utf8",
-            stdio: ["ignore", "pipe", "ignore"],
-        }).trim();
-    }
-    catch {
-        return given;
-    }
+    const index = flags.findLastIndex((argument) => argument === name || argument.startsWith(`${name}=`));
+    const argument = flags[index];
+    return argument === name
+        ? flags[index + 1]
+        : argument?.slice(name.length + 1);
 }
 const runtimeArgument = flag("--runtime");
 if (runtimeArgument === undefined)
     throw new Error("The start program needs --runtime. Use: cloving start");
-const runtime = resolve(runtimeArgument);
+const repository = await projectRoot(flag("--repository") ?? process.cwd());
+const configuration = await projectConfiguration(repository, process.env);
+const environment = configuration.environment;
+const selectedRuntime = await adoptRuntimeDirectory({
+    runtime: resolve(runtimeArgument),
+    repository,
+    legacyRuntime: flag("--legacy-runtime"),
+    adoptionRecord: flag("--adoption-record"),
+});
+const runtime = selectedRuntime.runtime;
 const directory = dirname(runtime);
-const repository = topLevel(resolve(flag("--repository") ?? process.cwd()));
 const updateMilliseconds = await selfUpdateMilliseconds({
-    seconds: process.env["CLOVING_SELF_UPDATE_SECONDS"],
+    seconds: environment["CLOVING_SELF_UPDATE_SECONDS"],
     repository,
     product: extname(import.meta.filename) === ".ts" ? topLevel(product) : product,
 });
 const client = flag("--client") ?? "cloving";
 const project = join(directory, "project");
-const concurrency = Number(process.env["CLOVING_HOST_BOUND"] ?? "8");
-const leaseText = process.env["CLOVING_OWNERSHIP_LEASE_SECONDS"];
+const concurrency = Number(environment["CLOVING_HOST_BOUND"] ?? "8");
+const leaseText = environment["CLOVING_OWNERSHIP_LEASE_SECONDS"];
 const leaseSeconds = leaseText === undefined ? undefined : Number(leaseText);
-const defaultCommand = [
-    "claude",
-    "-p",
-    "--output-format",
-    "stream-json",
-    "--verbose",
-    "--permission-mode",
-    "acceptEdits",
-    "--allowedTools",
-    "Bash,Read,Edit,Write,Glob,Grep,WebSearch,WebFetch",
-    "--max-budget-usd",
-    "5",
-];
-const defaultCodexCommand = [
-    "codex",
-    "exec",
-    "--json",
-    "--dangerously-bypass-approvals-and-sandbox",
-    "--skip-git-repo-check",
-    "-",
-];
 function selectedWorker() {
-    const worker = process.env["CLOVING_WORKER"] ?? "claude";
-    if (worker === "claude" || worker === "codex")
-        return worker;
-    throw new Error(`The daemon did not start: invalid-worker: ${worker}`);
+    return checkedWorker(environment["CLOVING_WORKER"] ?? "claude");
+}
+function checkedWorker(name) {
+    try {
+        return turnHarness({
+            worker: name,
+            ...(configuration.harness === undefined
+                ? {}
+                : { harness: configuration.harness }),
+        }).worker;
+    }
+    catch (error) {
+        throw new Error(`The daemon did not start: harness-no-worker: ${name}`, {
+            cause: error,
+        });
+    }
 }
 async function profilesFile() {
-    const path = process.env["CLOVING_PROFILES"];
+    const path = environment["CLOVING_PROFILES"];
     if (path === undefined)
-        return {};
+        return configuration.profiles;
     try {
         const value = JSON.parse(await readFile(path, "utf8"));
         if (isProfilesFile(value))
@@ -84,8 +79,18 @@ async function profilesFile() {
     }
     throw new Error("The daemon did not start: invalid-profiles");
 }
+function queueLanes() {
+    const value = environment["CLOVING_QUEUE_LANES"];
+    if (value === undefined)
+        return 1;
+    if (!/^[1-4]$/.test(value))
+        throw new Error("The daemon did not start: invalid-queue-lanes");
+    if (environment["CLOVING_QUEUE_SECONDS"] === undefined)
+        throw new Error("The daemon did not start: queue-lanes-needs-queue");
+    return Number(value);
+}
 function queueInterval(commands) {
-    const value = process.env["CLOVING_QUEUE_SECONDS"];
+    const value = environment["CLOVING_QUEUE_SECONDS"];
     if (value === undefined)
         return undefined;
     const seconds = Number(value);
@@ -101,7 +106,7 @@ function queueInterval(commands) {
     return seconds * 1000;
 }
 function listed(name) {
-    const given = process.env[name];
+    const given = environment[name];
     if (given === undefined)
         return undefined;
     const list = JSON.parse(given);
@@ -109,61 +114,6 @@ function listed(name) {
         throw new Error(`${name} is not a JSON list of texts.`);
     return list.map(String);
 }
-function record(value) {
-    return typeof value === "object" && value !== null ? { ...value } : {};
-}
-function ended(argumentsList, environment) {
-    return new Promise((settle, reject) => {
-        const child = spawn(process.execPath, argumentsList, {
-            cwd: project,
-            env: environment,
-            stdio: "inherit",
-        });
-        for (const signal of ["SIGINT", "SIGTERM"])
-            process.on(signal, () => child.kill(signal));
-        child.once("error", reject);
-        child.once("close", (status) => {
-            settle(status ?? 1);
-        });
-    });
-}
-async function linkedPackages() {
-    const modules = join(project, "node_modules");
-    await mkdir(join(modules, "cloving"), { recursive: true });
-    for (const name of Object.keys(packageMetadata.dependencies)) {
-        const location = findPackageJSON(name, join(product, "package.json"));
-        if (location === undefined)
-            throw new Error(`missing-runtime-dependency: ${name}`);
-        await mkdir(dirname(join(modules, name)), { recursive: true });
-        await symlink(dirname(location), join(modules, name));
-    }
-    const manifest = record(packageMetadata);
-    const exported = Object.keys(record(manifest["exports"])).filter((subpath) => subpath.startsWith("./workflows"));
-    await symlink(join(product, "src"), join(modules, "cloving", "src"));
-    await writeFile(join(modules, "cloving", "package.json"), JSON.stringify({
-        name: manifest["name"],
-        version: manifest["version"],
-        type: "module",
-        exports: Object.fromEntries(exported.map((subpath) => [
-            subpath,
-            `./src/${subpath.slice("./workflows/".length) || "workflows"}${extname(import.meta.filename)}`,
-        ])),
-    }));
-}
-async function registered() {
-    const compiled = JSON.parse(await readFile(join(project, "artifacts/manifest.json"), "utf8"));
-    if (!isWorkflowManifest(compiled))
-        throw new Error("The compiled workflow manifest is invalid.");
-    const manifests = workflowManifests(compiled);
-    const declarations = await workflowDeclarations(join(product, "workflows"), manifests);
-    const paths = new Map();
-    for (const [functionName, manifest] of Object.entries(workflowManifests(compiled, declarations))) {
-        const path = join(project, `${manifest.name}.workflow.json`);
-        await writeFile(path, JSON.stringify(manifest));
-        paths.set(functionName, path);
-    }
-    return { manifests: Object.fromEntries(paths), declarations };
-}
 function source(file) {
     return JSON.stringify(pathToFileURL(join(product, file.replace(/\.ts$/, extname(import.meta.filename)))).href);
 }
@@ -171,7 +121,6 @@ async function launchers(commands, workflows) {
     const settings = {
         ...commands,
         directory,
-        repository,
         concurrency,
         runtimeDirectory: runtime,
         ...workflows,
@@ -183,22 +132,27 @@ await import("./main.mjs");
     await writeFile(join(project, "main.mjs"), `import * as workflows from "./artifacts/client.mjs";
 import { POST } from "./artifacts/flow.mjs";
 import { startDaemon } from ${source("src/daemon/daemon.ts")};
-const opening = await startDaemon({ ...JSON.parse(${JSON.stringify(JSON.stringify(settings))}), workflows, handler: POST });
+${workflowBundleLoader()}
+const opening = await startDaemon({ ...JSON.parse(${JSON.stringify(JSON.stringify(settings))}), workflows, handler: POST, loadBundle: loadWorkflowBundle });
 if (opening.kind !== "started") {
   process.stderr.write("The daemon did not start: " + opening.reason + ": " + opening.detail + "\\n");
   process.exit(1);
 }
-process.stdout.write(${JSON.stringify(guide())});
+process.stdout.write(${JSON.stringify(guide(workflows))});
 const stop = async () => { await opening.daemon.close(); process.exit(0); };
 process.once("SIGINT", stop);
 process.once("SIGTERM", stop);
 `);
 }
-function guide() {
+function guide(workflows) {
     return [
         "Cloving runs. Stop it with Control-C.",
         `Repository: ${repository}`,
+        `Runtime directory: ${runtime}`,
         `Work trees: ${join(directory, "worktrees")}`,
+        ...(selectedRuntime.note === "" ? [] : [selectedRuntime.note]),
+        ...Object.entries(configuration.sources).map(([field, source]) => `${field}: ${source}`),
+        ...workflows.workflowRefusals.map((refusal) => `Refused workflow ${refusal.scope} ${refusal.label}: ${refusal.codes.join(", ")}`),
         "Commands for a second terminal:",
         `  ${client} run develop-ticket --ticket <ticket-id>`,
         `  ${client} run develop-issue --ticket <ticket-id>`,
@@ -210,43 +164,59 @@ function guide() {
         "",
     ].join("\n");
 }
-async function prepared() {
-    await rm(project, { recursive: true, force: true });
-    await mkdir(join(project, "workflows"), { recursive: true });
-    await writeFile(join(project, "package.json"), '{"name":"cloving-self-host","private":true,"type":"module"}');
-    for (const file of await workflowFiles(join(product, "workflows")))
-        await copyFile(join(product, "workflows", file), join(project, "workflows", file));
-    await linkedPackages();
-    const compiled = await ended([
-        join(product, `tools/compile-workflows${extname(import.meta.filename)}`),
-        "workflows",
-        "artifacts",
-    ], process.env);
-    if (compiled !== 0)
-        throw new Error("The workflows did not compile.");
-    return registered();
-}
-const tokenPrices = parseTokenPrices(process.env["CLOVING_TOKEN_PRICES"]);
+const tokenPrices = parseTokenPrices(environment["CLOVING_TOKEN_PRICES"]);
+const commands = parseCommands(environment["CLOVING_COMMANDS"]);
+const commandSeconds = parseCommandSeconds(environment["CLOVING_COMMAND_SECONDS"]);
+const fallbackModels = parseFallbackModels(environment["CLOVING_FALLBACK_MODELS"]);
 const profiles = await profilesFile();
+const schedules = parseSchedulesSetting(environment["CLOVING_SCHEDULES"]);
 const worker = selectedWorker();
-const command = worker === "codex"
-    ? (listed("CLOVING_CODEX_COMMAND") ?? defaultCodexCommand)
-    : (listed("CLOVING_CLAUDE_COMMAND") ?? defaultCommand);
+function nativeCommands() {
+    const names = new Set([
+        worker,
+        ...Object.values(profiles).map((profile) => checkedWorker(profile.harness ?? worker)),
+    ]);
+    return Object.fromEntries([...names].filter(nativeHarness).flatMap((name) => {
+        const variable = harnessDefaults(name).variable;
+        const command = variable === undefined ? undefined : listed(variable);
+        return command === undefined ? [] : [[name, command]];
+    }));
+}
+const workerCommands = nativeCommands();
+const harness = configuration.harness ?? {};
+const command = turnHarness({ worker, workerCommands, harness }).command;
 const nextCommand = listed("CLOVING_NEXT_COMMAND");
 const ticketCommand = listed("CLOVING_TICKET_COMMAND");
+const lanes = queueLanes();
 const queueMilliseconds = queueInterval({ nextCommand, ticketCommand });
-const manifests = await prepared();
+const manifests = await preparedProject({
+    product,
+    project,
+    repository,
+    home: homedir(),
+    xdgConfigHome: environment["XDG_CONFIG_HOME"],
+    userFolder: absoluteWorkflowFolder(environment["CLOVING_WORKFLOW_FOLDER"]),
+});
 await launchers({
+    commands,
+    commandSeconds,
+    schedules,
     tokenPrices,
+    fallbackModels,
     worker,
+    workerCommands,
+    harness,
+    shell: environment["SHELL"] ?? "/bin/sh",
     profiles,
     command,
     ticketCommand,
     nextCommand,
     queueMilliseconds,
+    queueLanes: lanes,
     resetQueue: flags.includes("--reset-queue"),
     notifyCommand: listed("CLOVING_NOTIFY_COMMAND"),
     githubCommand: listed("CLOVING_GH_COMMAND"),
+    requiredChecks: listed("CLOVING_REQUIRED_CHECKS") ?? null,
     checkCommand: listed("CLOVING_CHECK_COMMAND"),
     privateTexts: listed("CLOVING_PRIVATE_TEXTS"),
 }, manifests);
@@ -257,5 +227,6 @@ process.exitCode = await hostedPrograms({
     directory,
     repository,
     updateMilliseconds,
-    watchPort: process.env["CLOVING_WATCH_PORT"] ?? "3663",
+    watchPort: environment["CLOVING_WATCH_PORT"] ?? "3663",
+    phoneRelay: environment["CLOVING_PHONE_RELAY"] ?? "on",
 });

package/dist/tools/workflow-bundle-dependencies.d.ts

@@ -0,0 +1,5 @@
+export declare function workflowPackageRoot(directory: string): Promise<string>;
+export declare function workflowBundleDependencies(input: {
+    readonly product: string;
+    readonly packages: readonly string[];
+}): Promise<string>;

package/dist/tools/workflow-bundle-dependencies.js

@@ -0,0 +1,93 @@
+import { createHash } from "node:crypto";
+import { access, readFile, realpath } from "node:fs/promises";
+import { findPackageJSON } from "node:module";
+import { dirname, join, resolve } from "node:path";
+import { record } from "../src/protocol.js";
+export async function workflowPackageRoot(directory) {
+    const root = resolve(directory);
+    try {
+        await access(join(root, "package.json"));
+        return root;
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+            throw error;
+    }
+    const parent = dirname(root);
+    if (parent === root)
+        throw new Error("workflow-package-unavailable");
+    return workflowPackageRoot(parent);
+}
+function references(manifest) {
+    const dependencies = record(manifest["dependencies"]) ?? {};
+    const optional = record(manifest["optionalDependencies"]) ?? {};
+    const peers = record(manifest["peerDependencies"]) ?? {};
+    const peerMetadata = record(manifest["peerDependenciesMeta"]) ?? {};
+    return Object.keys({ ...dependencies, ...optional, ...peers })
+        .sort()
+        .map((name) => ({
+        name,
+        optional: Object.hasOwn(optional, name) ||
+            (!Object.hasOwn(dependencies, name) &&
+                record(peerMetadata[name])?.["optional"] === true),
+    }));
+}
+async function manifestFile(from, reference) {
+    try {
+        const file = findPackageJSON(reference.name, from);
+        if (file !== undefined)
+            return await realpath(file);
+    }
+    catch (error) {
+        if (!(error instanceof Error &&
+            "code" in error &&
+            error.code === "ERR_MODULE_NOT_FOUND"))
+            throw error;
+    }
+    if (reference.optional)
+        return undefined;
+    throw new Error("workflow-dependency-unavailable");
+}
+async function capturePackage(capture, input) {
+    const file = await manifestFile(input.from, input);
+    if (file === undefined)
+        return null;
+    const previous = capture.seen.get(file);
+    if (previous !== undefined)
+        return previous;
+    const manifest = await readFile(file, "utf8");
+    const parsed = JSON.parse(manifest);
+    const fields = record(parsed);
+    if (typeof fields?.["name"] !== "string" ||
+        typeof fields["version"] !== "string")
+        throw new Error("invalid-workflow-dependency");
+    const captured = { manifest, dependencies: {} };
+    const index = capture.packages.length;
+    capture.seen.set(file, index);
+    capture.packages.push(captured);
+    for (const reference of references(fields))
+        captured.dependencies[reference.name] = await capturePackage(capture, {
+            ...reference,
+            from: file,
+        });
+    return index;
+}
+export async function workflowBundleDependencies(input) {
+    const capture = { seen: new Map(), packages: [] };
+    const roots = {};
+    const manifest = join(await workflowPackageRoot(input.product), "package.json");
+    for (const name of [...input.packages].sort())
+        roots[name] = await capturePackage(capture, {
+            name,
+            from: manifest,
+            optional: false,
+        });
+    return JSON.stringify({
+        node: process.version,
+        product: createHash("sha256")
+            .update(await readFile(manifest))
+            .digest("hex"),
+        roots,
+        packages: capture.packages,
+    });
+}

package/dist/tools/workflow-bundles.d.ts

@@ -0,0 +1,20 @@
+interface BundleInput {
+    readonly directory: string;
+    readonly repository: string;
+    readonly project: string;
+}
+interface BundleLookup {
+    readonly directory: string;
+    readonly repository: string;
+    readonly id: string;
+}
+type BundleResult = {
+    readonly kind: "available";
+    readonly directory: string;
+} | {
+    readonly kind: "unavailable";
+    readonly code: "pinned-workflow-unavailable";
+};
+export declare function readRetainedWorkflowBundle(input: BundleLookup): Promise<BundleResult>;
+export declare function retainWorkflowBundle(input: BundleInput): Promise<string>;
+export {};