cloving 1.3.0 → 1.5.0

package/dist/tools/red-green-process.js

@@ -0,0 +1,59 @@
+import { spawn } from "node:child_process";
+import { addAbortListener } from "node:events";
+function stop(child) {
+    if (child.pid === undefined)
+        return;
+    try {
+        process.kill(-child.pid, "SIGKILL");
+    }
+    catch (error) {
+        if (Reflect.get(Object(error), "code") !== "ESRCH")
+            child.kill("SIGKILL");
+    }
+}
+export async function redGreenProcess(argumentsList, options) {
+    options.signal.throwIfAborted();
+    const child = spawn(argumentsList[0] ?? "", argumentsList.slice(1), {
+        cwd: options.directory,
+        stdio: ["ignore", "pipe", "pipe"],
+        detached: true,
+        env: { ...process.env, NODE_TEST_CONTEXT: undefined },
+    });
+    let output = "";
+    let stopped = false;
+    child.stdout.setEncoding("utf8");
+    child.stderr.setEncoding("utf8");
+    child.stdout.on("data", (chunk) => {
+        output += chunk;
+    });
+    child.stderr.on("data", (chunk) => {
+        output += chunk;
+    });
+    child.once("error", (error) => {
+        output += error.message;
+    });
+    const closed = new Promise((resolve) => {
+        child.once("close", (status) => {
+            resolve({ status: stopped ? null : status, output });
+        });
+    });
+    const terminate = () => {
+        stopped = true;
+        stop(child);
+        child.stdout.destroy();
+        child.stderr.destroy();
+    };
+    const listener = addAbortListener(options.signal, terminate);
+    const timeout = setTimeout(() => {
+        output += "\nred-green command timed out\n";
+        terminate();
+    }, options.milliseconds);
+    try {
+        return await closed;
+    }
+    finally {
+        clearTimeout(timeout);
+        listener[Symbol.dispose]();
+        stop(child);
+    }
+}

package/dist/tools/self-host-processes.d.ts

@@ -1,4 +1,5 @@
 export interface HostProcesses {
+    readonly environment?: Readonly<NodeJS.ProcessEnv>;
     readonly project: string;
     readonly product: string;
     readonly runtime: string;
@@ -7,5 +8,6 @@ export interface HostProcesses {
     readonly watchPort: string;
     readonly phoneRelay: string;
     readonly updateMilliseconds: number | undefined;
+    readonly stopMilliseconds?: number;
 }
 export declare function hostedPrograms(settings: HostProcesses): Promise<number>;

package/dist/tools/self-host-processes.js

@@ -1,4 +1,4 @@
-import { execFile, spawn } from "node:child_process";
+import { execFile, spawn, } from "node:child_process";
 import { addAbortListener } from "node:events";
 import { rm } from "node:fs/promises";
 import { extname, join } from "node:path";
@@ -10,7 +10,7 @@ function launched(argumentsList, settings) {
     const child = spawn(process.execPath, argumentsList, {
         cwd: settings.directory,
         env: settings.environment,
-        stdio: "inherit",
+        stdio: settings.stdio ?? "inherit",
     });
     const ended = new Promise((settle, reject) => {
         child.once("error", reject);
@@ -18,13 +18,26 @@ function launched(argumentsList, settings) {
             settle(status ?? 1);
         });
     });
-    return { child, ended };
+    return { name: settings.name, child, ended };
 }
-async function stopped(programs) {
+async function stopped(programs, milliseconds) {
     for (const program of programs)
-        if (!program.child.killed)
+        if (!program.child.killed &&
+            program.child.exitCode === null &&
+            program.child.signalCode === null)
             program.child.kill("SIGTERM");
-    await Promise.all(programs.map((program) => program.ended));
+    await Promise.all(programs.map(async (program) => {
+        const timer = setTimeout(() => {
+            if (program.child.kill("SIGKILL"))
+                process.stderr.write(`Cloving stopped the ${program.name} child with SIGKILL after ${String(milliseconds / 1000)} s\n`);
+        }, milliseconds);
+        try {
+            await program.ended;
+        }
+        finally {
+            clearTimeout(timer);
+        }
+    }));
 }
 function updatingGit(settings, stopping) {
     const git = promisify(execFile);
@@ -97,9 +110,10 @@ function productChild(settings, argumentsList) {
         join(settings.product, `src/${program}${extname(import.meta.filename)}`),
         ...rest,
     ], {
+        name: program === "cli" ? "watch" : program,
         directory: settings.product,
         environment: {
-            ...process.env,
+            ...(settings.environment ?? process.env),
             CLOVING_RUNTIME_DIR: settings.runtime,
             CLOVING_WATCH_REPOSITORY: settings.repository,
         },
@@ -107,35 +121,63 @@ function productChild(settings, argumentsList) {
 }
 function hostedChildren(settings) {
     const daemon = launched([join(settings.project, "start.mjs")], {
+        name: "daemon",
         directory: settings.project,
-        environment: process.env,
+        environment: settings.environment ?? process.env,
+        stdio: ["inherit", "inherit", "inherit", "ipc"],
     });
-    const watch = settings.watchPort === "off"
-        ? []
-        : [
-            productChild(settings, [
-                "cli",
-                "watch",
-                "--port",
-                settings.watchPort,
-            ]),
-        ];
     const relay = settings.phoneRelay === "off" ? [] : [productChild(settings, ["relay"])];
-    return { daemon, programs: [daemon, ...watch, ...relay] };
+    return { daemon, programs: [daemon, ...relay] };
+}
+function daemonStart(daemon) {
+    return new Promise((settle) => {
+        daemon.child.once("message", () => {
+            settle("started");
+        });
+        daemon.ended.then(() => {
+            settle("ended");
+        }, () => {
+            settle("ended");
+        });
+    });
+}
+function watchAfterStart(settings, hosting) {
+    if (settings.watchPort === "off")
+        return Promise.resolve();
+    return daemonStart(hosting.daemon).then((start) => {
+        if (start === "ended" || hosting.stopped())
+            return;
+        const watch = productChild(settings, [
+            "cli",
+            "watch",
+            "--port",
+            settings.watchPort,
+        ]);
+        hosting.programs.splice(1, 0, watch);
+    });
 }
 export async function hostedPrograms(settings) {
     const { daemon, programs } = hostedChildren(settings);
     const stopping = new AbortController();
     let updating = false;
-    const interrupt = () => {
+    let shutdown;
+    const watching = watchAfterStart(settings, {
+        daemon,
+        programs,
+        stopped: () => shutdown !== undefined,
+    });
+    const stop = () => (shutdown ??= stopped(programs, settings.stopMilliseconds ?? 60000));
+    const signal = (signal) => {
         stopping.abort();
         for (const program of programs)
-            program.child.kill("SIGINT");
+            program.child.kill(signal);
+        shutdown = stop();
+    };
+    const interrupt = () => {
+        signal("SIGINT");
     };
     const terminate = () => {
-        stopping.abort();
-        for (const program of programs)
-            program.child.kill("SIGTERM");
+        signal("SIGTERM");
     };
     process.on("SIGINT", interrupt);
     process.on("SIGTERM", terminate);
@@ -143,7 +185,7 @@ export async function hostedPrograms(settings) {
         stopping: stopping.signal,
         stop: async () => {
             updating = true;
-            await stopped(programs);
+            await stop();
         },
     });
     try {
@@ -159,7 +201,8 @@ export async function hostedPrograms(settings) {
     }
     finally {
         stopping.abort();
-        await stopped(programs);
+        await watching;
+        await stop();
         await polling;
         process.off("SIGINT", interrupt);
         process.off("SIGTERM", terminate);

package/dist/tools/self-host-project.d.ts

@@ -8,6 +8,7 @@ interface ProjectSettings {
     readonly home: string;
     readonly xdgConfigHome?: string | undefined;
     readonly userFolder?: string | undefined;
+    readonly recordedProject?: string;
 }
 export interface RegisteredWorkflows {
     readonly workflows: readonly WorkflowListing[];
@@ -18,4 +19,5 @@ export interface RegisteredWorkflows {
     readonly workflowRefusals: readonly WorkflowRefusal[];
 }
 export declare function preparedProject(settings: ProjectSettings): Promise<RegisteredWorkflows>;
+export declare function preparedBundle(settings: ProjectSettings): Promise<RegisteredWorkflows>;
 export {};

package/dist/tools/self-host-project.js

@@ -1,13 +1,12 @@
 import { spawn } from "node:child_process";
 import { createHash } from "node:crypto";
-import { copyFile, mkdir, readFile, rm, symlink, writeFile, } from "node:fs/promises";
+import { mkdir, readFile, rm, symlink, writeFile } from "node:fs/promises";
 import { findPackageJSON } from "node:module";
-import { basename, dirname, extname, join, resolve } from "node:path";
+import { dirname, extname, join, resolve } from "node:path";
 import packageMetadata from "#package" with { type: "json" };
 import { record } from "../src/protocol.js";
 import { isWorkflowDeclaration, } from "../src/workflow-declaration.js";
-import { bundledWorkflowIds, } from "../src/workflow-discovery.js";
-import { workflowDeclarations, workflowFiles, workflowManifests, } from "./self-host-workflows.js";
+import { workflowManifests } from "./self-host-workflows.js";
 import { loadWorkflowFolders, } from "./workflow-folders.js";
 import { isWorkflowManifest } from "./workflow-manifest.js";
 import { retainWorkflowBundle } from "./workflow-bundles.js";
@@ -17,9 +16,6 @@ async function initializeProject(settings) {
     await rm(project, { recursive: true, force: true });
     await mkdir(join(project, "workflows"), { recursive: true, mode: 0o700 });
     await writeFile(join(project, "package.json"), '{"name":"cloving-self-host","private":true,"type":"module"}');
-    const files = await workflowFiles(join(product, "workflows"));
-    for (const file of files)
-        await copyWorkflow(settings, file);
     const modules = join(project, "node_modules");
     await mkdir(join(modules, "cloving"), { recursive: true });
     const dependencies = new Map(Object.keys(packageMetadata.dependencies).map((name) => [name, join(product, "package.json")]));
@@ -46,7 +42,6 @@ async function initializeProject(settings) {
             `./src/${subpath.slice("./workflows/".length) || "workflows"}${extname(import.meta.filename)}`,
         ])),
     }));
-    return files;
 }
 async function compile(settings) {
     const { product, project } = settings;
@@ -70,23 +65,6 @@ async function compile(settings) {
         throw new Error("The compiled workflow manifest is invalid.");
     return compiled;
 }
-function reservations(input) {
-    const { files, manifests, declarations } = input;
-    return [
-        ...files.map((file) => ({
-            baseName: basename(file, extname(file)),
-        })),
-        ...Object.entries(manifests).map(([functionName, manifest]) => ({
-            baseName: manifest.name,
-            workflowId: manifest.workflowId,
-            functionName,
-        })),
-        ...[
-            ...Object.values(bundledWorkflowIds),
-            ...declarations.map((item) => item.workflowId),
-        ].map((workflowId) => ({ workflowId })),
-    ];
-}
 async function stageCandidates(project, candidates) {
     if (candidates.length === 0)
         return [];
@@ -135,36 +113,18 @@ function operatorDeclaration(staged, compiled) {
         functionName: workflow.workflowId.slice(workflow.workflowId.lastIndexOf("//") + 2),
     };
 }
-async function copyWorkflow(settings, file) {
-    const { product, project } = settings;
-    await copyFile(join(product, "workflows", file), join(project, "workflows", file));
-    const sidecar = `${basename(file, extname(file))}.declaration.json`;
-    const declaration = await readFile(join(product, "workflows", sidecar)).catch((error) => {
-        if (error instanceof Error && "code" in error && error.code === "ENOENT")
-            return undefined;
-        throw error;
-    });
-    if (declaration !== undefined)
-        await writeFile(join(project, "workflows", sidecar), declaration);
-}
-function bundledListings(input) {
-    const { product, files, manifests, declarations } = input;
-    const byFunction = new Map(declarations.map(({ functionName, ...declaration }) => [
-        functionName,
-        declaration,
-    ]));
-    return Object.entries(manifests).map(([functionName, manifest]) => {
-        const declaration = byFunction.get(functionName);
-        const file = files.find((file) => basename(file, extname(file)) === manifest.name);
-        return {
-            scope: "bundled",
-            workflowId: declaration?.workflowId ?? manifest.name,
-            file: join(product, "workflows", file ?? manifest.name),
-            admitted: true,
-            codes: [],
-            ...(declaration === undefined ? {} : { declaration }),
-        };
-    });
+async function recordProject(settings) {
+    const { project, recordedProject } = settings;
+    if (recordedProject === undefined)
+        return;
+    for (const name of [
+        "flow.mjs.__wf_tmp.js.debug.json",
+        "steps.mjs.debug.json",
+    ]) {
+        const path = join(project, "artifacts", name);
+        const text = await readFile(path, "utf8");
+        await writeFile(path, text.replaceAll(`${project}/`, `${recordedProject}/`));
+    }
 }
 async function writeManifests(project, compiled) {
     const manifests = {};
@@ -179,22 +139,22 @@ async function writeManifests(project, compiled) {
     return manifests;
 }
 export async function preparedProject(settings) {
+    const prepared = await preparedBundle(settings);
+    const modules = join(dirname(settings.project), "workflow-bundles/node_modules");
+    await rm(modules, { force: true });
+    await symlink(join(settings.project, "node_modules"), modules);
+    return prepared;
+}
+export async function preparedBundle(settings) {
     const { product, project } = settings;
-    const files = await initializeProject(settings);
-    let compiled = await compile(settings);
-    const bundled = workflowManifests(compiled);
-    const declarations = await workflowDeclarations(join(project, "workflows"), bundled);
-    const folders = await loadWorkflowFolders({
-        ...settings,
-        bundled: reservations({ files, manifests: bundled, declarations }),
-    }, { packageRoot: await workflowPackageRoot(product) });
+    await initializeProject(settings);
+    const folders = await loadWorkflowFolders(settings, {
+        packageRoot: await workflowPackageRoot(product),
+    });
     const staged = await stageCandidates(project, folders.candidates);
-    if (staged.length > 0)
-        compiled = await compile(settings);
-    const combined = [
-        ...declarations,
-        ...staged.map((candidate) => operatorDeclaration({ ...candidate, project }, compiled)),
-    ];
+    const compiled = await compile(settings);
+    await recordProject(settings);
+    const combined = staged.map((candidate) => operatorDeclaration({ ...candidate, project }, compiled));
     const manifests = await writeManifests(project, workflowManifests(compiled, combined));
     await writeFile(join(project, "declarations.json"), JSON.stringify(combined));
     await writeFile(join(project, "dependencies.json"), await workflowBundleDependencies({
@@ -206,15 +166,8 @@ export async function preparedProject(settings) {
         repository: folders.repository,
         directory: join(dirname(project), "workflow-bundles"),
     });
-    const workflows = bundledListings({
-        product,
-        files,
-        manifests: bundled,
-        declarations,
-    });
-    workflows.push(...folders.workflows);
     return {
-        workflows,
+        workflows: folders.workflows,
         repository: folders.repository,
         bundleDigest,
         manifests,

package/dist/tools/self-host-runtime.js

@@ -45,7 +45,7 @@ export async function adoptRuntimeDirectory(input) {
 }
 async function belongsToRepository(directory, repository) {
     try {
-        const ids = (await readdir(directory)).filter((id) => id !== ".history");
+        const ids = (await readdir(directory)).filter((id) => id !== ".history" && id !== "node_modules");
         if (ids.length === 0)
             return false;
         for (const id of ids)

package/dist/tools/self-host.js

@@ -1,10 +1,9 @@
 import { turnHarness, harnessDefaults, nativeHarness, } from "../src/harness-defaults.js";
 import { absoluteWorkflowFolder } from "../src/project-settings.js";
-import { projectConfiguration, projectRoot, topLevel, } from "./project-configuration.js";
-import { isProfilesFile } from "../src/turn-profile.js";
+import { retentionSettings, } from "../src/record-retention.js";
+import { configurationDigests, configuredTurnSettings, projectConfiguration, projectRoot, topLevel, } from "./project-configuration.js";
 import { parseCommands, parseCommandSeconds, } from "../src/command-settings.js";
-import { parseFallbackModels, parseTokenPrices, } from "../src/token-prices.js";
-import { readFile, writeFile } from "node:fs/promises";
+import { writeFile } from "node:fs/promises";
 import { homedir } from "node:os";
 import { dirname, extname, join, resolve } from "node:path";
 import { pathToFileURL } from "node:url";
@@ -14,6 +13,7 @@ import { hostedPrograms } from "./self-host-processes.js";
 import { workflowBundleLoader } from "./self-host-workflows.js";
 import { adoptRuntimeDirectory } from "./self-host-runtime.js";
 import { parseSchedulesSetting } from "../src/workflow-schedule.js";
+import { checkSubmission } from "../src/submission.js";
 const product = resolve(import.meta.dirname, "..");
 const flags = process.argv.slice(2);
 function flag(name) {
@@ -65,19 +65,12 @@ function checkedWorker(name) {
         });
     }
 }
-async function profilesFile() {
-    const path = environment["CLOVING_PROFILES"];
-    if (path === undefined)
-        return configuration.profiles;
-    try {
-        const value = JSON.parse(await readFile(path, "utf8"));
-        if (isProfilesFile(value))
-            return value.profiles;
-    }
-    catch {
-        throw new Error("The daemon did not start: invalid-profiles");
-    }
-    throw new Error("The daemon did not start: invalid-profiles");
+function startRefusal(error) {
+    if (error instanceof Error && error.message === "invalid-profiles")
+        throw new Error("The daemon did not start: invalid-profiles", {
+            cause: error,
+        });
+    throw error;
 }
 function queueLanes() {
     const value = environment["CLOVING_QUEUE_LANES"];
@@ -89,7 +82,18 @@ function queueLanes() {
         throw new Error("The daemon did not start: queue-lanes-needs-queue");
     return Number(value);
 }
-function queueInterval(commands) {
+function queueSpendLimit() {
+    const value = environment["CLOVING_QUEUE_DAILY_SPEND_USD"];
+    if (value === undefined)
+        return undefined;
+    const limit = Number(value);
+    if (!Number.isFinite(limit) || limit <= 0)
+        throw new Error("The daemon did not start: invalid-queue-spend-limit");
+    if (environment["CLOVING_QUEUE_SECONDS"] === undefined)
+        throw new Error("The daemon did not start: queue-spend-limit-needs-queue");
+    return limit;
+}
+function queueInterval() {
     const value = environment["CLOVING_QUEUE_SECONDS"];
     if (value === undefined)
         return undefined;
@@ -99,10 +103,6 @@ function queueInterval(commands) {
         seconds < 10 ||
         seconds > 3600)
         throw new Error("The daemon did not start: invalid-queue-seconds");
-    if (!commands.nextCommand?.length)
-        throw new Error("The daemon did not start: queue-needs-next-command");
-    if (!commands.ticketCommand?.length)
-        throw new Error("The daemon did not start: queue-needs-ticket-command");
     return seconds * 1000;
 }
 function listed(name) {
@@ -118,6 +118,12 @@ function source(file) {
     return JSON.stringify(pathToFileURL(join(product, file.replace(/\.ts$/, extname(import.meta.filename)))).href);
 }
 async function launchers(commands, workflows) {
+    const queueEnabled = commands.queueMilliseconds !== undefined &&
+        workflows.declarations.some((declaration) => declaration.queue === true);
+    if (queueEnabled && !commands.nextCommand?.length)
+        throw new Error("The daemon did not start: queue-needs-next-command");
+    if (queueEnabled && !commands.ticketCommand?.length)
+        throw new Error("The daemon did not start: queue-needs-ticket-command");
     const settings = {
         ...commands,
         directory,
@@ -139,11 +145,29 @@ if (opening.kind !== "started") {
   process.exit(1);
 }
 process.stdout.write(${JSON.stringify(guide(workflows))});
+process.send?.("started");
+process.channel?.unref();
 const stop = async () => { await opening.daemon.close(); process.exit(0); };
 process.once("SIGINT", stop);
 process.once("SIGTERM", stop);
 `);
 }
+function runCommand(declaration) {
+    const { workflowId, input } = declaration;
+    for (const [field, value, argument] of [
+        ["ticket", "ticket-id", "--ticket <ticket-id>"],
+        ["prompt", "task text", '--prompt "<text>"'],
+    ]) {
+        const checked = checkSubmission(input, {
+            workflowId,
+            source: field,
+            submission: { [field]: value },
+        });
+        if (checked.kind === "valid")
+            return `  ${client} run ${workflowId} ${argument}`;
+    }
+    return `  ${client} run ${workflowId} --input <json-file>`;
+}
 function guide(workflows) {
     return [
         "Cloving runs. Stop it with Control-C.",
@@ -154,21 +178,18 @@ function guide(workflows) {
         ...Object.entries(configuration.sources).map(([field, source]) => `${field}: ${source}`),
         ...workflows.workflowRefusals.map((refusal) => `Refused workflow ${refusal.scope} ${refusal.label}: ${refusal.codes.join(", ")}`),
         "Commands for a second terminal:",
-        `  ${client} run develop-ticket --ticket <ticket-id>`,
-        `  ${client} run develop-issue --ticket <ticket-id>`,
-        `  ${client} run develop-prompt --prompt "<the text of the task>"`,
-        `  ${client} run develop-prd --prompt "<the idea>"`,
+        ...(workflows.declarations.length === 0
+            ? ["No workflow. Run: cloving init"]
+            : workflows.declarations.map(runCommand)),
         `  ${client} status`,
         `  ${client} questions`,
         `  ${client} watch`,
         "",
     ].join("\n");
 }
-const tokenPrices = parseTokenPrices(environment["CLOVING_TOKEN_PRICES"]);
+const { tokenPrices, fallbackModels, profiles } = await configuredTurnSettings(configuration).catch(startRefusal);
 const commands = parseCommands(environment["CLOVING_COMMANDS"]);
 const commandSeconds = parseCommandSeconds(environment["CLOVING_COMMAND_SECONDS"]);
-const fallbackModels = parseFallbackModels(environment["CLOVING_FALLBACK_MODELS"]);
-const profiles = await profilesFile();
 const schedules = parseSchedulesSetting(environment["CLOVING_SCHEDULES"]);
 const worker = selectedWorker();
 function nativeCommands() {
@@ -187,16 +208,21 @@ const harness = configuration.harness ?? {};
 const command = turnHarness({ worker, workerCommands, harness }).command;
 const nextCommand = listed("CLOVING_NEXT_COMMAND");
 const ticketCommand = listed("CLOVING_TICKET_COMMAND");
+const queueDailySpendLimitUsd = queueSpendLimit();
 const lanes = queueLanes();
-const queueMilliseconds = queueInterval({ nextCommand, ticketCommand });
-const manifests = await preparedProject({
-    product,
-    project,
+const queueMilliseconds = queueInterval();
+const retention = retentionSettings(environment);
+const places = {
     repository,
     home: homedir(),
     xdgConfigHome: environment["XDG_CONFIG_HOME"],
+};
+const folders = {
+    ...places,
     userFolder: absoluteWorkflowFolder(environment["CLOVING_WORKFLOW_FOLDER"]),
-});
+};
+const profilesPath = environment["CLOVING_PROFILES"];
+const manifests = await preparedProject({ product, project, ...folders });
 await launchers({
     commands,
     commandSeconds,
@@ -208,19 +234,35 @@ await launchers({
     harness,
     shell: environment["SHELL"] ?? "/bin/sh",
     profiles,
+    settingDigests: configurationDigests(configuration, profiles),
     command,
     ticketCommand,
     nextCommand,
     queueMilliseconds,
     queueLanes: lanes,
+    queueDailySpendLimitUsd,
     resetQueue: flags.includes("--reset-queue"),
+    retention,
     notifyCommand: listed("CLOVING_NOTIFY_COMMAND"),
     githubCommand: listed("CLOVING_GH_COMMAND"),
     requiredChecks: listed("CLOVING_REQUIRED_CHECKS") ?? null,
     checkCommand: listed("CLOVING_CHECK_COMMAND"),
     privateTexts: listed("CLOVING_PRIVATE_TEXTS"),
+    reloadCommand: [
+        process.execPath,
+        join(product, `tools/self-host-reload${extname(import.meta.filename)}`),
+        JSON.stringify({
+            directory,
+            ...places,
+            ...(profilesPath === undefined
+                ? {}
+                : { profilesFile: resolve(profilesPath) }),
+            fileVariables: Object.keys(environment).filter((name) => process.env[name] === undefined),
+        }),
+    ],
 }, manifests);
 process.exitCode = await hostedPrograms({
+    environment,
     project,
     product,
     runtime,

package/dist/tools/template-command.d.ts

@@ -0,0 +1 @@
+export declare function templateCommand(argumentsList: readonly string[]): Promise<number>;