cloving 1.1.0 → 1.2.0

package/dist/tools/self-host.js

@@ -1,13 +1,11 @@
 import { isProfilesFile } from "../src/turn-profile.js";
-import { parseTokenPrices } from "../src/token-prices.js";
-import { execFileSync, spawn } from "node:child_process";
-import { copyFile, mkdir, readFile, rm, symlink, writeFile, } from "node:fs/promises";
+import { parseFallbackModels, parseTokenPrices, } from "../src/token-prices.js";
+import { execFileSync } from "node:child_process";
+import { readFile, realpath, writeFile } from "node:fs/promises";
+import { homedir } from "node:os";
 import { dirname, extname, join, resolve } from "node:path";
-import packageMetadata from "#package" with { type: "json" };
-import { findPackageJSON } from "node:module";
 import { pathToFileURL } from "node:url";
-import { workflowDeclarations, workflowFiles, workflowManifests, } from "./self-host-workflows.js";
-import { isWorkflowManifest } from "./workflow-manifest.js";
+import { preparedProject, } from "./self-host-project.js";
 import { selfUpdateMilliseconds } from "./self-update-host.js";
 import { hostedPrograms } from "./self-host-processes.js";
 const product = resolve(import.meta.dirname, "..");
@@ -32,7 +30,7 @@ if (runtimeArgument === undefined)
     throw new Error("The start program needs --runtime. Use: cloving start");
 const runtime = resolve(runtimeArgument);
 const directory = dirname(runtime);
-const repository = topLevel(resolve(flag("--repository") ?? process.cwd()));
+const repository = await realpath(topLevel(resolve(flag("--repository") ?? process.cwd())));
 const updateMilliseconds = await selfUpdateMilliseconds({
     seconds: process.env["CLOVING_SELF_UPDATE_SECONDS"],
     repository,
@@ -84,6 +82,16 @@ async function profilesFile() {
     }
     throw new Error("The daemon did not start: invalid-profiles");
 }
+function queueLanes() {
+    const value = process.env["CLOVING_QUEUE_LANES"];
+    if (value === undefined)
+        return 1;
+    if (!/^[1-4]$/.test(value))
+        throw new Error("The daemon did not start: invalid-queue-lanes");
+    if (process.env["CLOVING_QUEUE_SECONDS"] === undefined)
+        throw new Error("The daemon did not start: queue-lanes-needs-queue");
+    return Number(value);
+}
 function queueInterval(commands) {
     const value = process.env["CLOVING_QUEUE_SECONDS"];
     if (value === undefined)
@@ -109,61 +117,6 @@ function listed(name) {
         throw new Error(`${name} is not a JSON list of texts.`);
     return list.map(String);
 }
-function record(value) {
-    return typeof value === "object" && value !== null ? { ...value } : {};
-}
-function ended(argumentsList, environment) {
-    return new Promise((settle, reject) => {
-        const child = spawn(process.execPath, argumentsList, {
-            cwd: project,
-            env: environment,
-            stdio: "inherit",
-        });
-        for (const signal of ["SIGINT", "SIGTERM"])
-            process.on(signal, () => child.kill(signal));
-        child.once("error", reject);
-        child.once("close", (status) => {
-            settle(status ?? 1);
-        });
-    });
-}
-async function linkedPackages() {
-    const modules = join(project, "node_modules");
-    await mkdir(join(modules, "cloving"), { recursive: true });
-    for (const name of Object.keys(packageMetadata.dependencies)) {
-        const location = findPackageJSON(name, join(product, "package.json"));
-        if (location === undefined)
-            throw new Error(`missing-runtime-dependency: ${name}`);
-        await mkdir(dirname(join(modules, name)), { recursive: true });
-        await symlink(dirname(location), join(modules, name));
-    }
-    const manifest = record(packageMetadata);
-    const exported = Object.keys(record(manifest["exports"])).filter((subpath) => subpath.startsWith("./workflows"));
-    await symlink(join(product, "src"), join(modules, "cloving", "src"));
-    await writeFile(join(modules, "cloving", "package.json"), JSON.stringify({
-        name: manifest["name"],
-        version: manifest["version"],
-        type: "module",
-        exports: Object.fromEntries(exported.map((subpath) => [
-            subpath,
-            `./src/${subpath.slice("./workflows/".length) || "workflows"}${extname(import.meta.filename)}`,
-        ])),
-    }));
-}
-async function registered() {
-    const compiled = JSON.parse(await readFile(join(project, "artifacts/manifest.json"), "utf8"));
-    if (!isWorkflowManifest(compiled))
-        throw new Error("The compiled workflow manifest is invalid.");
-    const manifests = workflowManifests(compiled);
-    const declarations = await workflowDeclarations(join(product, "workflows"), manifests);
-    const paths = new Map();
-    for (const [functionName, manifest] of Object.entries(workflowManifests(compiled, declarations))) {
-        const path = join(project, `${manifest.name}.workflow.json`);
-        await writeFile(path, JSON.stringify(manifest));
-        paths.set(functionName, path);
-    }
-    return { manifests: Object.fromEntries(paths), declarations };
-}
 function source(file) {
     return JSON.stringify(pathToFileURL(join(product, file.replace(/\.ts$/, extname(import.meta.filename)))).href);
 }
@@ -171,7 +124,6 @@ async function launchers(commands, workflows) {
     const settings = {
         ...commands,
         directory,
-        repository,
         concurrency,
         runtimeDirectory: runtime,
         ...workflows,
@@ -210,23 +162,8 @@ function guide() {
         "",
     ].join("\n");
 }
-async function prepared() {
-    await rm(project, { recursive: true, force: true });
-    await mkdir(join(project, "workflows"), { recursive: true });
-    await writeFile(join(project, "package.json"), '{"name":"cloving-self-host","private":true,"type":"module"}');
-    for (const file of await workflowFiles(join(product, "workflows")))
-        await copyFile(join(product, "workflows", file), join(project, "workflows", file));
-    await linkedPackages();
-    const compiled = await ended([
-        join(product, `tools/compile-workflows${extname(import.meta.filename)}`),
-        "workflows",
-        "artifacts",
-    ], process.env);
-    if (compiled !== 0)
-        throw new Error("The workflows did not compile.");
-    return registered();
-}
 const tokenPrices = parseTokenPrices(process.env["CLOVING_TOKEN_PRICES"]);
+const fallbackModels = parseFallbackModels(process.env["CLOVING_FALLBACK_MODELS"]);
 const profiles = await profilesFile();
 const worker = selectedWorker();
 const command = worker === "codex"
@@ -234,16 +171,25 @@ const command = worker === "codex"
     : (listed("CLOVING_CLAUDE_COMMAND") ?? defaultCommand);
 const nextCommand = listed("CLOVING_NEXT_COMMAND");
 const ticketCommand = listed("CLOVING_TICKET_COMMAND");
+const lanes = queueLanes();
 const queueMilliseconds = queueInterval({ nextCommand, ticketCommand });
-const manifests = await prepared();
+const manifests = await preparedProject({
+    product,
+    project,
+    repository,
+    home: homedir(),
+    xdgConfigHome: process.env["XDG_CONFIG_HOME"],
+});
 await launchers({
     tokenPrices,
+    fallbackModels,
     worker,
     profiles,
     command,
     ticketCommand,
     nextCommand,
     queueMilliseconds,
+    queueLanes: lanes,
     resetQueue: flags.includes("--reset-queue"),
     notifyCommand: listed("CLOVING_NOTIFY_COMMAND"),
     githubCommand: listed("CLOVING_GH_COMMAND"),

package/dist/tools/workflow-bundle-dependencies.d.ts

@@ -0,0 +1,5 @@
+export declare function workflowPackageRoot(directory: string): Promise<string>;
+export declare function workflowBundleDependencies(input: {
+    readonly product: string;
+    readonly packages: readonly string[];
+}): Promise<string>;

package/dist/tools/workflow-bundle-dependencies.js

@@ -0,0 +1,93 @@
+import { createHash } from "node:crypto";
+import { access, readFile, realpath } from "node:fs/promises";
+import { findPackageJSON } from "node:module";
+import { dirname, join, resolve } from "node:path";
+import { record } from "../src/protocol.js";
+export async function workflowPackageRoot(directory) {
+    const root = resolve(directory);
+    try {
+        await access(join(root, "package.json"));
+        return root;
+    }
+    catch (error) {
+        if (!(error instanceof Error && "code" in error && error.code === "ENOENT"))
+            throw error;
+    }
+    const parent = dirname(root);
+    if (parent === root)
+        throw new Error("workflow-package-unavailable");
+    return workflowPackageRoot(parent);
+}
+function references(manifest) {
+    const dependencies = record(manifest["dependencies"]) ?? {};
+    const optional = record(manifest["optionalDependencies"]) ?? {};
+    const peers = record(manifest["peerDependencies"]) ?? {};
+    const peerMetadata = record(manifest["peerDependenciesMeta"]) ?? {};
+    return Object.keys({ ...dependencies, ...optional, ...peers })
+        .sort()
+        .map((name) => ({
+        name,
+        optional: Object.hasOwn(optional, name) ||
+            (!Object.hasOwn(dependencies, name) &&
+                record(peerMetadata[name])?.["optional"] === true),
+    }));
+}
+async function manifestFile(from, reference) {
+    try {
+        const file = findPackageJSON(reference.name, from);
+        if (file !== undefined)
+            return await realpath(file);
+    }
+    catch (error) {
+        if (!(error instanceof Error &&
+            "code" in error &&
+            error.code === "ERR_MODULE_NOT_FOUND"))
+            throw error;
+    }
+    if (reference.optional)
+        return undefined;
+    throw new Error("workflow-dependency-unavailable");
+}
+async function capturePackage(capture, input) {
+    const file = await manifestFile(input.from, input);
+    if (file === undefined)
+        return null;
+    const previous = capture.seen.get(file);
+    if (previous !== undefined)
+        return previous;
+    const manifest = await readFile(file, "utf8");
+    const parsed = JSON.parse(manifest);
+    const fields = record(parsed);
+    if (typeof fields?.["name"] !== "string" ||
+        typeof fields["version"] !== "string")
+        throw new Error("invalid-workflow-dependency");
+    const captured = { manifest, dependencies: {} };
+    const index = capture.packages.length;
+    capture.seen.set(file, index);
+    capture.packages.push(captured);
+    for (const reference of references(fields))
+        captured.dependencies[reference.name] = await capturePackage(capture, {
+            ...reference,
+            from: file,
+        });
+    return index;
+}
+export async function workflowBundleDependencies(input) {
+    const capture = { seen: new Map(), packages: [] };
+    const roots = {};
+    const manifest = join(await workflowPackageRoot(input.product), "package.json");
+    for (const name of [...input.packages].sort())
+        roots[name] = await capturePackage(capture, {
+            name,
+            from: manifest,
+            optional: false,
+        });
+    return JSON.stringify({
+        node: process.version,
+        product: createHash("sha256")
+            .update(await readFile(manifest))
+            .digest("hex"),
+        roots,
+        packages: capture.packages,
+    });
+}

package/dist/tools/workflow-bundles.d.ts

@@ -0,0 +1,20 @@
+interface BundleInput {
+    readonly directory: string;
+    readonly repository: string;
+    readonly project: string;
+}
+interface BundleLookup {
+    readonly directory: string;
+    readonly repository: string;
+    readonly id: string;
+}
+type BundleResult = {
+    readonly kind: "available";
+    readonly directory: string;
+} | {
+    readonly kind: "unavailable";
+    readonly code: "pinned-workflow-unavailable";
+};
+export declare function readRetainedWorkflowBundle(input: BundleLookup): Promise<BundleResult>;
+export declare function retainWorkflowBundle(input: BundleInput): Promise<string>;
+export {};

package/dist/tools/workflow-bundles.js

@@ -0,0 +1,173 @@
+import { createHash } from "node:crypto";
+import { constants } from "node:fs";
+import { lstat, mkdir, mkdtemp, open, readdir, realpath, rename, rm, writeFile, } from "node:fs/promises";
+import { dirname, join } from "node:path";
+const unavailable = {
+    kind: "unavailable",
+    code: "pinned-workflow-unavailable",
+};
+export async function readRetainedWorkflowBundle(input) {
+    if (!/^[a-f0-9]{64}$/.test(input.id))
+        return unavailable;
+    try {
+        const directory = join(input.directory, input.id);
+        if (!(await lstat(directory)).isDirectory())
+            return unavailable;
+        const repository = await realpath(input.repository);
+        const contents = bundleContents(repository, await projectFiles(directory));
+        const stored = await regularFile(join(directory, "bundle.json"));
+        if (digest(contents) !== input.id || stored.toString("utf8") !== contents)
+            return unavailable;
+        return { kind: "available", directory };
+    }
+    catch {
+        return unavailable;
+    }
+}
+function digest(bytes) {
+    return createHash("sha256").update(bytes).digest("hex");
+}
+async function regularFile(path) {
+    const file = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
+    try {
+        if (!(await file.stat()).isFile())
+            throw new Error("invalid-bundle-file");
+        return await file.readFile();
+    }
+    finally {
+        await file.close();
+    }
+}
+async function treeFiles(root, name) {
+    const path = join(root, name);
+    const entry = await lstat(path);
+    if (entry.isFile())
+        return [{ name, bytes: await regularFile(path) }];
+    if (!entry.isDirectory())
+        throw new Error("invalid-bundle-file");
+    const files = [];
+    for (const child of (await readdir(path)).sort())
+        files.push(...(await treeFiles(root, `${name}/${child}`)));
+    return files;
+}
+async function projectFiles(project) {
+    const files = [];
+    const names = (await readdir(project)).filter((name) => name.endsWith(".json") && name !== "bundle.json");
+    for (const name of [...names, "artifacts", "workflows"].sort())
+        files.push(...(await treeFiles(project, name)));
+    const required = [
+        "package.json",
+        "declarations.json",
+        "dependencies.json",
+        ...[
+            "client.mjs",
+            "flow.mjs",
+            "steps.mjs",
+            "manifest.json",
+            "flow.mjs.__wf_tmp.js.debug.json",
+            "steps.mjs.debug.json",
+        ].map((name) => `artifacts/${name}`),
+    ];
+    if (required.some((name) => !files.some((file) => file.name === name)))
+        throw new Error("incomplete-workflow-bundle");
+    return files;
+}
+function bundleContents(repository, files) {
+    return JSON.stringify({
+        schemaVersion: 1,
+        repository,
+        files: Object.fromEntries(files.map(({ name, bytes }) => [name, digest(bytes)])),
+    });
+}
+async function validatedId(input) {
+    if ((await readRetainedWorkflowBundle(input)).kind !== "available")
+        throw new Error("pinned-workflow-unavailable");
+    return input.id;
+}
+async function existingEntry(path) {
+    return lstat(path).catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+}
+async function retainedBefore(input, contents) {
+    try {
+        const history = join(input.directory, ".history");
+        await mkdir(history, { recursive: true, mode: 0o700 });
+        if (!(await lstat(history)).isDirectory())
+            throw new Error("pinned-workflow-unavailable");
+        const directory = join(history, input.id);
+        const entry = await existingEntry(directory);
+        if (entry === undefined)
+            return false;
+        if (!entry.isDirectory())
+            throw new Error("pinned-workflow-unavailable");
+        const receipt = await regularFile(join(directory, "bundle.json"));
+        if (receipt.toString("utf8") !== contents)
+            throw new Error("pinned-workflow-unavailable");
+        return true;
+    }
+    catch {
+        throw new Error("pinned-workflow-unavailable");
+    }
+}
+async function rememberBundle(input, contents) {
+    const history = join(input.directory, ".history");
+    const temporary = await mkdtemp(join(history, ".pending-"));
+    try {
+        await writeFile(join(temporary, "bundle.json"), contents, {
+            flag: "wx",
+            mode: 0o600,
+        });
+        await publishDirectory(temporary, join(history, input.id));
+        await retainedBefore(input, contents);
+    }
+    finally {
+        await rm(temporary, { recursive: true, force: true });
+    }
+}
+async function publishDirectory(from, to) {
+    await rename(from, to).catch((error) => {
+        if (error instanceof Error &&
+            "code" in error &&
+            ["ENOTEMPTY", "EEXIST"].includes(String(error.code)))
+            return;
+        throw error;
+    });
+}
+export async function retainWorkflowBundle(input) {
+    const repository = await realpath(input.repository);
+    const files = await projectFiles(input.project);
+    const contents = bundleContents(repository, files);
+    const id = digest(contents);
+    await mkdir(input.directory, { recursive: true, mode: 0o700 });
+    const destination = join(input.directory, id);
+    const lookup = { ...input, id };
+    if (await retainedBefore(lookup, contents))
+        return validatedId(lookup);
+    if (await existingEntry(destination)) {
+        await validatedId(lookup);
+        await rememberBundle(lookup, contents);
+        return id;
+    }
+    const temporary = await mkdtemp(join(input.directory, ".pending-"));
+    try {
+        for (const { name, bytes } of files) {
+            const path = join(temporary, name);
+            await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+            await writeFile(path, bytes, { flag: "wx", mode: 0o600 });
+        }
+        await writeFile(join(temporary, "bundle.json"), contents, {
+            flag: "wx",
+            mode: 0o600,
+        });
+        await publishDirectory(temporary, destination);
+    }
+    finally {
+        await rm(temporary, { recursive: true, force: true });
+    }
+    await validatedId(lookup);
+    await rememberBundle(lookup, contents);
+    return id;
+}

package/dist/tools/workflow-candidate-counts.d.ts

@@ -0,0 +1,8 @@
+import ts from "typescript";
+import { type CandidateSymbols } from "./workflow-candidate-syntax.ts";
+interface Totals {
+    readonly knownCallCount: number;
+    readonly knownUnitTotal: number;
+}
+export declare function candidateEffectTotals(source: ts.SourceFile, symbols: CandidateSymbols): Totals;
+export {};