cloving 1.1.0 → 1.2.0

package/dist/src/daemon/handoff-worker.js

@@ -0,0 +1,232 @@
+import { setTimeout as delay } from "node:timers/promises";
+import { send } from "../cli/client.js";
+import { record } from "../protocol.js";
+import { runIdentity } from "../service/submission-handler.js";
+import { located, readRecord, writeRecord } from "./operation-journal.js";
+const attempts = 10;
+const uncertain = [
+    "outcome-unknown",
+    "service-unavailable",
+    "service-busy",
+    "storage-failure",
+    "authority-unavailable",
+    "resolver-timeout",
+    "resolver-failure",
+];
+function own(intent) {
+    return (intent.kind === "protected-effect" &&
+        intent.capability === "handoff" &&
+        intent.method === "run" &&
+        intent.target !== undefined &&
+        intent.payload !== undefined &&
+        intent.leaseGeneration !== undefined);
+}
+function completed(runId) {
+    return {
+        kind: "completed",
+        worker: "handoff",
+        receipt: runId,
+        result: runId,
+    };
+}
+async function saved(folder) {
+    const stored = record(await readRecord(folder, "result.json"));
+    if (stored === undefined)
+        return undefined;
+    if (stored["kind"] === "completed" && typeof stored["result"] === "string")
+        return completed(stored["result"]);
+    if (stored["kind"] === "failed" && typeof stored["reason"] === "string")
+        return { kind: "failed", reason: stored["reason"] };
+    throw new Error("invalid-handoff-journal");
+}
+async function retained(folder, outcome) {
+    if (["completed", "failed"].includes(outcome.kind))
+        await writeRecord(folder, {
+            name: "result.json",
+            value: outcome,
+            permissions: "private",
+        });
+    return outcome;
+}
+function outcome(reply) {
+    if (reply.kind !== "error") {
+        const runId = reply.result["runId"];
+        return reply.result["kind"] === "run-admitted" && typeof runId === "string"
+            ? completed(runId)
+            : { kind: "unknown" };
+    }
+    if (reply.error.code === "workflow-active")
+        return undefined;
+    if (reply.error.code === "chain-limit")
+        return { kind: "failed", reason: "chain-limit" };
+    return uncertain.includes(reply.error.code)
+        ? { kind: "unknown" }
+        : { kind: "failed", reason: "handoff-input-refused" };
+}
+async function input(settings, intent) {
+    const text = await settings.document(intent.payload);
+    try {
+        return record(JSON.parse(text ?? "null"));
+    }
+    catch {
+        return undefined;
+    }
+}
+async function submit(settings, call) {
+    const { intent, signal } = call;
+    if (!own(intent))
+        return { kind: "failed", reason: "unsupported-effect" };
+    const { folder } = located(settings.journal, intent.operationId);
+    const known = await saved(folder);
+    if (known !== undefined)
+        return known;
+    const submission = await input(settings, intent);
+    if (submission === undefined)
+        return retained(folder, {
+            kind: "failed",
+            reason: "handoff-input-refused",
+        });
+    const previous = await busyAttempts(folder);
+    for (let busy = previous; busy < attempts; busy += 1) {
+        if (!(await allowed(settings, { intent, signal })))
+            return { kind: "unknown" };
+        const reply = await submitted(settings, {
+            intent,
+            submission,
+            signal,
+            folder,
+        });
+        if (reply === undefined)
+            return { kind: "unknown" };
+        const result = outcome(reply);
+        if (result !== undefined)
+            return retained(folder, result);
+        await writeRecord(folder, {
+            name: "attempts.json",
+            value: { busy: busy + 1 },
+            permissions: "private",
+        });
+        if (busy + 1 < attempts)
+            await delay(settings.retryMilliseconds ?? 1000, undefined, {
+                signal,
+            }).catch(() => undefined);
+    }
+    return retained(folder, { kind: "failed", reason: "handoff-start-timeout" });
+}
+async function submitted(settings, call) {
+    const { intent, submission, folder, signal } = call;
+    const marker = (status) => writeRecord(folder, {
+        name: "submission.json",
+        value: { processId: process.pid, pending: status === "pending" },
+        permissions: "private",
+    });
+    await marker("pending");
+    if (!(await allowed(settings, { intent, signal }))) {
+        await marker("settled");
+        return undefined;
+    }
+    const reply = await send(settings.runtimeDirectory, {
+        operation: "run",
+        parameters: {
+            workflowId: intent.target,
+            source: "input",
+            submission,
+            commandId: `handoff:${intent.operationId}`,
+        },
+    });
+    if (reply.kind !== "error" || reply.error.code !== "outcome-unknown")
+        await marker("settled");
+    return reply;
+}
+async function allowed(settings, call) {
+    try {
+        call.signal.throwIfAborted();
+        const status = await settings.runStatus(call.intent.runId);
+        call.signal.throwIfAborted();
+        return status === "running";
+    }
+    catch {
+        return false;
+    }
+}
+async function inFlight(folder) {
+    const marker = record(await readRecord(folder, "submission.json"));
+    if (marker?.["pending"] !== true)
+        return false;
+    const pid = marker["processId"];
+    if (typeof pid !== "number" || !Number.isSafeInteger(pid) || pid <= 0)
+        throw new Error("invalid-handoff-journal");
+    try {
+        process.kill(pid, 0);
+        return true;
+    }
+    catch (error) {
+        return !(error instanceof Error &&
+            "code" in error &&
+            error.code === "ESRCH");
+    }
+}
+async function busyAttempts(folder) {
+    const previous = record(await readRecord(folder, "attempts.json"))?.["busy"] ?? 0;
+    if (typeof previous !== "number" ||
+        !Number.isSafeInteger(previous) ||
+        previous < 0 ||
+        previous > attempts)
+        throw new Error("invalid-handoff-journal");
+    return previous;
+}
+async function observed(settings, intent) {
+    if (!own(intent))
+        return { kind: "failed", reason: "unsupported-effect" };
+    const known = await saved(located(settings.journal, intent.operationId).folder);
+    if (known !== undefined)
+        return known;
+    const reply = await send(settings.runtimeDirectory, {
+        operation: "status",
+        parameters: { runId: runIdentity(`handoff:${intent.operationId}`) },
+    });
+    if (reply.kind === "error")
+        return reply.error.code === "unknown-run"
+            ? {
+                kind: (await inFlight(located(settings.journal, intent.operationId).folder))
+                    ? "unknown"
+                    : "absent",
+            }
+            : { kind: "unknown" };
+    const runId = record(reply.result["run"])?.["runId"];
+    return typeof runId === "string" ? completed(runId) : { kind: "unknown" };
+}
+export function handoffWorker(settings) {
+    const pending = new Map();
+    const deliver = (intent, signal) => {
+        const known = pending.get(intent.operationId);
+        if (known !== undefined)
+            return known.result;
+        const stopping = new AbortController();
+        const result = submit(settings, {
+            intent,
+            signal: AbortSignal.any([signal, stopping.signal]),
+        }).finally(() => pending.delete(intent.operationId));
+        pending.set(intent.operationId, { stopping, result });
+        return result;
+    };
+    return {
+        deliver,
+        reconcile: (intent) => observed(settings, intent),
+        retry: async (intent, signal) => {
+            if (!own(intent))
+                return { kind: "failed", reason: "unsupported-effect" };
+            const { folder } = located(settings.journal, intent.operationId);
+            if (!(await inFlight(folder)))
+                return { kind: "unknown" };
+            return deliver(intent, signal);
+        },
+        stop: async (intent) => {
+            const active = pending.get(intent.operationId);
+            active?.stopping.abort();
+            await active?.result;
+            const known = await observed(settings, intent);
+            return known.kind === "absent" ? { kind: "stopped" } : known;
+        },
+    };
+}

package/dist/src/daemon/issue-queue.d.ts

@@ -2,6 +2,7 @@ interface QueueSettings {
     readonly directory: string;
     readonly runtimeDirectory: string;
     readonly intervalMilliseconds: number;
+    readonly lanes?: number;
     readonly resetQueue?: boolean;
 }
 export declare function issueQueue(settings: QueueSettings): {

package/dist/src/daemon/issue-queue.js

@@ -76,6 +76,34 @@ async function recover(settings, pending) {
         await pending.save();
     }
 }
+async function trackedRuns(runtimeDirectory, commands) {
+    if (commands.length === 0)
+        return [];
+    if (commands.length === 1) {
+        const commandId = String(commands[0]);
+        const run = await runStatus(runtimeDirectory, commandId);
+        if (run === undefined)
+            throw new Error("queue-run-missing");
+        return [{ commandId, run }];
+    }
+    const reply = await send(runtimeDirectory, {
+        operation: "status",
+        parameters: { runIds: commands.map(runIdentity) },
+    });
+    if (reply.kind === "error")
+        throw new Error(reply.error.code);
+    const runs = reply.result["runs"];
+    if (!Array.isArray(runs))
+        throw new Error("invalid-run-list");
+    return commands.map((commandId) => {
+        const run = runs
+            .map(record)
+            .find((item) => item?.["runId"] === runIdentity(commandId));
+        if (run === undefined)
+            throw new Error("queue-run-missing");
+        return { commandId, run };
+    });
+}
 function handedOff(run) {
     const outcome = record(run["outcome"]);
     return (run["state"] === "completed" &&
@@ -88,33 +116,39 @@ function checkFailureLimit(pending) {
     if (pending.failures.length >= 3)
         throw new QueueFailureLimit(`3 runs in a row ended without handoff: ${pending.failures.join(", ")}`);
 }
+async function retired(pending, ending) {
+    const { commandId, run } = ending;
+    pending.commands.delete(commandId);
+    if (handedOff(run))
+        pending.failures.length = 0;
+    else
+        pending.failures.push(runIdentity(commandId));
+    await pending.save();
+    checkFailureLimit(pending);
+    if (handedOff(run))
+        return;
+    const outcome = record(run["outcome"]);
+    process.stderr.write(`Queue run ended ${String(outcome?.["kind"] ?? run["state"])}${outcome?.["issueUpdate"] === "failed" ? "; issueUpdate: failed" : ""}: ${runIdentity(commandId)}; continuing\n`);
+}
 async function completed(settings, pending) {
     checkFailureLimit(pending);
-    let pendingRun = false;
+    let pendingRuns = 0;
+    const endings = [];
     const tracked = [...pending.commands].filter((commandId) => commandId !== pending.retryCommandId);
-    for (const commandId of tracked) {
-        const run = await runStatus(settings.runtimeDirectory, commandId);
-        if (run === undefined)
-            throw new Error("queue-run-missing");
-        if (!terminal(run)) {
-            pendingRun = true;
-            continue;
-        }
-        pending.commands.delete(commandId);
-        if (handedOff(run))
-            pending.failures.length = 0;
+    for (const { commandId, run } of await trackedRuns(settings.runtimeDirectory, tracked)) {
+        if (terminal(run))
+            endings.push({ commandId, run });
         else
-            pending.failures.push(runIdentity(commandId));
-        await pending.save();
-        checkFailureLimit(pending);
-        if (!handedOff(run)) {
-            const outcome = record(run["outcome"]);
-            process.stderr.write(`Queue run ended ${String(outcome?.["kind"] ?? run["state"])}${outcome?.["issueUpdate"] === "failed" ? "; issueUpdate: failed" : ""}: ${runIdentity(commandId)}; continuing\n`);
-        }
+            pendingRuns += 1;
     }
-    return pendingRun;
+    endings.sort((first, second) => Number(first.run["terminalSequence"] ?? first.run["latestSequence"] ?? 0) -
+        Number(second.run["terminalSequence"] ?? second.run["latestSequence"] ?? 0));
+    for (const ending of endings)
+        await retired(pending, ending);
+    return pendingRuns;
 }
 async function active(runtimeDirectory) {
+    let count = 0;
     let after = "";
     do {
         const reply = await send(runtimeDirectory, {
@@ -126,17 +160,16 @@ async function active(runtimeDirectory) {
         const runs = reply.result["runs"];
         if (!Array.isArray(runs))
             throw new Error("invalid-run-list");
-        if (runs.some((value) => {
+        count += runs.filter((value) => {
             const run = record(value);
             return run?.["workflowId"] === "develop-issue" && !terminal(run);
-        }))
-            return true;
+        }).length;
         const next = reply.result["next"];
         if (next !== null && (typeof next !== "string" || next <= after))
             throw new Error("invalid-run-page");
         after = next;
     } while (after !== null);
-    return false;
+    return count;
 }
 async function submit(settings, pending) {
     const commandId = pending.retryCommandId ?? randomUUID();
@@ -163,19 +196,38 @@ async function submit(settings, pending) {
         process.stderr.write(`Queue ${reply.error.code}: command ${commandId}; will retry\n`);
         return;
     }
-    if (!["no-ready-ticket", "workflow-active"].includes(reply.error.code))
+    if (![
+        "no-ready-ticket",
+        "workflow-active",
+        "ticket-active",
+        "ticket-dependency-open",
+    ].includes(reply.error.code))
         throw new Error(reply.error.code);
     pending.commands.delete(commandId);
     await pending.save();
 }
+async function pendingUpdate(directory) {
+    return readFile(join(directory, "pending-self-update"), "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+}
+async function holdStarted(pending, observed) {
+    const { mark, activeRuns } = observed;
+    const timestamp = Number(mark);
+    if (!Number.isFinite(timestamp) || timestamp <= 0)
+        throw new Error("invalid-pending-self-update");
+    if (pending.selfUpdateHold?.mark !== mark || activeRuns > 0) {
+        pending.selfUpdateHold = { mark, startedAt: Date.now() };
+        await pending.save();
+    }
+    return pending.selfUpdateHold.startedAt;
+}
 function updateHold(directory, pending) {
     let notice;
-    return async () => {
-        const mark = await readFile(join(directory, "pending-self-update"), "utf8").catch((error) => {
-            if (error instanceof Error && "code" in error && error.code === "ENOENT")
-                return undefined;
-            throw error;
-        });
+    return async (activeRuns = 0) => {
+        const mark = await pendingUpdate(directory);
         if (mark === undefined && pending.selfUpdateHold !== undefined) {
             pending.selfUpdateHold = undefined;
             await pending.save();
@@ -184,14 +236,8 @@ function updateHold(directory, pending) {
             notice = undefined;
             return false;
         }
-        const timestamp = Number(mark);
-        if (!Number.isFinite(timestamp) || timestamp <= 0)
-            throw new Error("invalid-pending-self-update");
-        if (pending.selfUpdateHold?.mark !== mark) {
-            pending.selfUpdateHold = { mark, startedAt: Date.now() };
-            await pending.save();
-        }
-        const holding = Date.now() - pending.selfUpdateHold.startedAt < 30 * 60 * 1000;
+        const startedAt = await holdStarted(pending, { mark, activeRuns });
+        const holding = activeRuns > 0 || Date.now() - startedAt < 30 * 60 * 1000;
         const line = holding
             ? "Queue holding for pending self-update\n"
             : "Queue continuing after pending self-update time limit\n";
@@ -227,11 +273,17 @@ async function cycle(settings, signal) {
         await delay(settings.intervalMilliseconds, undefined, { signal }).catch(() => undefined);
         if (stopped())
             return;
-        if (await completed(settings, pending))
+        const pendingRuns = await completed(settings, pending);
+        const lanes = settings.lanes ?? 1;
+        if (pendingRuns >= lanes) {
+            await holding(pendingRuns);
             continue;
-        if (await active(settings.runtimeDirectory))
+        }
+        const activeRuns = await active(settings.runtimeDirectory);
+        const held = await holding(activeRuns);
+        if (activeRuns >= lanes || activeRuns < pendingRuns)
             continue;
-        if (await holding())
+        if (held)
             continue;
         if (stopped())
             return;

package/dist/src/daemon/notify-journal.d.ts

@@ -0,0 +1,18 @@
+import type { Notification } from "./notify-message.ts";
+export interface DeliveryEntry {
+    readonly key: string;
+    readonly message: Notification | null;
+    readonly state: "pending" | "sent" | "failed";
+    readonly attempts: number;
+    readonly lastAttemptAt: number | null;
+    readonly lastFailure: string | null;
+}
+export interface RecordedNotifications {
+    readonly entries: Map<string, DeliveryEntry>;
+    readonly retiredKeys: Set<string>;
+}
+export declare function recordedNotifications(file: string): Promise<RecordedNotifications | undefined>;
+export declare function saveNotifications(file: string, journal: {
+    readonly entries: ReadonlyMap<string, DeliveryEntry>;
+    readonly retiredKeys: ReadonlySet<string>;
+}): Promise<void>;

package/dist/src/daemon/notify-journal.js

@@ -0,0 +1,108 @@
+import { open, readFile, rename } from "node:fs/promises";
+import { record } from "../protocol.js";
+export async function recordedNotifications(file) {
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+    if (text === undefined)
+        return undefined;
+    const value = JSON.parse(text);
+    const journal = record(value);
+    if (journal?.["version"] === 1 && Array.isArray(journal["entries"])) {
+        const entries = journal["entries"].map(deliveryEntry);
+        const found = new Map(entries.map((entry) => [entry.key, entry]));
+        if (found.size !== entries.length)
+            throw new Error("invalid-notify-record");
+        return {
+            entries: found,
+            retiredKeys: recordedKeys(journal["retiredKeys"] ?? []),
+        };
+    }
+    const keys = recordedKeys(value);
+    return {
+        retiredKeys: new Set(),
+        entries: new Map([...keys].map((key) => [
+            key,
+            {
+                key,
+                message: null,
+                state: "sent",
+                attempts: 0,
+                lastAttemptAt: Date.now(),
+                lastFailure: null,
+            },
+        ])),
+    };
+}
+function recordedKeys(value) {
+    if (!Array.isArray(value) ||
+        !value.every((key) => typeof key === "string"))
+        throw new Error("invalid-notify-record");
+    return new Set(value.map(String));
+}
+function notification(value) {
+    const message = record(value);
+    if (message === undefined)
+        return false;
+    return (["decision", "blocked", "failed"].includes(String(message["event"])) &&
+        ["event", "runId", "workflowId", "subject", "body"].every((field) => typeof message[field] === "string"));
+}
+function deliveryState(value) {
+    return value === "pending" || value === "sent" || value === "failed";
+}
+function attemptCount(value) {
+    return (typeof value === "number" &&
+        Number.isInteger(value) &&
+        value >= 0 &&
+        value <= 5);
+}
+function attemptTime(value) {
+    return (value === null ||
+        (typeof value === "number" && Number.isSafeInteger(value) && value >= 0));
+}
+function deliveryEntry(value) {
+    const { key, message, state, attempts, lastAttemptAt, lastFailure } = record(value) ?? {};
+    if (typeof key !== "string" ||
+        !deliveryState(state) ||
+        !attemptCount(attempts))
+        throw new Error("invalid-notify-record");
+    if (!attemptTime(lastAttemptAt) ||
+        !(lastFailure === null || typeof lastFailure === "string"))
+        throw new Error("invalid-notify-record");
+    const content = { message, state };
+    if (!entryMessage(content))
+        throw new Error("invalid-notify-record");
+    return {
+        key,
+        message: content.message,
+        state,
+        attempts,
+        lastAttemptAt,
+        lastFailure,
+    };
+}
+function entryMessage(entry) {
+    return entry.message === null
+        ? entry.state === "sent"
+        : notification(entry.message);
+}
+export async function saveNotifications(file, journal) {
+    const temporary = await open(`${file}.tmp`, "w", 0o600);
+    try {
+        await temporary.chmod(0o600);
+        await temporary.writeFile(JSON.stringify({
+            version: 1,
+            entries: [...journal.entries.values()],
+            ...(journal.retiredKeys.size > 0
+                ? { retiredKeys: [...journal.retiredKeys] }
+                : {}),
+        }) + "\n");
+        await temporary.sync();
+    }
+    finally {
+        await temporary.close();
+    }
+    await rename(`${file}.tmp`, file);
+}

package/dist/src/daemon/notify-message.js

@@ -1,3 +1,4 @@
+import { operationalDecision } from "../decision-visibility.js";
 import { decisionLines, runStatus } from "../cli/render.js";
 import { record } from "../protocol.js";
 function body(text, commands) {
@@ -17,7 +18,7 @@ function message(run, content) {
     };
 }
 export function decisionNotification(run, decision) {
-    const lines = decisionLines(decision);
+    const lines = decisionLines(operationalDecision(decision, String(run["workflowId"])));
     const allowed = decision["allowed"];
     const count = Array.isArray(allowed) ? allowed.length : 0;
     const split = lines.length - count;