cloving 1.0.1 → 1.2.0

package/dist/src/daemon/issue-queue.js

@@ -1,10 +1,27 @@
 import { randomUUID } from "node:crypto";
-import { readFile, rename, writeFile } from "node:fs/promises";
+import { readFile, rename, rm, writeFile } from "node:fs/promises";
 import { join } from "node:path";
 import { setTimeout as delay } from "node:timers/promises";
 import { send } from "../cli/client.js";
 import { record } from "../protocol.js";
 import { runIdentity } from "../service/submission-handler.js";
+function savedHold(value) {
+    if (value === undefined)
+        return undefined;
+    const held = record(value);
+    if (typeof held?.["mark"] !== "string" ||
+        typeof held["startedAt"] !== "number" ||
+        !Number.isFinite(held["startedAt"]) ||
+        held["startedAt"] <= 0)
+        throw new Error("invalid-queue-commands");
+    return { mark: held["mark"], startedAt: held["startedAt"] };
+}
+function identifiers(value) {
+    if (!Array.isArray(value) ||
+        !value.every((id) => typeof id === "string" && id.length > 0))
+        throw new Error("invalid-queue-commands");
+    return value.map(String);
+}
 async function journal(directory) {
     const file = join(directory, "queue-commands.json");
     const text = await readFile(file, "utf8").catch((error) => {
@@ -13,21 +30,26 @@ async function journal(directory) {
         throw error;
     });
     const value = JSON.parse(text);
-    if (!Array.isArray(value) ||
-        !value.every((id) => typeof id === "string" && id.length > 0))
+    const saved = record(value);
+    const commands = new Set(identifiers(Array.isArray(value) ? value : saved?.["commands"]));
+    const failures = Array.isArray(value) ? [] : identifiers(saved?.["failures"]);
+    if (failures.length > 3)
         throw new Error("invalid-queue-commands");
-    const commands = new Set(value.map(String));
-    return {
+    const pending = {
         commands,
+        failures,
         retryCommandId: undefined,
+        selfUpdateHold: savedHold(saved?.["selfUpdateHold"]),
         save: async () => {
-            await writeFile(`${file}.tmp`, JSON.stringify([...commands]) + "\n", {
-                mode: 0o600,
-                flush: true,
-            });
+            await writeFile(`${file}.tmp`, JSON.stringify({
+                commands: [...commands],
+                failures,
+                selfUpdateHold: pending.selfUpdateHold,
+            }) + "\n", { mode: 0o600, flush: true });
             await rename(`${file}.tmp`, file);
         },
     };
+    return pending;
 }
 function terminal(run) {
     return ["completed", "failed", "cancelled"].includes(String(run["state"]));
@@ -54,34 +76,79 @@ async function recover(settings, pending) {
         await pending.save();
     }
 }
-async function completed(settings, pending) {
-    let failure;
-    let pendingRun = false;
-    const tracked = [...pending.commands].filter((commandId) => commandId !== pending.retryCommandId);
-    for (const commandId of tracked) {
-        const run = await runStatus(settings.runtimeDirectory, commandId);
+async function trackedRuns(runtimeDirectory, commands) {
+    if (commands.length === 0)
+        return [];
+    if (commands.length === 1) {
+        const commandId = String(commands[0]);
+        const run = await runStatus(runtimeDirectory, commandId);
         if (run === undefined)
             throw new Error("queue-run-missing");
-        if (!terminal(run)) {
-            pendingRun = true;
-            continue;
-        }
-        const outcome = record(run["outcome"]);
-        if (run["state"] !== "completed" ||
-            outcome?.["kind"] !== "handoff_requested")
-            failure ??= JSON.stringify({
-                runId: run["runId"],
-                state: run["state"],
-                outcome: run["outcome"] ?? null,
-            });
-        pending.commands.delete(commandId);
-        await pending.save();
+        return [{ commandId, run }];
     }
-    if (failure !== undefined)
-        throw new Error(failure);
-    return pendingRun;
+    const reply = await send(runtimeDirectory, {
+        operation: "status",
+        parameters: { runIds: commands.map(runIdentity) },
+    });
+    if (reply.kind === "error")
+        throw new Error(reply.error.code);
+    const runs = reply.result["runs"];
+    if (!Array.isArray(runs))
+        throw new Error("invalid-run-list");
+    return commands.map((commandId) => {
+        const run = runs
+            .map(record)
+            .find((item) => item?.["runId"] === runIdentity(commandId));
+        if (run === undefined)
+            throw new Error("queue-run-missing");
+        return { commandId, run };
+    });
+}
+function handedOff(run) {
+    const outcome = record(run["outcome"]);
+    return (run["state"] === "completed" &&
+        outcome?.["kind"] === "handoff_requested" &&
+        outcome["issueUpdate"] !== "failed");
+}
+class QueueFailureLimit extends Error {
+}
+function checkFailureLimit(pending) {
+    if (pending.failures.length >= 3)
+        throw new QueueFailureLimit(`3 runs in a row ended without handoff: ${pending.failures.join(", ")}`);
+}
+async function retired(pending, ending) {
+    const { commandId, run } = ending;
+    pending.commands.delete(commandId);
+    if (handedOff(run))
+        pending.failures.length = 0;
+    else
+        pending.failures.push(runIdentity(commandId));
+    await pending.save();
+    checkFailureLimit(pending);
+    if (handedOff(run))
+        return;
+    const outcome = record(run["outcome"]);
+    process.stderr.write(`Queue run ended ${String(outcome?.["kind"] ?? run["state"])}${outcome?.["issueUpdate"] === "failed" ? "; issueUpdate: failed" : ""}: ${runIdentity(commandId)}; continuing\n`);
+}
+async function completed(settings, pending) {
+    checkFailureLimit(pending);
+    let pendingRuns = 0;
+    const endings = [];
+    const tracked = [...pending.commands].filter((commandId) => commandId !== pending.retryCommandId);
+    for (const { commandId, run } of await trackedRuns(settings.runtimeDirectory, tracked)) {
+        if (terminal(run))
+            endings.push({ commandId, run });
+        else
+            pendingRuns += 1;
+    }
+    endings.sort((first, second) => Number(first.run["terminalSequence"] ?? first.run["latestSequence"] ?? 0) -
+        Number(second.run["terminalSequence"] ?? second.run["latestSequence"] ?? 0));
+    for (const ending of endings)
+        await retired(pending, ending);
+    return pendingRuns;
 }
 async function active(runtimeDirectory) {
+    let count = 0;
     let after = "";
     do {
         const reply = await send(runtimeDirectory, {
@@ -93,17 +160,16 @@ async function active(runtimeDirectory) {
         const runs = reply.result["runs"];
         if (!Array.isArray(runs))
             throw new Error("invalid-run-list");
-        if (runs.some((value) => {
+        count += runs.filter((value) => {
             const run = record(value);
             return run?.["workflowId"] === "develop-issue" && !terminal(run);
-        }))
-            return true;
+        }).length;
         const next = reply.result["next"];
         if (next !== null && (typeof next !== "string" || next <= after))
             throw new Error("invalid-run-page");
         after = next;
     } while (after !== null);
-    return false;
+    return count;
 }
 async function submit(settings, pending) {
     const commandId = pending.retryCommandId ?? randomUUID();
@@ -121,38 +187,115 @@ async function submit(settings, pending) {
     pending.retryCommandId = undefined;
     if (reply.kind === "result")
         return;
-    if (reply.error.code === "resolver-timeout") {
+    if ([
+        "resolver-timeout",
+        "next-ticket-not-available",
+        "ticket-not-available",
+    ].includes(reply.error.code)) {
         pending.retryCommandId = commandId;
-        process.stderr.write(`Queue resolver-timeout: command ${commandId}; will retry\n`);
+        process.stderr.write(`Queue ${reply.error.code}: command ${commandId}; will retry\n`);
         return;
     }
-    if (!["no-ready-ticket", "workflow-active"].includes(reply.error.code))
+    if (![
+        "no-ready-ticket",
+        "workflow-active",
+        "ticket-active",
+        "ticket-dependency-open",
+    ].includes(reply.error.code))
         throw new Error(reply.error.code);
     pending.commands.delete(commandId);
     await pending.save();
 }
+async function pendingUpdate(directory) {
+    return readFile(join(directory, "pending-self-update"), "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+}
+async function holdStarted(pending, observed) {
+    const { mark, activeRuns } = observed;
+    const timestamp = Number(mark);
+    if (!Number.isFinite(timestamp) || timestamp <= 0)
+        throw new Error("invalid-pending-self-update");
+    if (pending.selfUpdateHold?.mark !== mark || activeRuns > 0) {
+        pending.selfUpdateHold = { mark, startedAt: Date.now() };
+        await pending.save();
+    }
+    return pending.selfUpdateHold.startedAt;
+}
+function updateHold(directory, pending) {
+    let notice;
+    return async (activeRuns = 0) => {
+        const mark = await pendingUpdate(directory);
+        if (mark === undefined && pending.selfUpdateHold !== undefined) {
+            pending.selfUpdateHold = undefined;
+            await pending.save();
+        }
+        if (mark === undefined) {
+            notice = undefined;
+            return false;
+        }
+        const startedAt = await holdStarted(pending, { mark, activeRuns });
+        const holding = activeRuns > 0 || Date.now() - startedAt < 30 * 60 * 1000;
+        const line = holding
+            ? "Queue holding for pending self-update\n"
+            : "Queue continuing after pending self-update time limit\n";
+        const key = mark + line;
+        if (notice !== key)
+            process.stderr.write(line);
+        notice = key;
+        return holding;
+    };
+}
+async function admission(settings, attempt) {
+    const file = join(settings.directory, "queue-submission");
+    await writeFile(file, "", { mode: 0o600 });
+    try {
+        if (!(await attempt.holding()) && !attempt.signal.aborted)
+            await submit(settings, attempt.pending);
+    }
+    finally {
+        await rm(file, { force: true });
+    }
+}
 async function cycle(settings, signal) {
     const pending = await journal(settings.directory);
+    if (settings.resetQueue === true) {
+        pending.failures.length = 0;
+        await pending.save();
+    }
+    const holding = updateHold(settings.directory, pending);
+    await rm(join(settings.directory, "queue-submission"), { force: true });
     await recover(settings, pending);
     const stopped = () => signal.aborted;
     while (!stopped()) {
         await delay(settings.intervalMilliseconds, undefined, { signal }).catch(() => undefined);
         if (stopped())
             return;
-        if (await completed(settings, pending))
+        const pendingRuns = await completed(settings, pending);
+        const lanes = settings.lanes ?? 1;
+        if (pendingRuns >= lanes) {
+            await holding(pendingRuns);
+            continue;
+        }
+        const activeRuns = await active(settings.runtimeDirectory);
+        const held = await holding(activeRuns);
+        if (activeRuns >= lanes || activeRuns < pendingRuns)
             continue;
-        if (await active(settings.runtimeDirectory))
+        if (held)
             continue;
         if (stopped())
             return;
-        await submit(settings, pending);
+        await admission(settings, { pending, holding, signal });
     }
 }
 export function issueQueue(settings) {
     const stopping = new AbortController();
     const running = cycle(settings, stopping.signal).catch((error) => {
         const reason = error instanceof Error ? error.message : String(error);
-        process.stderr.write(`Queue stopped: ${JSON.stringify(reason)}\n`);
+        const diagnostic = error instanceof QueueFailureLimit ? reason : JSON.stringify(reason);
+        process.stderr.write(`Queue stopped: ${diagnostic}\n`);
     });
     return {
         stop: async () => {

package/dist/src/daemon/notify-journal.d.ts

@@ -0,0 +1,18 @@
+import type { Notification } from "./notify-message.ts";
+export interface DeliveryEntry {
+    readonly key: string;
+    readonly message: Notification | null;
+    readonly state: "pending" | "sent" | "failed";
+    readonly attempts: number;
+    readonly lastAttemptAt: number | null;
+    readonly lastFailure: string | null;
+}
+export interface RecordedNotifications {
+    readonly entries: Map<string, DeliveryEntry>;
+    readonly retiredKeys: Set<string>;
+}
+export declare function recordedNotifications(file: string): Promise<RecordedNotifications | undefined>;
+export declare function saveNotifications(file: string, journal: {
+    readonly entries: ReadonlyMap<string, DeliveryEntry>;
+    readonly retiredKeys: ReadonlySet<string>;
+}): Promise<void>;

package/dist/src/daemon/notify-journal.js

@@ -0,0 +1,108 @@
+import { open, readFile, rename } from "node:fs/promises";
+import { record } from "../protocol.js";
+export async function recordedNotifications(file) {
+    const text = await readFile(file, "utf8").catch((error) => {
+        if (error instanceof Error && "code" in error && error.code === "ENOENT")
+            return undefined;
+        throw error;
+    });
+    if (text === undefined)
+        return undefined;
+    const value = JSON.parse(text);
+    const journal = record(value);
+    if (journal?.["version"] === 1 && Array.isArray(journal["entries"])) {
+        const entries = journal["entries"].map(deliveryEntry);
+        const found = new Map(entries.map((entry) => [entry.key, entry]));
+        if (found.size !== entries.length)
+            throw new Error("invalid-notify-record");
+        return {
+            entries: found,
+            retiredKeys: recordedKeys(journal["retiredKeys"] ?? []),
+        };
+    }
+    const keys = recordedKeys(value);
+    return {
+        retiredKeys: new Set(),
+        entries: new Map([...keys].map((key) => [
+            key,
+            {
+                key,
+                message: null,
+                state: "sent",
+                attempts: 0,
+                lastAttemptAt: Date.now(),
+                lastFailure: null,
+            },
+        ])),
+    };
+}
+function recordedKeys(value) {
+    if (!Array.isArray(value) ||
+        !value.every((key) => typeof key === "string"))
+        throw new Error("invalid-notify-record");
+    return new Set(value.map(String));
+}
+function notification(value) {
+    const message = record(value);
+    if (message === undefined)
+        return false;
+    return (["decision", "blocked", "failed"].includes(String(message["event"])) &&
+        ["event", "runId", "workflowId", "subject", "body"].every((field) => typeof message[field] === "string"));
+}
+function deliveryState(value) {
+    return value === "pending" || value === "sent" || value === "failed";
+}
+function attemptCount(value) {
+    return (typeof value === "number" &&
+        Number.isInteger(value) &&
+        value >= 0 &&
+        value <= 5);
+}
+function attemptTime(value) {
+    return (value === null ||
+        (typeof value === "number" && Number.isSafeInteger(value) && value >= 0));
+}
+function deliveryEntry(value) {
+    const { key, message, state, attempts, lastAttemptAt, lastFailure } = record(value) ?? {};
+    if (typeof key !== "string" ||
+        !deliveryState(state) ||
+        !attemptCount(attempts))
+        throw new Error("invalid-notify-record");
+    if (!attemptTime(lastAttemptAt) ||
+        !(lastFailure === null || typeof lastFailure === "string"))
+        throw new Error("invalid-notify-record");
+    const content = { message, state };
+    if (!entryMessage(content))
+        throw new Error("invalid-notify-record");
+    return {
+        key,
+        message: content.message,
+        state,
+        attempts,
+        lastAttemptAt,
+        lastFailure,
+    };
+}
+function entryMessage(entry) {
+    return entry.message === null
+        ? entry.state === "sent"
+        : notification(entry.message);
+}
+export async function saveNotifications(file, journal) {
+    const temporary = await open(`${file}.tmp`, "w", 0o600);
+    try {
+        await temporary.chmod(0o600);
+        await temporary.writeFile(JSON.stringify({
+            version: 1,
+            entries: [...journal.entries.values()],
+            ...(journal.retiredKeys.size > 0
+                ? { retiredKeys: [...journal.retiredKeys] }
+                : {}),
+        }) + "\n");
+        await temporary.sync();
+    }
+    finally {
+        await temporary.close();
+    }
+    await rename(`${file}.tmp`, file);
+}

package/dist/src/daemon/notify-message.js

@@ -1,3 +1,4 @@
+import { operationalDecision } from "../decision-visibility.js";
 import { decisionLines, runStatus } from "../cli/render.js";
 import { record } from "../protocol.js";
 function body(text, commands) {
@@ -17,7 +18,7 @@ function message(run, content) {
     };
 }
 export function decisionNotification(run, decision) {
-    const lines = decisionLines(decision);
+    const lines = decisionLines(operationalDecision(decision, String(run["workflowId"])));
     const allowed = decision["allowed"];
     const count = Array.isArray(allowed) ? allowed.length : 0;
     const split = lines.length - count;