@indigoai-us/hq-cli 5.135.0 → 5.136.0

package/dist/lib/bot/run.js

@@ -13,6 +13,14 @@
  * - first start only: the intro DM, then (when bot.json has `kickoff`) ONE
  *   model turn on that prompt as if the owner sent it, answered in the DM, so
  *   a guided bot starts working without waiting for the owner to type
+ * - job items (channel "job"): a run the cloud fired on an EventBridge
+ *   schedule. Nothing is scheduled on this computer. The run is one model turn
+ *   in its own session (one per jobId), reported in the owner's DM with a
+ *   "scheduled job" header through the same split/recovery path as every other
+ *   reply, and reported back to the cloud EXACTLY ONCE — nothing else clears
+ *   the job's in-progress flag. A run whose staleAfter has passed still runs,
+ *   once, labelled a catch-up; older pending runs of the same job are reported
+ *   skipped-precondition instead of piling up.
  * - delivered room items (channels / group chats) are eligible for replies; claimed
  *   first (first-claim-wins), one session per room, reply posted in the room
  * - conversations run side by side: up to MAX_PARALLEL_TURNS turns at once,
@@ -57,6 +65,8 @@ import { FALLBACK_CLAUDE_PERMISSION_MODE, isClaudeBypassDisabled } from "./runti
 import { runRuntimeTurn, RuntimeError } from "./runtime/index.js";
 import { dmSessionScope } from "./paths.js";
 import { isDmItem, resolveReplyTarget } from "./room-policy.js";
+import { buildJobTurnPrompt, isCatchUpRun, isJobItem, jobReportHeader, jobRunOutput, machineTimeZone, planJobRuns, stripJobHeader, SUPERSEDED_DISPOSITION } from "./job-run.js";
+import { clearPendingJobCompletion, isCompletionDue, isCompletionExpired, isCompletionForOtherAgent, readPendingJobCompletions, recordPendingJobCompletion } from "./job-completions.js";
 import { isRuntimeSignInFailure, SIGN_IN_RETRY_MS, signInNeededText } from "./runtime-sign-in.js";
 import { HQ_BOT_AGENT_UID_ENV } from "../../commands/onboard-bot-membership.js";
 import { captureHeartbeatIdentityMismatch, isHeartbeatIdentityMismatch, } from "../heartbeat-identity.js";
@@ -111,6 +121,18 @@ export const MAX_PARALLEL_TURNS = 2;
 export const RECOVERY_ATTEMPT_LIMIT = 5;
 /** What the person sees when a reply could not be delivered at all. */
 export const UNDELIVERABLE_REPLY_TEXT = "I wrote a reply but couldn't deliver it. Ask me again and I'll keep it shorter.";
+/**
+ * The same, for a scheduled run. Nobody asked, so "ask me again" is wrong: the
+ * run DID finish, and its outcome is still recorded with the job in HQ.
+ */
+export const UNDELIVERABLE_JOB_REPORT_TEXT = "This scheduled run finished, but I couldn't deliver its report here. Its result is saved with the job's run history.";
+/**
+ * Scheduled work must never crowd out the people the bot talks to. A job turn
+ * can run for a long time, so at most ONE runs at a time, and a job never takes
+ * the last free slot: a DM or a room message can always start immediately
+ * instead of waiting, with no explanation, behind two long jobs.
+ */
+export const MAX_PARALLEL_JOB_TURNS = 1;
 export const KICKOFF_MESSAGE_ID = "kickoff";
 /**
  * The environment a model turn's `hq` commands run in.
@@ -158,13 +180,17 @@ export const RESTARTED_REASON = "I was restarted while I was working on it";
 /**
  * The lane an inbox item runs in: the model session its turn resumes. Owner
  * DMs retain the legacy DM session; other senders and rooms each have their
- * own. An unsupported item gets a lane of its own.
+ * own, and so does each scheduled job (two different jobs must never share a
+ * model session, and a job must never pollute the owner's DM conversation).
+ * An item that never reaches a model turn gets a lane of its own.
  */
 export function inboxLaneKey(item, config) {
     if (isDmItem(item))
         return dmSessionScope(item.fromPersonUid ?? "", config.ownerUid);
     const target = resolveReplyTarget(item, config);
-    return target?.kind === "room" ? target.sessionScope : `item:${item.messageId}`;
+    if (target?.kind === "room" || target?.kind === "job")
+        return target.sessionScope;
+    return `item:${item.messageId}`;
 }
 class FailureWindow {
     windowMs;
@@ -247,6 +273,9 @@ export async function runBot(deps) {
     const turnCwd = config.hqRoot;
     // HQ's setup bot is a guided conversation: no "working on it" filler between steps.
     const isSetupBot = config.workerId === SETUP_WORKER_ID;
+    // Scheduled runs are reported in the owner's own zone: "every weekday at 9am"
+    // was set up here, so "due 13:00 UTC" would read as the wrong time.
+    const timeZone = machineTimeZone();
     // Personal: every turn acts as the owner. Company: every turn acts as the bot.
     const kind = effectiveBotKind(config);
     const companies = effectiveBotCompanies(config);
@@ -506,7 +535,12 @@ export async function runBot(deps) {
         // kickoff). Room prompts must keep their [#channel …] header first, and a
         // room turn can be triggered by other people, who must not be handed the
         // owner's companies. A company bot acts as itself and never gets it.
-        if (deps.ownerContext && sessionScope === "dm" && kind === "personal") {
+        // Owner context also goes on the owner's scheduled jobs: `turnEnv` strips
+        // the machine credentials on EVERY personal-bot turn, so a job turn runs
+        // `hq` as the owner too and needs the same block saying whose account it
+        // is on. Rooms stay excluded (see above).
+        const actsPurelyForOwner = sessionScope === "dm" || sessionScope.startsWith("job:");
+        if (deps.ownerContext && actsPurelyForOwner && kind === "personal") {
             try {
                 const owner = await deps.ownerContext();
                 if (owner.status === "unavailable") {
@@ -692,6 +726,83 @@ export async function runBot(deps) {
         };
         return { send, saved };
     };
+    // ── Scheduled jobs ────────────────────────────────────────────────────────
+    /** Newest pending run key already picked up, per jobId (one catch-up per wake). */
+    const handledJobFires = new Map();
+    /** Runs an older fire of a job that a newer pending run supersedes. */
+    const supersededJobRuns = new Set();
+    /**
+     * Report one fired run as finished. This is the ONLY thing that clears the
+     * job's in-progress flag in the cloud, so a failure is logged loudly and the
+     * completion is kept for the next poll — the caller still acks, because the
+     * work is done and must never run twice.
+     */
+    const completeJobRun = async (jobId, messageId, completion) => {
+        try {
+            await api.completeJobRun(config.agentUid, jobId, messageId, completion);
+            clearPendingJobCompletion(dir, messageId);
+            log("info", `reported run ${messageId} of job ${jobId} as ${completion.status}`);
+            return true;
+        }
+        catch (err) {
+            const reason = err instanceof Error ? err.message : String(err);
+            const pending = recordPendingJobCompletion(dir, { agentUid: config.agentUid, jobId, messageId, status: completion.status, ...(completion.output !== undefined ? { output: completion.output } : {}), ...(completion.disposition !== undefined ? { disposition: completion.disposition } : {}) }, now());
+            log("error", `COULD NOT report run ${messageId} of job ${jobId} as ${completion.status}: ${reason}. ` +
+                `Until this is reported the cloud keeps the job marked in progress and skips its next fires. ` +
+                `Retry ${pending.attempts} in ${Math.round((pending.nextAttemptAt - now().getTime()) / 1000)}s.`);
+            return false;
+        }
+    };
+    /**
+     * Re-send completions the cloud has not accepted yet. Never re-runs work.
+     * Backed off per entry and given up on by age, so a minute-long network drop
+     * recovers instead of burning a fixed attempt budget in twenty seconds.
+     */
+    const retryPendingJobCompletions = async () => {
+        const nowMs = now().getTime();
+        for (const pending of readPendingJobCompletions(dir)) {
+            // A folder that outlived a deleted, same-named bot: never report another
+            // identity's run as this one's (see job-completions.ts, "After a reinstall").
+            if (isCompletionForOtherAgent(pending, config.agentUid)) {
+                clearPendingJobCompletion(dir, pending.messageId);
+                log("warn", `dropped an unreported run ${pending.messageId} of job ${pending.jobId}: it belongs to ${pending.agentUid}, ` +
+                    `not this bot (${config.agentUid}) — left behind by an earlier bot with the same name`);
+                continue;
+            }
+            if (isCompletionExpired(pending, nowMs)) {
+                clearPendingJobCompletion(dir, pending.messageId);
+                log("error", `giving up on reporting run ${pending.messageId} of job ${pending.jobId} after ${pending.attempts} attempts ` +
+                    `over an hour; the cloud has released the job's in-progress slot by itself by now`);
+                continue;
+            }
+            if (!isCompletionDue(pending, nowMs))
+                continue;
+            await completeJobRun(pending.jobId, pending.messageId, {
+                status: pending.status,
+                ...(pending.output !== undefined ? { output: pending.output } : {}),
+                ...(pending.disposition !== undefined ? { disposition: pending.disposition } : {}),
+            });
+        }
+    };
+    /**
+     * Kick the completion retries WITHOUT blocking the poll. Awaiting them in the
+     * inbox loop meant one hanging completion POST stalled DM delivery behind it.
+     * One pass at a time; the tracked promise is drained on shutdown.
+     */
+    let completionRetryTask = Promise.resolve();
+    let completionRetryRunning = false;
+    const kickCompletionRetries = () => {
+        if (completionRetryRunning)
+            return;
+        completionRetryRunning = true;
+        completionRetryTask = retryPendingJobCompletions()
+            .catch((err) => {
+            log("warn", `retrying job completions failed: ${err instanceof Error ? err.message : String(err)}`);
+        })
+            .finally(() => {
+            completionRetryRunning = false;
+        });
+    };
     /** Close a marker out: clear the thinking reaction, ack the message, forget it. */
     const closeMarker = async (marker, isKickoff) => {
         if (marker.target.kind === "dm" && marker.reactionEventId) {
@@ -707,6 +818,51 @@ export async function runBot(deps) {
         }
         clearInflight(dir, marker.messageId);
     };
+    /**
+     * A scheduled run's marker is about to be closed: report its REAL outcome to
+     * the cloud first, and record that it was reported, BEFORE the ack.
+     *
+     * - A saved reply means the work finished and only delivery failed: that is
+     *   `succeeded`, with the report as output — including when delivery was
+     *   given up on as impossible. The owner not getting the DM does not undo
+     *   the work, and not reporting would jam the job for an hour.
+     * - No saved reply means the turn was cut off: `failed`.
+     * - A marker that says it was already reported is never reported again: the
+     *   server's complete is last-write-wins, so a second report after a crash
+     *   between reporting and acking would overwrite the recorded outcome.
+     */
+    const reportRecoveredJobRun = async (marker) => {
+        if (!marker.jobId || marker.jobCompleted)
+            return;
+        const finished = typeof marker.reply === "string";
+        // A run that failed or was stopped carries its real reason; only a marker
+        // with none was genuinely cut off by a restart.
+        const failure = marker.jobFailure;
+        const completion = finished
+            ? { status: "succeeded", output: marker.jobOutput ?? jobRunOutput(stripJobHeader(marker.reply, marker.jobHeader)) }
+            : failure
+                ? {
+                    status: "failed",
+                    disposition: failure.reason.slice(0, 256),
+                    ...(failure.excerpt ? { output: jobRunOutput(failure.excerpt) } : {}),
+                }
+                : { status: "failed", disposition: "the run was cut off when the bot restarted" };
+        await completeJobRun(marker.jobId, marker.messageId, completion);
+        patchInflight(dir, marker.messageId, { jobCompleted: completion.status });
+    };
+    /**
+     * The body a recovery posts. A scheduled run's FIRST message carries its
+     * header, so a report delivered by this path still reads as a scheduled run
+     * and not as an unsolicited DM. A body saved in parts already has the header
+     * baked in (the job's own send put it there), so it is never added twice.
+     */
+    const recoveryBody = (marker) => {
+        // A job that failed on its own says why; "I was restarted" only when it was.
+        const reason = marker.jobFailure?.reason ?? RESTARTED_REASON;
+        const body = marker.reply ?? didNotFinishText(reason, (marker.progressPosted ?? 0) > 0);
+        const headerAlreadyOut = (marker.progressPosted ?? 0) > 0 || body.startsWith(marker.jobHeader ?? "\u0000");
+        return marker.jobHeader && !headerAlreadyOut ? `${marker.jobHeader}${body}` : body;
+    };
     /**
      * A turn that was cut off by a restart (or whose answer never got posted):
      * tell the person instead of running it again. Returns false when the post
@@ -720,11 +876,24 @@ export async function runBot(deps) {
      */
     const recoverInflight = async (marker) => {
         const isKickoff = marker.messageId === KICKOFF_MESSAGE_ID;
+        // A scheduled run left in this folder by a DIFFERENT bot (adopt into a
+        // folder that survived without its bot.json). Posting its report would put
+        // another bot's work in this bot's DM, and completing it would be a write
+        // under the wrong identity — the thing job-completions.ts rules out. Drop
+        // it: no post, no completion, no ack (the item is not in this inbox). The
+        // cloud releases that job's slot by itself within the hour. A marker with
+        // no recorded agent predates the field and is recovered as before.
+        if (marker.jobId && marker.agentUid && marker.agentUid !== config.agentUid) {
+            clearInflight(dir, marker.messageId);
+            log("warn", `dropped an unfinished scheduled run ${marker.messageId} of job ${marker.jobId}: it belongs to ${marker.agentUid}, ` +
+                `not this bot (${config.agentUid}) — left behind by an earlier bot in this folder; not posted, not reported`);
+            return true;
+        }
         if (!isKickoff && isProcessed(readInboxState(dir), marker.messageId)) {
             clearInflight(dir, marker.messageId);
             return true;
         }
-        const parts = marker.replyParts ?? splitBody(marker.reply ?? didNotFinishText(RESTARTED_REASON, (marker.progressPosted ?? 0) > 0));
+        const parts = marker.replyParts ?? splitBody(recoveryBody(marker));
         const from = marker.replyParts ? Math.min(Math.max(0, marker.replyPartsDelivered ?? 0), parts.length) : 0;
         try {
             await sendParts(marker.target, parts, from, marker.rootEventId, (count) => patchInflight(dir, marker.messageId, { replyParts: parts, replyPartsDelivered: count }));
@@ -734,7 +903,18 @@ export async function runBot(deps) {
             const attempt = (marker.recoveryFailures ?? 0) + 1;
             if (attempt >= RECOVERY_ATTEMPT_LIMIT) {
                 log("warn", `gave up delivering the reply to ${marker.messageId} after ${attempt} attempts (${reason}); told the owner instead`);
-                await postOne(marker.target, UNDELIVERABLE_REPLY_TEXT, marker.rootEventId).catch(() => undefined);
+                if (marker.jobId) {
+                    // Nobody asked, so "ask me again" is wrong. The header goes on only if
+                    // no part of this run reached the chat yet.
+                    const headerOut = (marker.progressPosted ?? 0) > 0 || (err instanceof PartialPostError ? err.delivered : from) > 0;
+                    const text = marker.jobHeader && !headerOut ? `${marker.jobHeader}${UNDELIVERABLE_JOB_REPORT_TEXT}` : UNDELIVERABLE_JOB_REPORT_TEXT;
+                    await postOne(marker.target, text, marker.rootEventId).catch(() => undefined);
+                    // Giving up on DELIVERY is not giving up on the RUN: report it.
+                    await reportRecoveredJobRun(marker);
+                }
+                else {
+                    await postOne(marker.target, UNDELIVERABLE_REPLY_TEXT, marker.rootEventId).catch(() => undefined);
+                }
                 await closeMarker(marker, isKickoff);
                 return true;
             }
@@ -743,8 +923,11 @@ export async function runBot(deps) {
             log("warn", `could not tell the owner about unfinished ${marker.messageId} (attempt ${attempt}/${RECOVERY_ATTEMPT_LIMIT}): ${reason}`);
             return false;
         }
+        await reportRecoveredJobRun(marker);
         await closeMarker(marker, isKickoff);
-        log("info", marker.reply ? `posted the saved reply for ${marker.messageId}` : `told the owner ${marker.messageId} did not finish (restart); not re-run`);
+        log("info", marker.reply
+            ? `posted the saved reply for ${marker.messageId}`
+            : `told the owner ${marker.messageId} did not finish (${marker.jobFailure ? marker.jobFailure.reason : "restart"}); not re-run`);
         return true;
     };
     /**
@@ -822,6 +1005,186 @@ export async function runBot(deps) {
             await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
         }
     };
+    /**
+     * One fired job: a model turn on the job's own prompt, in the job's own
+     * session, reported in the owner's DM under a "scheduled job" header, and
+     * reported to the cloud exactly once.
+     */
+    const handleJobItem = async (item, target) => {
+        const id = item.messageId;
+        const { job, jobId } = target;
+        const peerUid = target.peerUid;
+        // An older pending run of this job: a newer one runs instead. Reported so
+        // the cloud releases the slot and the history says what happened.
+        if (supersededJobRuns.has(id)) {
+            await completeJobRun(jobId, id, { status: "skipped-precondition", disposition: SUPERSEDED_DISPOSITION });
+            markProcessed(dir, id, now().toISOString());
+            await api.ackInbox(config.agentUid, id);
+            supersededJobRuns.delete(id);
+            log("info", `skipped older run ${id} of job ${jobId}: a newer pending run of the same job runs instead`);
+            return;
+        }
+        const catchUp = isCatchUpRun(job, now());
+        const attempt = (attempts.get(id) ?? 0) + 1;
+        attempts.set(id, attempt);
+        const tReceived = now().getTime();
+        log("info", `received scheduled run ${id} of job ${jobId}${catchUp ? " (catch-up)" : ""}`);
+        // The header goes on the first message of the turn, so the owner sees at a
+        // glance that this is a scheduled run and not a reply to something they
+        // said. It rides on the marker too, so a report posted by the recovery path
+        // after a restart carries it as well.
+        //
+        // Every post goes through the SAME split/track path as any other reply
+        // (trackingSend → splitBody), so a long report is delivered in parts rather
+        // than refused whole. The header is only counted as delivered once a part
+        // actually reached the chat, and `saved` records the body EXACTLY as it was
+        // attempted (header included), so the parts the recovery path resends line
+        // up with the parts already delivered and nothing is repeated or dropped.
+        const header = jobReportHeader(job, catchUp, timeZone);
+        const markerTarget = { kind: "dm", peerUid };
+        const tracking = trackingSend(markerTarget);
+        let headerOut = false;
+        // What actually reached the chat, kept apart from "was anything posted".
+        // `poster.postedCount` counts progress posts too, so a turn that said
+        // "Checking the dashboards…" and then failed to post its REPORT would look
+        // delivered — and for a scheduled run the report is the only thing the
+        // owner sees; nobody will ask again.
+        const deliveredBodies = [];
+        let anySendFailed = false;
+        /** The body of the LAST send, when that send failed (the resume point). */
+        let lastFailedBody = null;
+        const withHeader = (body) => (headerOut ? body : `${header}${body}`);
+        const send = async (body) => {
+            const full = withHeader(body);
+            try {
+                await tracking.send(full);
+                headerOut = true;
+                deliveredBodies.push(body);
+                lastFailedBody = null;
+            }
+            catch (err) {
+                anySendFailed = true;
+                lastFailedBody = full;
+                if (err instanceof PartialPostError && err.delivered > 0)
+                    headerOut = true;
+                throw err;
+            }
+        };
+        const normalize = (text) => text.replace(/\s+/g, " ").trim();
+        /**
+         * True only when the final report itself reached the chat in full. With no
+         * failed post at all, the poster delivered everything it chose to (it skips
+         * a final that repeats a progress post, so "no failures" is the reliable
+         * signal). After any failure, the report must be inside a fully delivered
+         * post. Erring towards resending: a duplicate beats a report that vanished.
+         */
+        const reportDelivered = (reply) => poster.postedCount > 0 && (!anySendFailed || deliveredBodies.some((b) => normalize(b).includes(normalize(reply))));
+        /**
+         * The report, saved for resending. When the LAST post was the report and
+         * part of it already got through, resume that exact body at the parts
+         * delivered, so nothing is repeated. Otherwise resend the report whole,
+         * under its own header: the resend may come much later (after a restart)
+         * and must still read as a scheduled run.
+         */
+        const savedReport = (reply) => {
+            const jobOutput = jobRunOutput(reply);
+            const failed = lastFailedBody;
+            if (failed !== null && failed.trimEnd().endsWith(reply)) {
+                const resumed = tracking.saved(failed);
+                if ((resumed.replyPartsDelivered ?? 0) > 0)
+                    return { ...resumed, jobOutput };
+            }
+            return { ...tracking.saved(`${header}${reply}`), jobOutput };
+        };
+        /** Remember that this run's outcome is reported, before the ack. */
+        const markReported = (status) => {
+            patchInflight(dir, id, { jobCompleted: status });
+        };
+        const { outcome, poster } = await runTurnWithProgress({
+            marker: { messageId: id, target: markerTarget, jobId, jobHeader: header, agentUid: config.agentUid },
+            prompt: buildJobTurnPrompt(job, { now: now(), timeZone }),
+            sessionScope: target.sessionScope,
+            tReceived,
+            send,
+            // A scheduled run has nobody waiting in a chat: no "working on it" filler.
+            workingNotice: false,
+        });
+        if (outcome.ok) {
+            if (!reportDelivered(outcome.reply)) {
+                // The work finished but the REPORT did not fully reach the DM — even if
+                // a progress post did. Keep it (and the run un-reported and un-acked)
+                // so the next poll delivers it, resuming at the parts already read,
+                // before the run is completed. Never re-run.
+                patchInflight(dir, id, savedReport(outcome.reply));
+                log("warn", `report for run ${id} of job ${jobId} was not fully delivered; it is resent on the next poll`);
+                return;
+            }
+            await completeJobRun(jobId, id, { status: "succeeded", output: jobRunOutput(outcome.reply) });
+            markReported("succeeded");
+            markProcessed(dir, id, now().toISOString());
+            await api.ackInbox(config.agentUid, id);
+            clearInflight(dir, id);
+            attempts.delete(id);
+            setStatus({ repliesSent: (readStatus()?.repliesSent ?? 0) + 1, lastActivityAt: now().toISOString() });
+            log("info", `job ${jobId} run ${id} done (${outcome.reply.length} chars${catchUp ? ", catch-up" : ""})`);
+            return;
+        }
+        if (!outcome.aborted && poster.postedCount === 0 && isRuntimeSignInFailure(outcome.excerpt)) {
+            // Nothing ran: hold the run until the coding tool can sign in again. It
+            // is neither acked nor reported succeeded — the work is still owed.
+            clearInflight(dir, id);
+            attempts.delete(id);
+            markSignInBroken(outcome.excerpt);
+            await tellSignInNeeded(`job:${jobId}`, (body) => sendToTarget(markerTarget, body));
+            log("warn", `kept scheduled run ${id} of job ${jobId} until ${runtime.id} can sign in again`);
+            return; // un-acked, un-reported: run once the sign-in works
+        }
+        if (!outcome.started) {
+            // The model never ran, so nothing happened outside this Mac: retry.
+            clearInflight(dir, id);
+            const { excerpt } = recordTurnFailure(outcome.error, target.sessionScope);
+            if (failures.exceeded(now().getTime())) {
+                await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${excerpt}`);
+                return;
+            }
+            if (attempt >= MAX_ATTEMPTS_PER_MESSAGE) {
+                await send(`I couldn't run this one — ${runtime.id} kept failing: ${excerpt.slice(0, 160)}.`).catch(() => undefined);
+                await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(excerpt), disposition: `${runtime.id} could not be started` });
+                markReported("failed");
+                markProcessed(dir, id, now().toISOString());
+                await api.ackInbox(config.agentUid, id);
+                attempts.delete(id);
+                return;
+            }
+            await sleep(fullJitterDelayMs(failures.count(now().getTime()), { random: deps.random }));
+            return; // un-acked; retried on the next poll
+        }
+        // The model started: it may already have done things, so never re-run.
+        if (!outcome.aborted)
+            recordTurnFailure(outcome.error, target.sessionScope);
+        const reason = failureReason(outcome);
+        // Saved BEFORE the post: if "did not finish" cannot be posted, recovery
+        // reports and says this real reason, not "the bot was restarted".
+        patchInflight(dir, id, { jobFailure: { reason, ...(outcome.excerpt ? { excerpt: outcome.excerpt } : {}) } });
+        try {
+            await send(didNotFinishText(reason, poster.postedCount > 0));
+        }
+        catch (err) {
+            // Leave it un-acked with the marker: the next poll reports it.
+            log("warn", `could not post "did not finish" for run ${id} of job ${jobId}: ${err instanceof Error ? err.message : String(err)}`);
+            return;
+        }
+        await completeJobRun(jobId, id, { status: "failed", output: jobRunOutput(outcome.excerpt), disposition: reason.slice(0, 256) });
+        markReported("failed");
+        markProcessed(dir, id, now().toISOString());
+        await api.ackInbox(config.agentUid, id);
+        clearInflight(dir, id);
+        attempts.delete(id);
+        log("error", `scheduled run ${id} of job ${jobId} ${outcome.aborted ? "stopped by the owner" : "failed"} after it started; not re-run`);
+        if (!outcome.aborted && failures.exceeded(now().getTime())) {
+            await finish("failed", 0, `${FAILURE_LIMIT} model failures in 10 minutes; last: ${outcome.excerpt}`);
+        }
+    };
     const handleItem = async (item) => {
         const id = item.messageId;
         const inboxState = readInboxState(dir);
@@ -841,9 +1204,26 @@ export async function runBot(deps) {
         }
         const target = resolveReplyTarget(item, config);
         if (!target) {
+            // A job item the bot cannot run. When it names a job, the cloud is still
+            // holding that job's in-progress slot: acking alone would jam every later
+            // fire of it for an hour, and the next fire would be just as unusable —
+            // the same jam, forever. Report it so the slot is released.
+            const unusableJobId = isJobItem(item) && typeof item.jobId === "string" ? item.jobId.trim() : "";
+            if (unusableJobId) {
+                await completeJobRun(unusableJobId, id, {
+                    status: "skipped-precondition",
+                    disposition: "the fired run carried no prompt, so there was nothing to run",
+                });
+            }
             markProcessed(dir, id, now().toISOString());
             await api.ackInbox(config.agentUid, id);
-            log("info", `ignored inbox item ${id} (${item.channel || "dm"}${item.channelScope ? ` ${item.channelScope}` : ""}, not addressed to me)`);
+            log("info", isJobItem(item)
+                ? `ignored job item ${id}: it carries no ${unusableJobId ? "prompt" : "job id"}`
+                : `ignored inbox item ${id} (${item.channel || "dm"}${item.channelScope ? ` ${item.channelScope}` : ""}, not addressed to me)`);
+            return;
+        }
+        if (target.kind === "job") {
+            await handleJobItem(item, target);
             return;
         }
         const text = (item.text ?? "").trim();
@@ -909,6 +1289,7 @@ export async function runBot(deps) {
                 : {}),
         });
         if (outcome.ok) {
+            // Known gap (#614, not changed here): a progress post counts, so a final answer that fails after one is not resent.
             if (poster.postedCount === 0) {
                 // The work finished but nothing reached the chat: keep the answer —
                 // in the parts it will be posted as, and how many of them already got
@@ -1001,6 +1382,10 @@ export async function runBot(deps) {
     const pending = new Set();
     const running = new Set();
     const canStart = () => !stopping && state === "running" && !hasPromotionHold(dir);
+    /** Job turns yield to people: one at a time, and never the last free slot. */
+    const isJobLane = (lane) => lane.startsWith("job:");
+    const jobLaneBudget = Math.max(1, maxParallelTurns - 1);
+    const canStartJobLane = () => [...busyLanes].filter(isJobLane).length < MAX_PARALLEL_JOB_TURNS && busyLanes.size < jobLaneBudget;
     const pump = () => {
         for (const [lane, queue] of lanes) {
             if (busyLanes.size >= maxParallelTurns || !canStart())
@@ -1010,6 +1395,10 @@ export async function runBot(deps) {
                 return;
             if (busyLanes.has(lane))
                 continue;
+            // `continue`, never `return`: a job that cannot start yet must not hold
+            // up the owner's DM sitting behind it in the map.
+            if (isJobLane(lane) && !canStartJobLane())
+                continue;
             const item = queue.shift();
             if (queue.length === 0)
                 lanes.delete(lane);
@@ -1067,10 +1456,17 @@ export async function runBot(deps) {
                     log("error", `kickoff retry crashed: ${err instanceof Error ? err.message : String(err)}`);
                 });
             }
+            // Completions the cloud has not accepted yet: re-send the report (never
+            // the work), alongside the poll rather than in front of it.
+            kickCompletionRetries();
             try {
                 const items = await api.pullInbox(config.agentUid);
                 inboxFails = 0;
                 setStatus({ lastInboxPollAt: now().toISOString(), lastInboxPollOk: true, lastInboxError: undefined });
+                // One catch-up per job per wake: when several runs of the same job are
+                // pending, the newest runs and the older ones are reported skipped.
+                for (const id of planJobRuns(items, handledJobFires).superseded)
+                    supersededJobRuns.add(id);
                 for (const item of items)
                     enqueue(item);
                 pump();
@@ -1093,6 +1489,7 @@ export async function runBot(deps) {
         // messages were never started or acked, so the inbox delivers them again.
         while (running.size > 0)
             await Promise.allSettled([...running]);
+        await completionRetryTask;
     })();
     function readStatus() {
         return readBotStatus(dir);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@indigoai-us/hq-cli",
-  "version": "5.135.0",
+  "version": "5.136.0",
   "description": "HQ by Indigo management CLI \u2014 modules and cloud sync",
   "main": "dist/index.js",
   "bin": {