cli-jaw 2.17.50 → 2.17.51

package/dist/src/memory/heartbeat.js

@@ -42,16 +42,77 @@ const HEARTBEAT_SCOPE = 'default';
  *  if it were the mention. A lane nobody else submits to keeps the background turn
  *  unsteerable while the session id still puts the answer in the right history. */
 const MENTION_WATCH_SCOPE_PREFIX = 'mention-watch:';
+/** How long one tick may spend ANSWERING mentions.
+ *
+ *  A mention-watch tick is a LOOP of orchestrator turns — `maxHits` of them, each
+ *  bounded only by an idle timeout — and `heartbeatBusy` is held across all of it,
+ *  so one busy morning queues every other job in the home behind it.
+ *
+ *  Ten minutes matches the script runner's own ceiling at the `execFile` call below,
+ *  so no single tick outlasts the slowest bounded runner in this file by design. It
+ *  clocks the answering phase only: the scan can legitimately sleep for minutes on
+ *  pacing, and charging that to the answer allowance would let a slow scan end a tick
+ *  having answered nothing. */
+const MENTION_WATCH_ANSWER_BUDGET_MS = 10 * 60_000;
 import { applyOutputPolicy, loadPolicyHooksConfig } from '../core/policy-hooks.js';
 import { setRecordPending } from '../core/policy-flags.js';
 import { parseHeartbeatReport } from './heartbeat-report.js';
+import { foldRunRecord, isHeartbeatJobFailing, } from './heartbeat-run-record.js';
 import { describeHeartbeatSchedule, formatHeartbeatNow, getHeartbeatMinuteSlotKey, getHeartbeatScheduleTimeZone, matchesHeartbeatCron, normalizeHeartbeatSchedule, startHeartbeatCronLoop, validateHeartbeatCron, } from './heartbeat-schedule.js';
 const heartbeatTimers = new Map();
 const heartbeatCronSlots = new Map();
+const intervalAnchors = new Map();
 let heartbeatWatcher = null;
 let heartbeatBusy = false;
+// One job, one run. `heartbeatBusy` already stops two try-bodies from
+// interleaving, because it is set synchronously before the first await, so this
+// is not about concurrency. It is about the SAME job being executed twice in a
+// row: a tick deferred during an active PABCD cycle stays queued, a later tick
+// for that job runs directly once the cycle ends, and the finishing run then
+// drains the copy it left behind. Two identical reports, seconds apart. The
+// queue dedupe below cannot see this because the second arrival never queues.
+const inFlightJobs = new Set();
+// Bumped whenever the schedule is torn down, which includes every settings
+// reload and every PUT through the API. A run that was admitted under an older
+// generation has been superseded: its job object, destination and prompt may all
+// have been replaced on disk, so it must not send, write an anchor, or drain.
+let heartbeatGeneration = 0;
+let heartbeatAbort = null;
 const pendingJobs = [];
 const liveDestinationHolds = new Map();
+// What each job's last admitted tick did. Process-local on purpose for now: the
+// durable home is the anchor table and that schema work is its own unit. A record
+// that forgets on restart still beats today's nothing, and the API says so.
+const runRecords = new Map();
+// Jobs observed running without destination-bound authority. Set membership is
+// what makes the notice once-per-job instead of once-per-tick; a warning on every
+// five-minute tick is noise an operator learns to skip past.
+const unenforcedDestinationJobs = new Set();
+/** Environment names the heartbeat script runner must not inherit.
+ *
+ *  The caller's whole design is to hand the child ONE scoped secret, the Slack tool
+ *  grant activated further down. Spreading `process.env` underneath it handed the
+ *  same child the raw bot token whenever a channel was configured through the
+ *  environment, which this repository explicitly supports — and that makes the
+ *  scoped grant decoration.
+ *
+ *  Matched by PREFIX so a channel variable added later is excluded by default
+ *  rather than leaked until someone remembers. Deliberately slightly broad:
+ *  allowlists like SLACK_CHANNEL_IDS go too. A script that genuinely needs one can
+ *  be given it explicitly; inheriting a credential by accident is what stops. */
+const CHANNEL_SECRET_ENV_PREFIXES = ['SLACK_', 'TELEGRAM_', 'DISCORD_'];
+export function heartbeatScriptEnv(source, extra) {
+    const env = {};
+    for (const [key, value] of Object.entries(source)) {
+        if (value === undefined)
+            continue;
+        if (CHANNEL_SECRET_ENV_PREFIXES.some(prefix => key.startsWith(prefix)))
+            continue;
+        env[key] = value;
+    }
+    // Applied AFTER the filter, so the grant the script actually needs survives it.
+    return { ...env, ...extra };
+}
 function heartbeatJobKey(job) {
     return String(job.id ?? job.name ?? '');
 }
@@ -136,8 +197,97 @@ function queueHeartbeatJob(job, reason, policy) {
     });
     return true;
 }
+/** Drop any queued copy of a job that is about to run, or has just run. */
+function forgetPendingJob(jobId) {
+    for (let i = pendingJobs.length - 1; i >= 0; i -= 1) {
+        if (String(pendingJobs[i]?.job["id"] ?? "") === jobId)
+            pendingJobs.splice(i, 1);
+    }
+}
+/** The signal a live tick should observe; null until the first run arms one. */
+export function heartbeatRunSignal() {
+    return heartbeatAbort?.signal;
+}
 export function getHeartbeatRuntimeState() {
-    return pendingSnapshot();
+    return {
+        ...pendingSnapshot(),
+        // Jobs whose last ticks all failed or all refused. They keep their timers —
+        // this is visibility, not a kill switch.
+        failing: [...runRecords.entries()]
+            .filter(([, record]) => isHeartbeatJobFailing(record))
+            .map(([jobId]) => jobId),
+        // Jobs whose destination-bound Slack grant does not exist for their
+        // transport. Reported rather than implied: a Discord or Telegram heartbeat
+        // otherwise looks exactly like a guarded Slack one.
+        unenforcedDestinations: [...unenforcedDestinationJobs],
+    };
+}
+/** The last admitted tick's outcome for a job, or undefined if it has not run in
+ *  this process. */
+export function getHeartbeatRunRecord(jobId) {
+    return runRecords.get(jobId);
+}
+/** Milliseconds until the next boundary of `periodMs` measured from `anchor`.
+ *
+ *  Always inside `(0, periodMs]`. Landing exactly on a boundary returns a FULL
+ *  period rather than zero, so an arm that happens to coincide with one does not
+ *  fire immediately and then again a moment later.
+ *
+ *  A clock that stepped BACKWARDS is clamped to one period instead of returning
+ *  the raw distance to the anchor. The raw distance is unbounded: a machine whose
+ *  clock jumps back a month would arm a timer for a month, which is not a schedule
+ *  and is something `setInterval` could never express. */
+export function nextIntervalDelay(anchor, periodMs, now) {
+    const elapsed = now - anchor;
+    if (elapsed < 0)
+        return Math.min(-elapsed, periodMs);
+    const remainder = elapsed % periodMs;
+    return remainder === 0 ? periodMs : periodMs - remainder;
+}
+/** The instant a job's interval grid is measured from.
+ *
+ *  Exported because the anchor is otherwise unobservable, and "a save did not
+ *  reset the phase" is exactly the property worth asserting. */
+export function getHeartbeatIntervalAnchor(jobId) {
+    return intervalAnchors.get(jobId)?.anchor;
+}
+/** Arm an `every` job on a grid that survives a rebuild.
+ *
+ *  `setInterval` measured from the arm, and `startHeartbeat` re-arms on boot
+ *  (`server.ts`), on every `PUT /api/heartbeat`, on every `heartbeat.json` write
+ *  the watcher sees, and on a mention-watch fresh start. A home saved more often
+ *  than a job's period therefore never reached that job's first fire, and nothing
+ *  logged it because each arm looked correct on its own.
+ *
+ *  The anchor fixes that: the grid is measured from a fixed instant, so a rebuild
+ *  resumes the schedule instead of restarting the wait.
+ *
+ *  A period CHANGE re-anchors. Keeping the old origin under a new period would let
+ *  the next boundary land milliseconds away, so editing 60m to 61m could tick at
+ *  once — something the old `setInterval` could not do, and not a regression worth
+ *  trading for this. Same period keeps the grid; a different one starts a new one. */
+function scheduleIntervalJob(job, periodMs) {
+    const jobId = String(job["id"]);
+    const existing = intervalAnchors.get(jobId);
+    const anchor = existing && existing.periodMs === periodMs ? existing.anchor : Date.now();
+    intervalAnchors.set(jobId, { anchor, periodMs });
+    // Captured, not read live: `runHeartbeatJob` is async, so a teardown can land
+    // while it is awaiting. The cron loop needs no equivalent because its
+    // `runCurrent` is synchronous and its re-arm cannot be interleaved.
+    const generation = heartbeatGeneration;
+    const arm = () => {
+        const timer = setTimeout(() => {
+            if (generation !== heartbeatGeneration)
+                return;
+            // Re-arm from the same anchor BEFORE running, so a slow turn cannot
+            // push the next boundary out.
+            arm();
+            void runHeartbeatJob(job);
+        }, nextIntervalDelay(anchor, periodMs, Date.now()));
+        timer.unref?.();
+        heartbeatTimers.set(jobId, timer);
+    };
+    arm();
 }
 export function startHeartbeat() {
     stopHeartbeat();
@@ -171,19 +321,55 @@ export function startHeartbeat() {
             scheduleCronJob(job);
             continue;
         }
-        const ms = schedule.minutes * 60_000;
-        const timer = setInterval(() => runHeartbeatJob(job), ms);
-        timer.unref?.();
-        heartbeatTimers.set(job.id, timer);
+        scheduleIntervalJob(job, schedule.minutes * 60_000);
     }
+    // Per-job maps outlive their jobs otherwise. Keyed off ABSENCE FROM THE FILE
+    // rather than `enabled`, because a disabled job that is re-enabled should keep
+    // its cadence. `liveDestinationHolds` is keyed by id-or-name, so its live set
+    // uses the same derivation instead of assuming every job carries an id.
+    const liveIds = new Set(jobs.map(job => String(job?.id ?? '')).filter(Boolean));
+    const liveKeys = new Set(jobs.map(job => heartbeatJobKey(job)).filter(Boolean));
+    for (const id of [...intervalAnchors.keys()])
+        if (!liveIds.has(id))
+            intervalAnchors.delete(id);
+    for (const id of [...heartbeatCronSlots.keys()])
+        if (!liveIds.has(id))
+            heartbeatCronSlots.delete(id);
+    for (const id of [...runRecords.keys()])
+        if (!liveIds.has(id))
+            runRecords.delete(id);
+    for (const id of [...unenforcedDestinationJobs])
+        if (!liveIds.has(id))
+            unenforcedDestinationJobs.delete(id);
+    for (const key of [...liveDestinationHolds.keys()])
+        if (!liveKeys.has(key))
+            liveDestinationHolds.delete(key);
     const n = heartbeatTimers.size;
     log.info(`[heartbeat] ${n} job${n !== 1 ? 's' : ''} active`);
 }
 export function stopHeartbeat() {
+    // Tearing down timers was never enough. A tick already inside its try-body
+    // kept the job object it was admitted with and went on to send, anchor and
+    // write the mention-watch ledger, and the queue it left behind was still
+    // drained by whatever finished next, including from the external callers in
+    // orchestrator/pipeline, routes/orchestrate, cli/handlers-runtime and
+    // agent/spawn/queue. Bump the generation and abort first so an in-flight run
+    // stops at its next checkpoint, then drop the queue it would have replayed.
+    heartbeatGeneration += 1;
+    heartbeatAbort?.abort();
+    heartbeatAbort = null;
+    pendingJobs.length = 0;
     for (const timer of heartbeatTimers.values())
         clearTimeout(timer);
     heartbeatTimers.clear();
-    heartbeatCronSlots.clear();
+    // Cron slots deliberately survive. startHeartbeatCronLoop runs the current
+    // minute immediately on arm, so clearing the map here let a save plus the
+    // file watcher rebuild the schedule and fire the same minute again. Keeping
+    // the slot means the immediate tick recognises work it already did.
+    //
+    // Interval anchors survive for the same reason, one level up: they exist so a
+    // rebuild resumes a job's grid rather than restarting its wait, and clearing
+    // them here would reinstate exactly the defect they close.
 }
 export function decideHeartbeatReport(report, policy) {
     if (policy === 'silent')
@@ -204,7 +390,7 @@ export function runHeartbeatScript(command, extraEnv = {}) {
         execFile(file, args, {
             timeout: 10 * 60_000,
             maxBuffer: 64 * 1024,
-            env: { ...process.env, ...extraEnv },
+            env: heartbeatScriptEnv(process.env, extraEnv),
         }, (error, stdout, stderr) => {
             const code = error && typeof error === 'object' && 'code' in error && typeof error.code === 'number' ? error.code : error ? 1 : 0;
             resolve(parseHeartbeatReport([stdout, stderr].filter(Boolean).join('\n'), code));
@@ -299,7 +485,13 @@ async function runMentionWatchJob(job, watch) {
         token,
         selfUserId: getSlackSelfUserId(),
         allowlist: readSlackAllowlist(sc["channelIds"]),
+        answerBudgetMs: MENTION_WATCH_ANSWER_BUDGET_MS,
         log: (message) => log.info(`[heartbeat:${job["name"]}] ${message}`),
+        // The tick already knows how to stop: it checks deps.signal before each
+        // answer and reports stoppedBecause 'aborted'. Nothing ever handed it one,
+        // so a scan that started before a settings reload kept answering and kept
+        // writing receipts under the old configuration.
+        ...(heartbeatRunSignal() ? { signal: heartbeatRunSignal() } : {}),
         // Yield to anything a person is waiting on. Re-read per item because the
         // previous answer may have taken minutes.
         yieldNow: (hit) => {
@@ -326,17 +518,31 @@ async function runMentionWatchJob(job, watch) {
             // now is the session minted: doing it in the guard would create a
             // permanent, undeletable row for every thread merely looked at.
             const placement = mentionThreadPlacement(hit, 'mint');
-            const collected = await sessionLanes.runDetachedTurn(placement.scope, () => orchestrateAndCollectData(prompt, {
-                origin: 'heartbeat', requestId: crypto.randomUUID(),
-                scope: placement.scope, chatSessionId: placement.chatSessionId,
-            }));
-            const text = applyOutputPolicy(String(collected.text), { scope: 'heartbeat', channel: 'slack' }).text;
-            const quietConfig = loadPolicyHooksConfig()?.flags?.heartbeatQuietOk;
-            const extraQuietMarkers = quietConfig?.enabled ? (quietConfig.markers || []) : [];
-            if (!text.trim() || isHeartbeatQuietOutput(text, extraQuietMarkers))
-                return null;
-            answerAnchors.set(hit.channelId + '/' + hit.ts, anchor);
-            return text;
+            const requestId = crypto.randomUUID();
+            const target = slackThreadTarget(hit);
+            const release = await reserveHeartbeatDestinationGrant({ state: 'bound', target, verification: 'unverified' }, requestId, { scope: placement.scope, chatSessionId: placement.chatSessionId });
+            if (!release) {
+                log.error(`[heartbeat:${job["name"]}] refuse: slack_grant_unavailable — mention-watch answer not run`);
+                throw new Error('slack_grant_unavailable');
+            }
+            try {
+                const collected = await sessionLanes.runDetachedTurn(placement.scope, () => orchestrateAndCollectData(prompt, {
+                    origin: 'heartbeat', requestId,
+                    scope: placement.scope, chatSessionId: placement.chatSessionId,
+                    remoteKey: placement.remoteKey,
+                    target,
+                }));
+                const text = applyOutputPolicy(String(collected.text), { scope: 'heartbeat', channel: 'slack' }).text;
+                const quietConfig = loadPolicyHooksConfig()?.flags?.heartbeatQuietOk;
+                const extraQuietMarkers = quietConfig?.enabled ? (quietConfig.markers || []) : [];
+                if (!text.trim() || isHeartbeatQuietOutput(text, extraQuietMarkers))
+                    return null;
+                answerAnchors.set(hit.channelId + '/' + hit.ts, anchor);
+                return text;
+            }
+            finally {
+                release();
+            }
         },
         send: async (hit, text) => {
             const key = hit.channelId + '/' + hit.ts;
@@ -379,7 +585,17 @@ async function runMentionWatchJob(job, watch) {
         },
     });
     const stopped = outcome.stoppedBecause ? ' (stopped: ' + outcome.stoppedBecause + ')' : '';
-    log.info(`[heartbeat:${job["name"]}] mention watch: ${outcome.answered} answered, ${outcome.quiet} quiet, ${outcome.failed} failed${stopped}`);
+    // Counts alone cannot tell a caught-up watch from one still draining, which is
+    // the distinction the scanner computes and this line used to discard. Neither
+    // flag means backlog on its own: `scanIncomplete` also covers a 429 or an
+    // aborted walk, and `hitCapReached` is the case where `scanIncomplete` stays
+    // false while whole channels went unread.
+    const drain = [
+        outcome.scanIncomplete ? 'scan incomplete' : '',
+        outcome.hitCapReached ? 'hit cap reached' : '',
+    ].filter(Boolean).join(', ');
+    log.info(`[heartbeat:${job["name"]}] mention watch: ${outcome.answered} answered, ${outcome.quiet} quiet, `
+        + `${outcome.failed} failed${stopped}${drain ? ` — ${drain}` : ' — caught up'}`);
     return true;
 }
 /** Where the answer goes: the thread that carried the mention.
@@ -478,7 +694,7 @@ function buildMentionWatchPrompt(job, watch, hit) {
         job["prompt"] || '',
     ].join('\n');
 }
-async function reserveHeartbeatDestinationGrant(binding, requestId) {
+async function reserveHeartbeatDestinationGrant(binding, requestId, activation = { scope: HEARTBEAT_SCOPE, chatSessionId: 'default' }) {
     if (binding.target.channel !== 'slack')
         return () => { };
     const token = String(settings["slack"]?.botToken ?? '').trim();
@@ -491,11 +707,19 @@ async function reserveHeartbeatDestinationGrant(binding, requestId) {
         destination: binding.target,
         credentialKey: slackCredentialKey(token),
         enforceDestination: true,
-    }, { requestId, scope: HEARTBEAT_SCOPE, chatSessionId: 'default' });
+    }, { requestId, scope: activation.scope, chatSessionId: activation.chatSessionId });
     return reserved ? () => revokeSlackToolGrant(requestId) : null;
 }
 export async function runHeartbeatJob(job, deps = {}) {
     const runner = job["runner"] || 'main';
+    const jobId = String(job["id"] ?? job["name"] ?? '');
+    // Checked before every other guard. A job already executing must be skipped
+    // outright, not queued: queueing it is what produced the back-to-back double
+    // report, because the finishing run drains what the queue still holds.
+    if (jobId && inFlightJobs.has(jobId)) {
+        log.info(`[heartbeat:${job["name"]}] already running, skip`);
+        return;
+    }
     if (runner === 'main' && getState('default') !== 'IDLE') {
         const queued = queueHeartbeatJob(job, 'pabcd_active', 'defer');
         log.info(`[heartbeat:${job["name"]}] ${queued ? 'deferred' : 'already deferred'} during active PABCD (${pendingJobs.length} pending)`);
@@ -516,6 +740,21 @@ export async function runHeartbeatJob(job, deps = {}) {
         return;
     }
     heartbeatBusy = true;
+    // Admission point. From here the run owns this id, and the queue must not
+    // keep a copy of it: a deferred entry for this job was satisfied by this very
+    // tick. The generation is captured so the finally block can tell whether the
+    // schedule was torn down underneath it.
+    const generation = heartbeatGeneration;
+    if (jobId) {
+        inFlightJobs.add(jobId);
+        forgetPendingJob(jobId);
+    }
+    if (!heartbeatAbort)
+        heartbeatAbort = new AbortController();
+    const startedAt = Date.now();
+    // Deliberately `error`: a path that leaves without naming its outcome is a bug,
+    // and a record saying so is more useful than one quietly claiming success.
+    let outcome = { execution: 'error', delivery: 'not_requested', reason: 'no outcome recorded' };
     try {
         // A mention watch replaces the prompt path entirely: its prompt describes
         // how to answer a message that has not been found yet, so running it bare
@@ -525,9 +764,17 @@ export async function runHeartbeatJob(job, deps = {}) {
         if (watch != null) {
             if (!isHeartbeatMentionWatch(watch)) {
                 log.error(`[heartbeat:${job["name"]}] invalid mention watch — not run`);
+                outcome = { execution: 'skipped', delivery: 'not_requested', reason: 'invalid_mention_watch' };
                 return;
             }
-            await runMentionWatchJob(job, watch);
+            // The boolean was already returned and already discarded. It is the only
+            // thing that separates a watch that ran from one that refused for a
+            // reason of its own — a disabled Slack, a quarantined ledger, an
+            // unverifiable workspace — so it stops being thrown away here.
+            const ran = await runMentionWatchJob(job, watch);
+            outcome = ran
+                ? { execution: 'ok', delivery: 'not_requested' }
+                : { execution: 'skipped', delivery: 'not_requested', reason: 'mention_watch_not_runnable' };
             return;
         }
         // Resolve and, for a Slack thread, prove the destination BEFORE spending
@@ -541,9 +788,19 @@ export async function runHeartbeatJob(job, deps = {}) {
         if (destinationBinding.state === 'held') {
             updateHeartbeatLiveDestinationHold(job, destinationBinding.reason);
             log.error(`[heartbeat:${job["name"]}] refuse: ${destinationBinding.reason} — ${heartbeatHoldMessage(destinationBinding.reason)}`);
+            outcome = { execution: 'skipped', delivery: 'not_requested', reason: destinationBinding.reason };
             return;
         }
         updateHeartbeatLiveDestinationHold(job, null);
+        // `reserveHeartbeatDestinationGrant` returns a bare releaser for any
+        // non-Slack target, which reads at the call site exactly like a successful
+        // reservation. The public contract already says the grant is Slack's; the
+        // runtime said nothing, so a Discord tick looked guarded in the logs.
+        if (jobId && destinationBinding.target.channel !== 'slack' && !unenforcedDestinationJobs.has(jobId)) {
+            unenforcedDestinationJobs.add(jobId);
+            log.warn(`[heartbeat:${job["name"]}] ${destinationBinding.target.channel} destination runs without `
+                + `destination-bound authority — the enforceDestination grant exists only for Slack`);
+        }
         const schedule = normalizeHeartbeatSchedule(job["schedule"]);
         const timeZone = getHeartbeatScheduleTimeZone(schedule);
         const now = formatHeartbeatNow(schedule);
@@ -570,6 +827,7 @@ export async function runHeartbeatJob(job, deps = {}) {
             const guarded = await withDestinationGuard(requestId => runEmployee(job, prompt, requestId, destinationBinding.target).then(report => report.raw));
             if (!guarded.ok) {
                 log.error(`[heartbeat:${job["name"]}] refuse: slack_grant_unavailable — employee authority could not be reserved`);
+                outcome = { execution: 'skipped', delivery: 'not_requested', reason: 'slack_grant_unavailable' };
                 return;
             }
             rawResult = guarded.value;
@@ -588,6 +846,7 @@ export async function runHeartbeatJob(job, deps = {}) {
             });
             if (!guarded.ok) {
                 log.error(`[heartbeat:${job["name"]}] refuse: slack_grant_unavailable — script authority could not be reserved`);
+                outcome = { execution: 'skipped', delivery: 'not_requested', reason: 'slack_grant_unavailable' };
                 return;
             }
             const scriptReport = guarded.value;
@@ -609,6 +868,7 @@ export async function runHeartbeatJob(job, deps = {}) {
             const first = await collect();
             if (!first) {
                 log.error(`[heartbeat:${job["name"]}] refuse: slack_grant_unavailable — destination-bound Slack authority could not be reserved`);
+                outcome = { execution: 'skipped', delivery: 'not_requested', reason: 'slack_grant_unavailable' };
                 return;
             }
             const collected = first.data.agyPlannerOnly === true
@@ -616,6 +876,7 @@ export async function runHeartbeatJob(job, deps = {}) {
                 : first;
             if (!collected) {
                 log.error(`[heartbeat:${job["name"]}] refuse: slack_grant_unavailable — retry authority could not be reserved`);
+                outcome = { execution: 'skipped', delivery: 'not_requested', reason: 'slack_grant_unavailable' };
                 return;
             }
             rawResult = String(collected.text);
@@ -625,6 +886,7 @@ export async function runHeartbeatJob(job, deps = {}) {
         const extraQuietMarkers = quietConfig?.enabled ? (quietConfig.markers || []) : [];
         if (isHeartbeatQuietOutput(result, extraQuietMarkers)) {
             log.info(`[heartbeat:${job["name"]}] silent`);
+            outcome = { execution: 'ok', delivery: 'suppressed' };
             return;
         }
         const report = parseHeartbeatReport(result);
@@ -642,7 +904,11 @@ export async function runHeartbeatJob(job, deps = {}) {
         const sendResult = !decision.send
             ? { ok: true }
             : await sendChannelOutput({ channel: destinationBinding.target.channel, type: 'text', text: formatted,
-                target: destinationBinding.target, allowActiveFallback: false });
+                target: destinationBinding.target, allowActiveFallback: false })
+                // A transport that THROWS is still a delivery failure, not a failed
+                // job. Without this it lands in the catch below beside a crashed
+                // runner and counts against the execution streak.
+                .catch((error) => ({ ok: false, error: error.message }));
         if (!sendResult.ok) {
             log.error(`[heartbeat:${job["name"]}] send failed: ${sendResult.error}`);
         }
@@ -656,18 +922,48 @@ export async function runHeartbeatJob(job, deps = {}) {
                 log.error(`[heartbeat:${job["name"]}] anchor save failed:`, e.message);
             }
         }
+        // Delivery is decided here, from the decision and the result, rather than
+        // hung on a line further up. A job whose policy says stay quiet synthesizes
+        // `{ ok: true }` WITHOUT sending, so reading success off that would file a
+        // tick that posted nothing as delivered.
+        outcome = {
+            execution: 'ok',
+            delivery: !decision.send ? 'not_requested' : sendResult.ok ? 'delivered' : 'not_delivered',
+            ...(sendResult.ok ? {} : { reason: 'send_failed' }),
+        };
     }
     catch (err) {
         log.error(`[heartbeat:${job["name"]}] error:`, err.message);
+        outcome = { execution: 'error', delivery: 'not_requested', reason: err.message };
     }
     finally {
         heartbeatBusy = false;
-        await drainPending();
+        if (jobId)
+            inFlightJobs.delete(jobId);
+        if (jobId) {
+            const record = foldRunRecord(runRecords.get(jobId), {
+                jobId, startedAt, finishedAt: Date.now(),
+                superseded: generation !== heartbeatGeneration,
+                ...outcome,
+            });
+            runRecords.set(jobId, record);
+            broadcast('heartbeat_run', record);
+        }
+        // A superseded run must not hand work to the next one. Draining here after
+        // stopHeartbeat would replay jobs from a configuration the operator has
+        // already replaced, which is exactly what made stop look advisory.
+        if (generation === heartbeatGeneration)
+            await drainPending();
     }
 }
 export async function drainPending() {
     if (pendingJobs.length === 0)
         return;
+    // Reachable from orchestrator/pipeline, routes/orchestrate, cli/handlers-runtime
+    // and agent/spawn/queue, none of which know whether the schedule is still
+    // armed. Without this a stopped heartbeat kept starting jobs.
+    if (heartbeatTimers.size === 0)
+        return;
     if (isAgentBusy(HEARTBEAT_SCOPE) || messageQueue.length > 0 || hasPendingWorkerReplays(HEARTBEAT_SCOPE))
         return;
     const next = pendingJobs.shift()?.job;