crewly 1.20.227 → 1.20.229

package/dist/backend/backend/src/services/apps/app-wake.service.js

@@ -27,16 +27,74 @@
  * - A failed delivery keeps the batch (and the cursor before it) and is
  *   retried with backoff; after WAKE_FAILS_BEFORE_ORC_NOTICE failures the
  *   orchestrator is told once.
+ * - Nothing may stall the loop (2026-10-05, 科技晨报: one poll pass never
+ *   finished, the self-rescheduling loop stopped for good, and five owner
+ *   comments sat in Cloud for hours with nothing logged). Each app's poll
+ *   has a deadline (POLL_APP_DEADLINE_MS; a late result is discarded), a
+ *   pass that still has not finished after POLL_PASS_STALL_MS is abandoned
+ *   and a fresh one starts, and a delivery that has not settled after
+ *   WAKE_DELIVER_TIMEOUT_MS counts as failed and is retried (a late success
+ *   still settles the batch).
+ * - A batch being delivered is detached: changes arriving meanwhile open the
+ *   next batch instead of being marked delivered without being in the
+ *   message.
+ * - A watchdog on every scheduled pass re-arms a batch whose timer is gone
+ *   and, once per batch, logs and tells the orchestrator about owner changes
+ *   that have not reached their agent after WAKE_STUCK_NOTICE_MS.
+ * - Owner changes are delivered as owner-authored (`owner: true`, crewly#1105
+ *   queue priority): a busy agent gets them queued ahead of system traffic,
+ *   not after it goes idle.
+ * - Batches live in memory; the persisted cursor stays before the first
+ *   change of every batch not yet delivered, so after a restart the same
+ *   changes are read again and batched again.
  * - An `ask` reaches a named agent only when it is in the publisher's team
  *   and already running; it never starts a stopped agent.
+ * - @mentions (crewly-services apps/SPEC.md §12.1): every pass also reads
+ *   this instance's mention inbox (`GET /mentions`, any app of the account,
+ *   wherever it was published; bounded by POLL_APP_DEADLINE_MS like an app
+ *   poll) and wakes each mentioned agent that runs here (starting it like
+ *   the publisher; the orchestrator for "@Orc"), with the same untrusted
+ *   marking and anchor, through the same batches, delivery timeout, retries,
+ *   watchdog and owner-priority tag. The publisher still gets the comment
+ *   from the app feed, noting who was mentioned, unless it was mentioned
+ *   itself (then the mention is the one message). A mention of an agent that
+ *   is gone from this machine goes to the orchestrator. The roster Cloud
+ *   offers the owner is pushed first (`AppRosterService`, only when changed).
  *
  * @module services/apps/app-wake.service
  */
 import { CREWLY_APPS_CONSTANTS } from '../../constants.js';
 import { LoggerService } from '../core/logger.service.js';
 import { AppsCloudError } from './apps-cloud.client.js';
-import { buildAppWakeMessage, safeAppName } from './app-wake-message.js';
+import { buildAppWakeMessage, mentionsOf, safeAppName } from './app-wake-message.js';
+import { ORCHESTRATOR_SESSION_NAME } from '../../constants.js';
 const C = CREWLY_APPS_CONSTANTS;
+/** Settles like `promise`, or like `onTimeout()` (its value, or what it throws) after `ms` — whichever comes first. */
+function withDeadline(promise, ms, onTimeout) {
+    let timer = null;
+    const deadline = new Promise((resolve, reject) => {
+        timer = setTimeout(() => {
+            try {
+                resolve(onTimeout());
+            }
+            catch (err) {
+                reject(err);
+            }
+        }, ms);
+        timer.unref?.();
+    });
+    return Promise.race([promise, deadline]).finally(() => {
+        if (timer)
+            clearTimeout(timer);
+    });
+}
+/** Thrown when one app's poll misses its deadline. */
+class PollDeadlineError extends Error {
+    constructor(ms) {
+        super(`poll did not finish within ${Math.round(ms / 1000)} s`);
+        this.name = 'PollDeadlineError';
+    }
+}
 /** Recipient key for the orchestrator (never a valid session name). */
 export const ORC_RECIPIENT = '\u0000orchestrator';
 /**
@@ -49,7 +107,16 @@ export class AppWakeService {
     timer = null;
     running = false;
     ticking = null;
+    /** When the running pass started, and its generation (an abandoned pass stops at its next step) */
+    tickStartedAt = 0;
+    generation = 0;
+    passStallNotified = false;
+    /** Current poll of each app; a poll whose token was replaced or dropped discards its result */
+    pollTokens = new Map();
+    /** Batches collecting changes, by `appId\0recipient` */
     batches = new Map();
+    /** Batches handed to `deliver` and not yet confirmed (in flight, or failed and waiting to retry) */
+    outgoing = new Map();
     lastWakeAt = new Map();
     /** Highest seq read per app (the registry holds what is safe to persist) */
     fetched = new Map();
@@ -59,6 +126,13 @@ export class AppWakeService {
     /** Visitor-wake cap state per app (the registry holds the persisted copy) */
     visitorWakes = new Map();
     visitorDirty = new Set();
+    /** Highest mention-inbox seq read, and seqs above the safe cursor already delivered */
+    mentionFetched = null;
+    mentionDelivered = null;
+    mentionBackoff = null;
+    myInstance = undefined;
+    /** Current mention-inbox read; one that missed its deadline discards its result */
+    mentionToken = null;
     constructor(deps) {
         this.deps = deps;
         this.now = deps.now ?? (() => Date.now());
@@ -77,10 +151,13 @@ export class AppWakeService {
         if (this.timer)
             clearTimeout(this.timer);
         this.timer = null;
-        for (const b of this.batches.values())
+        this.generation++;
+        for (const b of [...this.batches.values(), ...this.outgoing.values()])
             if (b.timer)
                 clearTimeout(b.timer);
         this.batches.clear();
+        this.outgoing.clear();
+        this.pollTokens.clear();
     }
     /**
      * Per-app backoff state (tests).
@@ -97,38 +174,74 @@ export class AppWakeService {
             return;
         this.timer = setTimeout(() => {
             this.timer = null;
-            void this.tick().finally(() => this.schedule());
+            this.watchdog();
+            // The loop never waits on a pass for longer than the stall limit: the
+            // next scheduled pass then abandons it and starts afresh.
+            void withDeadline(this.tick(), C.POLL_PASS_STALL_MS, () => undefined)
+                .catch((err) => this.logger.warn('App change poll pass failed', { error: err instanceof Error ? err.message : String(err) }))
+                .finally(() => this.schedule());
         }, C.POLL_INTERVAL_MS);
         this.timer.unref?.();
     }
     /**
      * One pass over the apps that are due. Exposed for tests and for an immediate poll.
+     * A pass still running after POLL_PASS_STALL_MS is abandoned and a fresh one starts.
      */
     async tick() {
-        if (this.ticking)
-            return this.ticking;
-        this.ticking = this.runTick().finally(() => {
+        if (this.ticking) {
+            const runningFor = this.now() - this.tickStartedAt;
+            if (runningFor < C.POLL_PASS_STALL_MS)
+                return this.ticking;
+            this.logger.warn('App change poll pass did not finish; abandoning it and starting a fresh one', { runningForMs: runningFor });
+            if (!this.passStallNotified) {
+                this.passStallNotified = true;
+                this.notifyOrc(`[APP CHANGES] Crewly's poll of app changes stalled for ${Math.round(runningFor / 60_000)} min and was restarted. ` +
+                    'Owner changes made in that time are being read again now. If this repeats, the backend may need a restart.');
+            }
+            this.generation++;
             this.ticking = null;
+        }
+        const gen = this.generation;
+        this.tickStartedAt = this.now();
+        const pass = this.runTick(gen).finally(() => {
+            if (this.ticking === pass)
+                this.ticking = null;
         });
-        return this.ticking;
+        this.ticking = pass;
+        await pass;
+        if (gen === this.generation)
+            this.passStallNotified = false;
     }
-    async runTick() {
+    async runTick(gen) {
         if (!this.deps.client.isAvailable())
             return;
+        if (this.deps.roster)
+            await withDeadline(this.deps.roster.pushIfChanged().catch(() => false), C.POLL_APP_DEADLINE_MS, () => false);
         const now = this.now();
-        const due = (await this.deps.registry.list()).filter((e) => !e.deleted && (this.backoff.get(e.appId)?.nextAt ?? 0) <= now);
+        const all = await withDeadline(this.deps.registry.list(), C.POLL_APP_DEADLINE_MS, () => null);
+        if (!all) {
+            this.logger.warn('Reading the apps registry did not finish; skipping this poll pass');
+            return;
+        }
+        const due = all.filter((e) => !e.deleted && (this.backoff.get(e.appId)?.nextAt ?? 0) <= now);
         let next = 0;
         const worker = async () => {
-            while (next < due.length) {
+            while (gen === this.generation && next < due.length) {
                 const app = due[next++];
                 await this.pollOne(app);
             }
         };
         await Promise.all(Array.from({ length: Math.min(C.POLL_CONCURRENCY, due.length) }, worker));
+        if (gen === this.generation)
+            await this.pollMentionsSafely(gen);
     }
     async pollOne(app) {
+        const token = Symbol(app.appId);
+        this.pollTokens.set(app.appId, token);
         try {
-            await this.pollApp(app);
+            await withDeadline(this.pollApp(app, token), C.POLL_APP_DEADLINE_MS, () => {
+                throw new PollDeadlineError(C.POLL_APP_DEADLINE_MS);
+            });
             this.backoff.delete(app.appId);
         }
         catch (err) {
@@ -143,6 +256,14 @@ export class AppWakeService {
             this.backoff.set(app.appId, { failures, nextAt: this.now() + wait });
             this.logger.warn('Polling app changes failed', { appId: app.appId, failures, retryInMs: wait, error: err instanceof Error ? err.message : String(err) });
         }
+        finally {
+            // A poll that missed its deadline may still finish later: its token is gone, so it changes nothing.
+            if (this.pollTokens.get(app.appId) === token)
+                this.pollTokens.delete(app.appId);
+        }
+    }
+    isCurrentPoll(appId, token) {
+        return this.pollTokens.get(appId) === token;
     }
     deliveredSet(app) {
         let set = this.delivered.get(app.appId);
@@ -152,11 +273,13 @@ export class AppWakeService {
         }
         return set;
     }
-    async pollApp(app) {
+    async pollApp(app, token) {
         const opts = { timeoutMs: C.POLL_REQUEST_TIMEOUT_MS };
         if (app.cursor === null || app.cursor === undefined) {
             // Never polled: start from now, never replay history.
             const head = await this.deps.client.request('GET', `/apps/${app.appId}/changes`, opts);
+            if (!this.isCurrentPoll(app.appId, token))
+                return;
             this.fetched.set(app.appId, head.seq);
             await this.deps.registry.setProgress(app.appId, head.seq, []);
             return;
@@ -166,6 +289,9 @@ export class AppWakeService {
         let since = Math.max(app.cursor, this.fetched.get(app.appId) ?? app.cursor);
         for (let page = 0; page < C.POLL_MAX_PAGES; page++) {
             const res = await this.deps.client.request('GET', `/apps/${app.appId}/changes`, { ...opts, query: { since, wait: 0 } });
+            // Missed its deadline (or a newer poll of this app started): discard.
+            if (!this.isCurrentPoll(app.appId, token))
+                return;
             const changes = Array.isArray(res.changes) ? res.changes : [];
             for (const change of changes) {
                 if (typeof change.seq === 'number' && change.seq > since && !done.has(change.seq))
@@ -188,7 +314,7 @@ export class AppWakeService {
         if (fetched === undefined)
             return;
         let safe = fetched;
-        for (const b of this.batches.values()) {
+        for (const b of [...this.batches.values(), ...this.outgoing.values()]) {
             if (b.appId === appId)
                 safe = Math.min(safe, b.firstSeq - 1);
         }
@@ -243,7 +369,7 @@ export class AppWakeService {
             return;
         const session = app.agentSession ?? null;
         const key = `${app.appId}\u0000${session ?? ORC_RECIPIENT}`;
-        if (this.batches.has(key))
+        if (this.batches.has(key) || this.outgoing.has(key))
             return;
         const batch = this.newBatch(app.appId, session, false, Number.POSITIVE_INFINITY);
         batch.skippedNotice = true;
@@ -270,8 +396,167 @@ export class AppWakeService {
             failures: 0,
             orcNotified: false,
             skippedNotice: false,
+            openedAt: this.now(),
+            inFlight: false,
+            reportSkipped: null,
+            stuckNotified: false,
+            mentioned: false,
+            mentionSeqs: [],
+            mentionFirstSeq: Number.POSITIVE_INFINITY,
+            goneMentions: [],
         };
     }
+    // -------------------------------------------------------------------------
+    // @mention inbox
+    // -------------------------------------------------------------------------
+    async instance() {
+        if (this.myInstance === undefined || this.myInstance === null) {
+            this.myInstance = this.deps.instanceId ? await this.deps.instanceId().catch(() => null) : null;
+        }
+        return this.myInstance;
+    }
+    /**
+     * Whether a comment change @mentions `session` on this instance (an agent
+     * with the same session on another machine is someone else).
+     */
+    async mentionsHere(change, session) {
+        const list = mentionsOf(change);
+        if (list.length === 0)
+            return false;
+        const mine = await this.instance();
+        return list.some((m) => m.session === session && (!m.instanceId || !mine || m.instanceId === mine));
+    }
+    async pollMentionsSafely(gen) {
+        if (this.mentionBackoff && this.mentionBackoff.nextAt > this.now())
+            return;
+        try {
+            // Bounded like an app poll: a hung inbox read never holds the pass.
+            const token = Symbol('mentions');
+            this.mentionToken = token;
+            await withDeadline(this.pollMentions(gen, token), C.POLL_APP_DEADLINE_MS, () => {
+                if (this.mentionToken === token)
+                    this.mentionToken = null;
+                throw new PollDeadlineError(C.POLL_APP_DEADLINE_MS);
+            });
+            this.mentionBackoff = null;
+        }
+        catch (err) {
+            const failures = (this.mentionBackoff?.failures ?? 0) + 1;
+            // An older Cloud without the inbox answers 404: check back rarely.
+            const wait = err instanceof AppsCloudError && err.status === 404 ? C.POLL_MAX_BACKOFF_MS : Math.min(C.POLL_INTERVAL_MS * 2 ** failures, C.POLL_MAX_BACKOFF_MS);
+            this.mentionBackoff = { failures, nextAt: this.now() + wait };
+            if (!(err instanceof AppsCloudError && err.status === 404)) {
+                this.logger.warn('Reading the app mention inbox failed', { failures, retryInMs: wait, error: err instanceof Error ? err.message : String(err) });
+            }
+        }
+    }
+    /**
+     * Mention-inbox backoff state (tests).
+     *
+     * @returns Failures in a row and when it is next due, or null when healthy
+     */
+    mentionsBackoff() {
+        return this.mentionBackoff ? { ...this.mentionBackoff } : null;
+    }
+    async pollMentions(gen, token) {
+        const stale = () => gen !== this.generation || this.mentionToken !== token;
+        const opts = { timeoutMs: C.POLL_REQUEST_TIMEOUT_MS };
+        const progress = await this.deps.registry.getMentionProgress();
+        if (!this.mentionDelivered)
+            this.mentionDelivered = new Set(progress.delivered);
+        if (progress.cursor === null && this.mentionFetched === null) {
+            // Never read: start from now, never replay history.
+            const head = await this.deps.client.request('GET', '/mentions', opts);
+            if (stale())
+                return;
+            const seq = typeof head.seq === 'number' ? head.seq : 0;
+            this.mentionFetched = seq;
+            await this.deps.registry.setMentionProgress(seq, []);
+            return;
+        }
+        const done = this.mentionDelivered;
+        let since = Math.max(progress.cursor ?? 0, this.mentionFetched ?? 0);
+        for (let page = 0; page < C.MENTIONS.MAX_PAGES; page++) {
+            const res = await this.deps.client.request('GET', '/mentions', { ...opts, query: { since } });
+            // Missed its deadline, or the pass was abandoned: discard.
+            if (stale())
+                return;
+            const items = Array.isArray(res.mentions) ? res.mentions : [];
+            for (const item of items) {
+                if (typeof item?.seq === 'number' && item.seq > since && !done.has(item.seq))
+                    await this.handleMention(item);
+            }
+            since = Math.max(since, typeof res.seq === 'number' ? res.seq : since);
+            this.mentionFetched = since;
+            if (items.length < C.MENTIONS.PAGE)
+                break;
+        }
+        await this.persistMentionProgress();
+    }
+    async persistMentionProgress() {
+        if (this.mentionFetched === null)
+            return;
+        let safe = this.mentionFetched;
+        for (const b of [...this.batches.values(), ...this.outgoing.values()])
+            safe = Math.min(safe, b.mentionFirstSeq - 1);
+        const set = this.mentionDelivered ?? new Set();
+        for (const seq of set)
+            if (seq <= safe)
+                set.delete(seq);
+        while (set.size > C.MAX_DELIVERED_SEQS)
+            set.delete(Math.min(...set));
+        this.mentionDelivered = set;
+        await this.deps.registry.setMentionProgress(safe, [...set]);
+    }
+    async handleMention(item) {
+        const done = this.mentionDelivered ?? new Set();
+        this.mentionDelivered = done;
+        // The comment or the app is gone: nothing to say.
+        if (!item.thread || typeof item.appId !== 'string' || typeof item.commentId !== 'string') {
+            done.add(item.seq);
+            return;
+        }
+        const isOrc = item.session === ORCHESTRATOR_SESSION_NAME;
+        const here = isOrc || (this.deps.isLocalAgent ? await this.deps.isLocalAgent(item.session).catch(() => false) : true);
+        const session = isOrc || !here ? null : item.session;
+        const recipient = session ?? ORC_RECIPIENT;
+        const key = `${item.appId}\u0000${recipient}`;
+        const change = {
+            seq: item.seq,
+            kind: 'comment',
+            comment: { id: item.commentId, op: item.op, ...(item.replyId ? { replyId: item.replyId } : {}), thread: item.thread },
+            actor: { kind: 'owner' },
+            ...(item.at ? { at: item.at } : {}),
+        };
+        let batch = this.batches.get(key);
+        if (!batch) {
+            const app = await this.deps.registry.get(item.appId);
+            const last = this.lastWakeAt.get(key) ?? app?.wakes?.[recipient] ?? -Infinity;
+            const delay = Math.max(C.BATCH_WINDOW_MS, last + C.COOLDOWN_MS - this.now());
+            batch = this.newBatch(item.appId, session, session !== null, Number.POSITIVE_INFINITY);
+            this.batches.set(key, batch);
+            this.arm(key, batch, delay);
+        }
+        else if (session !== null) {
+            batch.activate = true;
+        }
+        batch.mentioned = true;
+        if (typeof item.appName === 'string' && item.appName)
+            batch.appName = item.appName;
+        if (!here && !isOrc) {
+            const name = typeof item.name === 'string' && item.name ? item.name : item.session;
+            if (!batch.goneMentions.includes(name))
+                batch.goneMentions.push(name);
+        }
+        batch.mentionSeqs.push(item.seq);
+        if (batch.mentionSeqs.length > C.MAX_DELIVERED_SEQS)
+            batch.mentionSeqs.shift();
+        batch.mentionFirstSeq = Math.min(batch.mentionFirstSeq, item.seq);
+        batch.commentsTotal++;
+        batch.comments.push(change);
+        if (batch.comments.length > C.COMMENTS.MAX_PER_WAKE)
+            batch.comments.shift();
+    }
     async recipientFor(app, change) {
         const publisher = app.agentSession ?? null;
         const named = change.kind === 'event' && change.event?.type === 'ask' ? change.event.agent : undefined;
@@ -318,6 +603,9 @@ export class AppWakeService {
         }
         const recipientInfo = await this.recipientFor(app, change);
         const session = recipientInfo.session;
+        // The recipient was @mentioned in this comment: the mention inbox delivers it (one message, not two).
+        if (change.kind === 'comment' && (await this.mentionsHere(change, session ?? ORCHESTRATOR_SESSION_NAME)))
+            return;
         // A visitor never starts a stopped agent; only the owner's changes may.
         const activate = visitor ? false : recipientInfo.activate;
         const recipient = session ?? ORC_RECIPIENT;
@@ -332,8 +620,8 @@ export class AppWakeService {
         }
         else {
             batch.activate = batch.activate || activate;
-            if (batch.skippedNotice)
-                batch.firstSeq = Math.min(batch.firstSeq, change.seq);
+            // A batch opened by a skipped-visitor notice or a mention has no app seq yet.
+            batch.firstSeq = Math.min(batch.firstSeq, change.seq);
         }
         batch.seqs.push(change.seq);
         if (batch.seqs.length > C.MAX_DELIVERED_SEQS)
@@ -366,12 +654,44 @@ export class AppWakeService {
     arm(key, batch, delay) {
         if (batch.timer)
             clearTimeout(batch.timer);
-        batch.timer = setTimeout(() => void this.flush(key), delay);
+        batch.timer = setTimeout(() => {
+            batch.timer = null;
+            if (this.outgoing.get(key) === batch)
+                void this.retry(key);
+            else
+                void this.flush(key);
+        }, Math.max(0, delay));
         batch.timer.unref?.();
     }
+    /** Whether a batch holds only visitor submissions (or only a skipped-submissions notice). */
+    static visitorOnly(batch) {
+        return batch.dataTotal === 0 && batch.eventsTotal === 0 && batch.commentsTotal === 0;
+    }
+    /** Fold `from` (newer) into `into` (older): one message covers both. */
+    static merge(into, from) {
+        const keep = (list, max) => (list.length > max ? list.slice(list.length - max) : list);
+        into.activate = into.activate || from.activate;
+        into.dataChanges = keep([...into.dataChanges, ...from.dataChanges], C.MAX_BATCH_DATA_CHANGES);
+        into.events = keep([...into.events, ...from.events], C.MAX_EVENTS_PER_WAKE);
+        into.visitorChanges = keep([...into.visitorChanges, ...from.visitorChanges], C.MAX_BATCH_DATA_CHANGES);
+        into.comments = keep([...into.comments, ...from.comments], C.COMMENTS.MAX_PER_WAKE);
+        into.dataTotal += from.dataTotal;
+        into.eventsTotal += from.eventsTotal;
+        into.visitorTotal += from.visitorTotal;
+        into.commentsTotal += from.commentsTotal;
+        into.seqs = keep([...into.seqs, ...from.seqs], C.MAX_DELIVERED_SEQS);
+        into.firstSeq = Math.min(into.firstSeq, from.firstSeq);
+        into.skippedNotice = into.skippedNotice && from.skippedNotice;
+        into.openedAt = Math.min(into.openedAt, from.openedAt);
+        into.mentioned = into.mentioned || from.mentioned;
+        into.mentionSeqs = keep([...into.mentionSeqs, ...from.mentionSeqs], C.MAX_DELIVERED_SEQS);
+        into.mentionFirstSeq = Math.min(into.mentionFirstSeq, from.mentionFirstSeq);
+        into.appName = into.appName ?? from.appName;
+        into.goneMentions = [...new Set([...into.goneMentions, ...from.goneMentions])];
+    }
     /**
-     * Send one batch now (timer callback; tests). On failure the batch stays
-     * pending and is retried with backoff.
+     * Send one collecting batch now (timer callback; tests). On failure the
+     * batch stays pending and is retried with backoff.
      *
      * @param key - Batch key
      * @returns Whether a message was delivered
@@ -383,29 +703,79 @@ export class AppWakeService {
         if (batch.timer)
             clearTimeout(batch.timer);
         batch.timer = null;
-        const app = await this.deps.registry.get(batch.appId);
-        const recipient = batch.session ?? ORC_RECIPIENT;
+        // The previous batch for this agent is still out: fold this one into it
+        // when it is waiting to retry (its retry carries both), else check again
+        // shortly.
+        const out = this.outgoing.get(key);
+        if (out) {
+            if (out.inFlight) {
+                this.arm(key, batch, C.WAKE_INFLIGHT_RECHECK_MS);
+                return false;
+            }
+            this.batches.delete(key);
+            AppWakeService.merge(out, batch);
+            if (!out.timer)
+                this.arm(key, out, 0);
+            return false;
+        }
         // Visitor submissions alone never start the agent: wait until it runs.
-        const visitorOnly = batch.dataTotal === 0 && batch.eventsTotal === 0 && batch.commentsTotal === 0;
-        if (visitorOnly && batch.session !== null && this.deps.isRunning && !this.deps.isRunning(batch.session)) {
+        if (AppWakeService.visitorOnly(batch) && batch.session !== null && this.deps.isRunning && !this.deps.isRunning(batch.session)) {
             if (this.batches.get(key) === batch)
                 this.arm(key, batch, C.VISITOR_WAKE.PENDING_RECHECK_MS);
             return false;
         }
+        // Detach: changes arriving while this one is delivered open the next batch.
+        this.batches.delete(key);
+        this.outgoing.set(key, batch);
+        return this.send(key, batch);
+    }
+    /** Retry an outgoing batch (its retry timer), folding in whatever collected meanwhile. */
+    async retry(key) {
+        const batch = this.outgoing.get(key);
+        if (!batch || batch.inFlight)
+            return false;
+        if (batch.timer)
+            clearTimeout(batch.timer);
+        batch.timer = null;
+        const newer = this.batches.get(key);
+        if (newer) {
+            if (newer.timer)
+                clearTimeout(newer.timer);
+            this.batches.delete(key);
+            AppWakeService.merge(batch, newer);
+        }
+        return this.send(key, batch);
+    }
+    /**
+     * Deliver an outgoing batch once, bounded by WAKE_DELIVER_TIMEOUT_MS.
+     *
+     * @param key - Batch key
+     * @param batch - The batch (already in `outgoing`)
+     * @returns Whether it was delivered
+     */
+    async send(key, batch) {
+        batch.inFlight = true;
+        const app = await this.deps.registry.get(batch.appId).catch(() => null);
         // Skipped visitor submissions are reported in the next message to the
         // agent visitors wake (the publisher, else the orchestrator).
-        const isVisitorRecipient = (app?.agentSession ?? null) === batch.session;
+        const isVisitorRecipient = !!app && (app.agentSession ?? null) === batch.session;
         const vstate = app && isVisitorRecipient ? this.visitorState(app) : null;
-        const skipped = vstate?.skipped ?? 0;
-        if (visitorOnly && batch.visitorTotal === 0 && skipped === 0) {
+        if (batch.reportSkipped === null)
+            batch.reportSkipped = vstate?.skipped ?? 0;
+        const skipped = batch.reportSkipped;
+        if (AppWakeService.visitorOnly(batch) && batch.visitorTotal === 0 && skipped === 0) {
             // A notice-only batch with nothing left to report.
-            this.batches.delete(key);
+            batch.inFlight = false;
+            if (this.outgoing.get(key) === batch)
+                this.outgoing.delete(key);
             return false;
         }
         const text = buildAppWakeMessage({
             appId: batch.appId,
-            appName: app?.name ?? batch.appId,
-            isPublisher: (app?.agentSession ?? null) === batch.session,
+            appName: app?.name ?? batch.appName ?? batch.appId,
+            isPublisher: !!app && !app.deleted && (app.agentSession ?? null) === batch.session,
+            mentioned: batch.mentioned,
+            goneMentions: batch.goneMentions,
             dataChanges: batch.dataChanges,
             events: batch.events,
             dataTotal: batch.dataTotal,
@@ -417,32 +787,68 @@ export class AppWakeService {
             visitorSkipped: skipped,
             skillsPath: this.deps.skillsPath,
         });
-        let ok = false;
-        try {
-            ok = await this.deps.deliver(batch.session, text, { activate: batch.activate });
-        }
-        catch (err) {
+        const owner = !AppWakeService.visitorOnly(batch);
+        const seqs = batch.seqs.filter((n) => Number.isFinite(n));
+        const mseqs = batch.mentionSeqs.filter((n) => Number.isFinite(n));
+        const refParts = [
+            ...(seqs.length > 0 ? [`${Math.min(...seqs)}-${Math.max(...seqs)}:${seqs.length}`] : []),
+            ...(mseqs.length > 0 ? [`m${Math.min(...mseqs)}-${Math.max(...mseqs)}:${mseqs.length}`] : []),
+        ];
+        const ref = refParts.length > 0 ? `app:${batch.appId}:${refParts.join(':')}` : undefined;
+        const sentSeqs = new Set(batch.seqs);
+        const sentMentions = new Set(batch.mentionSeqs);
+        const attempt = Promise.resolve()
+            .then(() => this.deps.deliver(batch.session, text, { activate: batch.activate, ...(owner ? { owner: true } : {}), ...(ref ? { ref } : {}) }))
+            .catch((err) => {
             this.logger.warn('App change wake failed', { appId: batch.appId, session: batch.session, error: err instanceof Error ? err.message : String(err) });
-        }
-        if (!ok) {
-            batch.failures++;
-            const retry = Math.min(C.WAKE_RETRY_BASE_MS * 2 ** (batch.failures - 1), C.WAKE_RETRY_MAX_MS);
-            this.logger.warn('App change wake was not delivered; will retry', { appId: batch.appId, session: batch.session ?? 'orchestrator', failures: batch.failures, retryInMs: retry });
-            if (batch.failures >= C.WAKE_FAILS_BEFORE_ORC_NOTICE && !batch.orcNotified && batch.session !== null) {
-                batch.orcNotified = true;
-                const notice = `[APP CHANGES] Changes in app "${safeAppName(app?.name ?? batch.appId)}" (${batch.appId}) could not be delivered to ` +
-                    `${batch.session} after ${batch.failures} tries. Crewly keeps retrying. Check whether that agent is stuck or signed out.`;
-                await this.deps.deliver(null, notice, { activate: false }).catch(() => false);
-            }
-            if (this.batches.get(key) === batch)
-                this.arm(key, batch, retry);
             return false;
+        });
+        const outcome = await withDeadline(attempt.then((ok) => (ok ? 'ok' : 'failed')), C.WAKE_DELIVER_TIMEOUT_MS, () => 'timeout');
+        batch.inFlight = false;
+        if (this.outgoing.get(key) !== batch)
+            return false; // stopped meanwhile
+        if (outcome === 'ok')
+            return this.succeeded(key, batch, app, vstate, skipped);
+        if (outcome === 'timeout') {
+            this.logger.warn('App change wake did not finish in time; will retry', {
+                appId: batch.appId,
+                session: batch.session ?? 'orchestrator',
+                timeoutMs: C.WAKE_DELIVER_TIMEOUT_MS,
+            });
+            // A late success still settles the batch, unless a retry is already out
+            // or the batch has grown since (then the retry carries everything).
+            void attempt.then((ok) => {
+                if (!ok || this.outgoing.get(key) !== batch || batch.inFlight)
+                    return;
+                if (!batch.seqs.every((n) => sentSeqs.has(n)) || !batch.mentionSeqs.every((n) => sentMentions.has(n)))
+                    return;
+                this.logger.info('A timed-out app change wake arrived after all', { appId: batch.appId, session: batch.session ?? 'orchestrator' });
+                if (batch.timer)
+                    clearTimeout(batch.timer);
+                batch.timer = null;
+                void this.succeeded(key, batch, app, vstate, skipped);
+            });
         }
-        this.batches.delete(key);
+        batch.failures++;
+        const retry = Math.min(C.WAKE_RETRY_BASE_MS * 2 ** (batch.failures - 1), C.WAKE_RETRY_MAX_MS);
+        this.logger.warn('App change wake was not delivered; will retry', { appId: batch.appId, session: batch.session ?? 'orchestrator', failures: batch.failures, retryInMs: retry });
+        if (batch.failures >= C.WAKE_FAILS_BEFORE_ORC_NOTICE && !batch.orcNotified && batch.session !== null) {
+            batch.orcNotified = true;
+            this.notifyOrc(`[APP CHANGES] Changes in app "${safeAppName(app?.name ?? batch.appId)}" (${batch.appId}) could not be delivered to ` +
+                `${batch.session} after ${batch.failures} tries. Crewly keeps retrying. Check whether that agent is stuck or signed out.`);
+        }
+        this.arm(key, batch, retry);
+        return false;
+    }
+    async succeeded(key, batch, app, vstate, skipped) {
+        if (this.outgoing.get(key) !== batch)
+            return false;
+        this.outgoing.delete(key);
+        const recipient = batch.session ?? ORC_RECIPIENT;
         const at = this.now();
         this.lastWakeAt.set(key, at);
         await this.deps.registry.setLastWake(batch.appId, recipient, at).catch(() => undefined);
-        if (vstate) {
+        if (vstate && app) {
             if (batch.visitorTotal > 0 || batch.skippedNotice)
                 vstate.count++;
             vstate.skipped = Math.max(0, vstate.skipped - skipped);
@@ -453,6 +859,12 @@ export class AppWakeService {
         for (const seq of batch.seqs)
             set.add(seq);
         this.delivered.set(batch.appId, set);
+        if (batch.mentionSeqs.length > 0) {
+            const mset = this.mentionDelivered ?? new Set();
+            for (const seq of batch.mentionSeqs)
+                mset.add(seq);
+            this.mentionDelivered = mset;
+        }
         this.logger.info('Woke agent for app changes', {
             appId: batch.appId,
             session: batch.session ?? 'orchestrator',
@@ -460,14 +872,66 @@ export class AppWakeService {
             events: batch.eventsTotal,
             comments: batch.commentsTotal,
             visitorSubmissions: batch.visitorTotal,
+            ...(batch.mentioned ? { mentioned: true } : {}),
             ...(skipped > 0 ? { visitorSkippedReported: skipped } : {}),
+            ...(batch.failures > 0 ? { afterFailures: batch.failures } : {}),
         });
+        // Changes that arrived during the delivery wait for the cooldown it started.
+        const next = this.batches.get(key);
+        if (next)
+            this.arm(key, next, C.COOLDOWN_MS);
         await this.persistProgress(batch.appId).catch(() => undefined);
+        if (batch.mentionSeqs.length > 0)
+            await this.persistMentionProgress().catch(() => undefined);
         return true;
     }
+    /** Tell the orchestrator something, without waiting on it for long. */
+    notifyOrc(text) {
+        void withDeadline(Promise.resolve()
+            .then(() => this.deps.deliver(null, text, { activate: false }))
+            .catch(() => false), C.WAKE_DELIVER_TIMEOUT_MS, () => false);
+    }
+    /**
+     * Runs before every scheduled pass: a batch with no timer and no delivery
+     * running is re-armed (it would otherwise wait forever), and owner changes
+     * that have not reached their agent after WAKE_STUCK_NOTICE_MS are logged
+     * and reported to the orchestrator, once per batch. Visitor-only batches
+     * waiting for their agent to run are expected to wait and are left alone.
+     */
+    watchdog() {
+        const now = this.now();
+        for (const [key, batch] of [...this.batches.entries(), ...this.outgoing.entries()]) {
+            if (!batch.timer && !batch.inFlight) {
+                this.logger.warn('App change batch had no timer; re-armed', { appId: batch.appId, session: batch.session ?? 'orchestrator' });
+                this.arm(key, batch, 0);
+            }
+            if (AppWakeService.visitorOnly(batch) || batch.stuckNotified)
+                continue;
+            const ageMs = now - batch.openedAt;
+            if (ageMs < C.WAKE_STUCK_NOTICE_MS)
+                continue;
+            batch.stuckNotified = true;
+            const minutes = Math.round(ageMs / 60_000);
+            this.logger.warn('Owner changes in an app have not reached their agent', {
+                appId: batch.appId,
+                session: batch.session ?? 'orchestrator',
+                minutes,
+                inFlight: batch.inFlight,
+                failures: batch.failures,
+                comments: batch.commentsTotal,
+                dataChanges: batch.dataTotal,
+                events: batch.eventsTotal,
+            });
+            if (batch.session !== null) {
+                this.notifyOrc(`[APP CHANGES] The owner's changes in an app (${batch.appId}: ${batch.commentsTotal} comment(s), ${batch.dataTotal} data change(s), ` +
+                    `${batch.eventsTotal} message(s)) have waited ${minutes} min to reach ${batch.session}. Crewly keeps trying. ` +
+                    'Check whether that agent is stuck.');
+            }
+        }
+    }
     /** @returns Keys of batches waiting to be sent (tests) */
     pendingKeys() {
-        return [...this.batches.keys()];
+        return [...new Set([...this.batches.keys(), ...this.outgoing.keys()])];
     }
 }
 //# sourceMappingURL=app-wake.service.js.map