@songsid/agend 2.1.7-beta.4 → 2.1.7-beta.6

package/dist/fleet-manager.js

@@ -286,6 +286,10 @@ export function deriveSpawnConcurrency(freeMemMB, cores) {
     const byMemory = Math.floor(freeMemMB / 300);
     return Math.max(2, Math.min(10, byMemory, Math.max(1, cores)));
 }
+/** #926: how often reply obligations are checked, the owner-reminder grace, and the default overdue window. */
+const REPLY_OBLIGATION_SWEEP_MS = 30_000;
+const REPLY_REMINDER_GRACE_MS = 60_000;
+const DEFAULT_REPLY_OVERDUE_MINUTES = 15;
 export class FleetManager {
     dataDir;
     static signalTarget = null;
@@ -303,6 +307,8 @@ export class FleetManager {
     deliveryPumpScheduled = false;
     deliveryPumpRunning = false;
     deliveryPumpTimer = null;
+    /** #926: periodic reply-obligation sweep (owner reminders, requester overdue notices). */
+    replyObligationTimer = null;
     activeDurableTargets = new Set();
     /** Live view of lifecycle.daemons — used throughout; not deprecated. */
     get daemons() { return this.lifecycle.daemons; }
@@ -580,9 +586,6 @@ export class FleetManager {
         if (!this.deliveryOutboxRecovered) {
             const recovered = outbox.recoverForBoot(this.managerBootId);
             this.logger.info({ ...recovered }, "Recovered durable delivery outbox for this process boot");
-            if (recovered.uncertain > 0) {
-                this.notifyFleetError(`${recovered.uncertain} durable delivery outcome(s) became uncertain during process restart; inspect the sender's delivery notice before retrying.`);
-            }
             this.deliveryOutboxRecovered = true;
         }
         outbox.on("admitted", () => this.scheduleDeliveryOutboxPump());
@@ -598,10 +601,18 @@ export class FleetManager {
             }
         });
         outbox.on("generation_recovered", () => this.scheduleDeliveryOutboxPump());
+        this.replyObligationTimer = setInterval(() => this.sweepReplyObligations(), REPLY_OBLIGATION_SWEEP_MS);
+        this.replyObligationTimer.unref?.();
+        // #856: the text the target daemon received differs from what was
+        // admitted. Transport has never been seen to do this; a warning and the
+        // attempt row's flag are the whole response until it has.
+        outbox.on("content_digest_mismatch", (event) => {
+            this.logger.warn(event, "Delivered text differs from the admitted text (content digest mismatch)");
+        });
         outbox.on("expired", (event) => {
             this.scheduleDeliveryOutboxPump();
             if (event.count)
-                this.notifyFleetError(`${event.count} durable delivery row(s) expired before their target became available; sender outcome notice queued.`);
+                this.notifyFleetError(`${event.count} durable delivery row(s) reached the 24-hour outbox bound; ${event.uncertain ?? 0} unresolved submissions are uncertain and sender notices were queued.`);
         });
         this.scheduleDeliveryOutboxPump();
     }
@@ -632,6 +643,10 @@ export class FleetManager {
             duplicate: !admitted.inserted,
         };
     }
+    queryDurableDeliveryStatus(callerInstance, selector) {
+        this.ensureDeliveryOutbox();
+        return this.deliveryOutbox.queryStatusForInstance(callerInstance, selector);
+    }
     getDaemonBootId(instanceName) {
         return this.daemons.get(instanceName)?.bootId;
     }
@@ -651,8 +666,38 @@ export class FleetManager {
                 this.logger.error({ err, source: name, operationId: prior.operationId }, "Could not persist post-restart delivery outcome notice");
             }
         }
+        // #926: a restart kills the turn that was working on a request, and a
+        // resumed CLI does not pick it back up. Remind the owner once per restart.
+        try {
+            outbox.remindReplyObligations(name, { reason: "restart", graceMs: 0, since: new Date() });
+        }
+        catch (err) {
+            this.logger.error({ err, owner: name }, "Could not persist the post-restart reply reminder");
+        }
         this.scheduleDeliveryOutboxPump();
     }
+    /**
+     * #926: the two reply-obligation safety nets. Idle owners with an unanswered
+     * request get one reminder (after the grace); requesters get one overdue
+     * notice when the owner is not working and nothing came back in time.
+     */
+    sweepReplyObligations(now = new Date()) {
+        const outbox = this.deliveryOutbox;
+        if (!outbox || this.shuttingDown)
+            return;
+        const stateOf = (name) => this.daemons.get(name)?.getInstanceState();
+        try {
+            for (const name of this.daemons.keys()) {
+                if (stateOf(name) === "idle")
+                    outbox.remindReplyObligations(name, { now, graceMs: REPLY_REMINDER_GRACE_MS, reason: "turn-ended" });
+            }
+            const minutes = this.fleetConfig?.defaults?.reply_overdue_minutes ?? DEFAULT_REPLY_OVERDUE_MINUTES;
+            outbox.notifyOverdueReplyObligations({ now, overdueMs: minutes * 60_000, ownerIdle: name => stateOf(name) !== "working" });
+        }
+        catch (err) {
+            this.logger.error({ err }, "Reply-obligation sweep failed");
+        }
+    }
     scheduleDeliveryOutboxPump(delayMs = 0) {
         if (!this.deliveryOutbox || this.shuttingDown)
             return;
@@ -716,26 +761,42 @@ export class FleetManager {
         const target = claimed.targetInstance;
         const attempt = claimed.attemptNo;
         const payload = { ...claimed.payload };
+        const laneReleased = this.waitForDurableLaneRelease(claimed);
         const rawMeta = payload.meta && typeof payload.meta === "object"
             ? payload.meta
             : {};
-        payload.meta = {
-            ...rawMeta,
-            delivery_id: claimed.deliveryId,
-            delivery_attempt: String(claimed.attemptNo),
-            from_instance: typeof rawMeta.from_instance === "string" && rawMeta.from_instance
-                ? rawMeta.from_instance
-                : claimed.sourceInstance,
-            correlation_id: claimed.correlationId ?? String(rawMeta.correlation_id ?? ""),
-        };
         // Arm the visible liveness alert as soon as this generation owns the lane,
         // including time spent waiting for a handoff/idle gate to return.
-        const laneReleased = this.waitForDurableLaneRelease(claimed);
         try {
-            const sent = await this.deliverToInstance(target, payload, {
-                isCrossInstance: true,
-                waitForIdle: claimed.kind !== "steer",
-            });
+            let sent;
+            if (claimed.kind === "raw_paste") {
+                if (payload.type !== "raw_paste" || typeof payload.content !== "string") {
+                    throw new Error("Durable raw_paste row has an invalid payload");
+                }
+                // Preserve the schedule's exact bytes and raw route. The target daemon
+                // owns readiness, paste, one Enter, and the fenced outbox transition.
+                sent = await this.deliverToInstance(target, {
+                    type: "raw_paste",
+                    content: payload.content,
+                    delivery_id: claimed.deliveryId,
+                    delivery_attempt: String(claimed.attemptNo),
+                }, { waitForIdle: false });
+            }
+            else {
+                payload.meta = {
+                    ...rawMeta,
+                    delivery_id: claimed.deliveryId,
+                    delivery_attempt: String(claimed.attemptNo),
+                    from_instance: typeof rawMeta.from_instance === "string" && rawMeta.from_instance
+                        ? rawMeta.from_instance
+                        : claimed.sourceInstance,
+                    correlation_id: claimed.correlationId ?? String(rawMeta.correlation_id ?? ""),
+                };
+                sent = await this.deliverToInstance(target, payload, {
+                    isCrossInstance: true,
+                    waitForIdle: claimed.kind !== "steer",
+                });
+            }
             if (!sent) {
                 const current = outbox.get(claimed.deliveryId);
                 if (current?.state === "delivering") {
@@ -3043,7 +3104,7 @@ export class FleetManager {
                 ...DEFAULT_SCHEDULER_CONFIG,
                 ...this.fleetConfig?.defaults.scheduler,
             };
-            this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule) => this.handleScheduleTrigger(schedule), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
+            this.scheduler = new Scheduler(join(this.dataDir, "scheduler.db"), (schedule, runId) => this.handleScheduleTrigger(schedule, runId), schedulerConfig, (name) => this.fleetConfig?.instances?.[name] != null || !!this.classicChannels?.getAll().some(ch => ch.instanceName === name));
             this.scheduler.init();
             this.logger.info("Scheduler initialized");
             // Tips share the daily-report clock but remain an independent internal
@@ -5600,7 +5661,7 @@ export class FleetManager {
         }
     }
     // ===================== Scheduler =====================
-    async handleScheduleTrigger(schedule) {
+    async handleScheduleTrigger(schedule, stableRunId = randomUUID()) {
         const { target, reply_chat_id, reply_thread_id, message, label, id, source, silent } = schedule;
         const RATE_LIMIT_DEFER_THRESHOLD = 85;
         const rl = this.statuslineWatcher.getRateLimits(target);
@@ -5618,20 +5679,32 @@ export class FleetManager {
         }
         // Silent mode: paste directly to tmux pane — no channel message.
         if (silent) {
-            const ipc = this.instanceIpcClients.get(target);
-            if (ipc) {
-                ipc.send({
+            // The outbox commit is the schedule's durable acceptance point. The run
+            // key is the scheduled fire instant (or an invocation UUID for a manual
+            // trigger), so restart/catch-up reuses the same raw paste row.
+            this.ensureDeliveryOutbox();
+            const runKey = encodeURIComponent(stableRunId);
+            const sourceKey = `schedule:${encodeURIComponent(id)}:${runKey}:${encodeURIComponent(target)}:raw_paste`;
+            const operationId = `schedule:${id}:${stableRunId}`;
+            const sourceBootId = this.daemons.get(source)?.bootId ?? this.managerBootId;
+            const admitted = this.deliveryOutbox.admit({
+                operationId,
+                sourceKey,
+                sourceInstance: source,
+                sourceDaemonBootId: sourceBootId,
+                targetInstance: target,
+                kind: "raw_paste",
+                correlationId: operationId,
+                payload: {
                     type: "raw_paste",
                     content: message,
-                    delivery_epoch: this.getDeliveryEpoch(target),
-                });
-                this.scheduler.recordRun(id, "delivered");
-                this.logger.info({ target, scheduleId: id, label }, "Silent schedule injected via raw_paste");
-            }
-            else {
-                this.scheduler.recordRun(id, "instance_offline", "IPC not connected");
-                this.logger.warn({ target, scheduleId: id }, "Silent schedule: IPC not connected, skipping");
-            }
+                    schedule_id: id,
+                    schedule_run_id: stableRunId,
+                },
+            });
+            this.scheduler.recordRun(id, "queued", `durable raw_paste delivery_id=${admitted.delivery.deliveryId}`);
+            this.logger.info({ target, scheduleId: id, runId: stableRunId, deliveryId: admitted.delivery.deliveryId, duplicate: !admitted.inserted }, "Silent schedule durably admitted as raw_paste");
+            this.scheduleDeliveryOutboxPump();
             return;
         }
         const schedulerDefaults = this.fleetConfig?.defaults.scheduler;
@@ -11383,6 +11456,10 @@ Plus the operational skills (fleet-health, instance-lifecycle, scheduling, sessi
             clearInterval(this.eventLogPruneTimer);
             this.eventLogPruneTimer = null;
         }
+        if (this.replyObligationTimer) {
+            clearInterval(this.replyObligationTimer);
+            this.replyObligationTimer = null;
+        }
         if (this.logRotateTimer) {
             clearInterval(this.logRotateTimer);
             this.logRotateTimer = null;