@cydm/happy-elves 0.1.0-beta.491 → 0.1.0-beta.495

package/apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-cli",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.495",
   "private": true,
   "type": "module"
 }

package/apps/daemon/dist/orchestration/remote-work-budget.d.ts

@@ -0,0 +1 @@
+export declare function scheduleRemoteOrchestrationWork<T>(machineId: string, run: () => Promise<T>): Promise<T>;

package/apps/daemon/dist/orchestration/remote-work-budget.js

@@ -0,0 +1,51 @@
+/**
+ * One process-wide budget for every remote orchestration operation.
+ *
+ * Transcript catch-up, startup reconcile, and queue/handler initiated work
+ * all use this scheduler. Keeping the fence here prevents separate entry
+ * points from opening more sockets than the peer can service.
+ */
+const maxRemoteWork = 4;
+const maxRemoteWorkPerMachine = 2;
+const pending = [];
+const activeByMachine = new Map();
+let active = 0;
+let draining = false;
+export function scheduleRemoteOrchestrationWork(machineId, run) {
+    return new Promise((resolve, reject) => {
+        pending.push({ machineId, run: async () => await run(), resolve: (value) => resolve(value), reject });
+        void drainRemoteOrchestrationWork();
+    });
+}
+async function drainRemoteOrchestrationWork() {
+    if (draining)
+        return;
+    draining = true;
+    try {
+        for (;;) {
+            if (active >= maxRemoteWork)
+                return;
+            const index = pending.findIndex((work) => (activeByMachine.get(work.machineId) ?? 0) < maxRemoteWorkPerMachine);
+            if (index < 0)
+                return;
+            const work = pending.splice(index, 1)[0];
+            if (!work)
+                return;
+            active += 1;
+            activeByMachine.set(work.machineId, (activeByMachine.get(work.machineId) ?? 0) + 1);
+            void work.run().then(work.resolve, work.reject).finally(() => {
+                active -= 1;
+                const machineActive = activeByMachine.get(work.machineId) ?? 1;
+                if (machineActive <= 1)
+                    activeByMachine.delete(work.machineId);
+                else
+                    activeByMachine.set(work.machineId, machineActive - 1);
+                void drainRemoteOrchestrationWork();
+            });
+        }
+    }
+    finally {
+        draining = false;
+    }
+}
+//# sourceMappingURL=remote-work-budget.js.map

package/apps/daemon/dist/orchestration/runner.js

@@ -12,55 +12,129 @@ import { encryptedSessionMetadata } from "../session/metadata.js";
 import { loadSessionProjections } from "../session/store.js";
 import { appendAttention, expireInflightLeases, leaseNextAttention, latestAttentionForTask, markAttentionDelivered, markAttentionDeliveryFailed, markSubscriptionLiveness, recordTurnRuntimeBinding, reopenUnresolvedArtifactFeedbackAttentions, readOrchestrationStore, } from "./store.js";
 import { attentionFromReconcile, catchUpSubscriptionEvents } from "./watcher.js";
+import { scheduleRemoteOrchestrationWork } from "./remote-work-budget.js";
 import { resolveWakeIdentity } from "./wake-identity.js";
 const activeDeliveryTicks = new Set();
 const reconciledSubscriptions = new Set();
+const remoteScanStates = new Map();
+const pendingRemoteReconciles = new Map();
+let orchestrationTickInFlight = false;
 const defaultStaleMs = 30 * 60 * 1000;
+const defaultRemoteScanIntervalMs = 10_000;
+const defaultRemoteScanBackoffMs = 5_000;
+const maxRemoteScanBackoffMs = 60_000;
 export async function tickOrchestration(config) {
-    await ensureProjectedSessionsLoaded();
-    await expireInflightLeases();
-    await scanRemoteSubscriptions(config);
-    await reconcileSubscriptionsOnce(config);
-    await markStaleSubscriptions(config);
-    await reopenUnresolvedArtifactFeedbackAttentions();
-    const busyConsumerSessionIds = new Set();
-    for (const sessionId of sessions.keys()) {
-        if (sessionTurns.activeTurnId(sessionId))
-            busyConsumerSessionIds.add(sessionId);
-    }
-    const attention = await leaseNextAttention({
-        consumerSessionIds: new Set(sessions.keys()),
-        busyConsumerSessionIds,
-        leaseMs: deliveryLeaseMs(),
-    });
-    if (!attention)
+    if (orchestrationTickInFlight)
         return;
-    const key = `${attention.taskId}:${attention.eventCursor}`;
-    if (activeDeliveryTicks.has(key))
-        return;
-    activeDeliveryTicks.add(key);
+    orchestrationTickInFlight = true;
     try {
-        await deliverAttention(config, attention);
+        await ensureProjectedSessionsLoaded();
+        await expireInflightLeases();
+        // This only schedules bounded, per-subscription remote work. It never
+        // waits for a remote provider, so a slow/offline peer cannot delay local
+        // reconcile, artifact-feedback delivery, or another peer.
+        await scanRemoteSubscriptions(config);
+        await reconcileSubscriptionsOnce(config);
+        await markStaleSubscriptions(config);
+        await reopenUnresolvedArtifactFeedbackAttentions();
+        const busyConsumerSessionIds = new Set();
+        for (const sessionId of sessions.keys()) {
+            if (sessionTurns.activeTurnId(sessionId))
+                busyConsumerSessionIds.add(sessionId);
+        }
+        const attention = await leaseNextAttention({
+            consumerSessionIds: new Set(sessions.keys()),
+            busyConsumerSessionIds,
+            leaseMs: deliveryLeaseMs(),
+        });
+        if (!attention)
+            return;
+        const key = `${attention.taskId}:${attention.eventCursor}`;
+        if (activeDeliveryTicks.has(key))
+            return;
+        activeDeliveryTicks.add(key);
+        try {
+            await deliverAttention(config, attention);
+        }
+        finally {
+            activeDeliveryTicks.delete(key);
+        }
     }
     finally {
-        activeDeliveryTicks.delete(key);
+        orchestrationTickInFlight = false;
     }
 }
 async function scanRemoteSubscriptions(config) {
     const store = await readOrchestrationStore();
-    for (const subscription of store.subscriptions) {
-        if (subscription.status !== "active")
-            continue;
-        const crewMachineId = subscription.crewMachineId ?? config.machineId;
-        if (crewMachineId === config.machineId)
+    const now = Date.now();
+    const candidates = store.subscriptions
+        .filter((subscription) => subscription.status === "active")
+        .filter((subscription) => (subscription.crewMachineId ?? config.machineId) !== config.machineId)
+        .map((subscription) => {
+        const key = remoteSubscriptionKey(subscription);
+        const state = remoteScanStates.get(key) ?? { nextAt: 0, backoffMs: 0 };
+        remoteScanStates.set(key, state);
+        const persistedAt = Date.parse(subscription.lastRemoteScanAt ?? "");
+        const persistedNextAt = Number.isFinite(persistedAt) ? persistedAt + remoteScanIntervalMs() : 0;
+        return { subscription, key, state, dueAt: Math.max(state.nextAt, persistedNextAt) };
+    })
+        .filter((candidate) => candidate.dueAt <= now)
+        .sort((left, right) => left.dueAt - right.dueAt);
+    for (const candidate of candidates) {
+        if (candidate.state.inFlight)
             continue;
-        await scanRemoteTranscript(config, subscription).catch((error) => {
-            console.error(`Remote orchestration transcript scan failed for ${subscription.taskId}: ${error instanceof Error ? error.message : String(error)}`);
+        candidate.state.nextAt = Number.POSITIVE_INFINITY;
+        // catchUpSubscriptionEvents owns the budget for the actual remote RPC.
+        // Do not acquire it here as well: an outer scan slot waiting for an inner
+        // catch-up slot would deadlock a saturated peer backlog.
+        const operation = runRemoteSubscriptionScan(config, candidate.subscription, candidate.state)
+            .finally(() => {
+            candidate.state.inFlight = undefined;
+        });
+        candidate.state.inFlight = operation;
+    }
+    void drainRemoteReconciles();
+}
+function enqueueRemoteReconcile(config, subscription) {
+    pendingRemoteReconciles.set(remoteSubscriptionKey(subscription), { config, subscription });
+    void drainRemoteReconciles();
+}
+async function drainRemoteReconciles() {
+    for (const [key, pending] of pendingRemoteReconciles) {
+        const crewMachineId = pending.subscription.crewMachineId ?? pending.config.machineId;
+        pendingRemoteReconciles.delete(key);
+        void scheduleRemoteOrchestrationWork(crewMachineId, () => reconcileRemoteSubscription(pending.config, pending.subscription))
+            .catch((error) => {
+            console.error(`Remote orchestration reconcile failed for ${pending.subscription.taskId}: ${error instanceof Error ? error.message : String(error)}`);
         });
     }
 }
+async function runRemoteSubscriptionScan(config, subscription, state) {
+    let result;
+    try {
+        result = await scanRemoteTranscript(config, subscription);
+    }
+    catch (error) {
+        console.error(`Remote orchestration transcript scan crashed for ${subscription.taskId}: ${error instanceof Error ? error.message : String(error)}`);
+        result = { status: "failed" };
+    }
+    if (result.status === "ok") {
+        state.backoffMs = 0;
+        state.nextAt = Date.now() + remoteScanIntervalMs();
+        return;
+    }
+    state.backoffMs = Math.min(maxRemoteScanBackoffMs, state.backoffMs > 0 ? state.backoffMs * 2 : defaultRemoteScanBackoffMs);
+    state.nextAt = Date.now() + state.backoffMs;
+}
+function remoteSubscriptionKey(subscription) {
+    return `${subscription.crewMachineId ?? "local"}:${subscription.taskId}:${subscription.crewSessionId}:${subscription.turnId}`;
+}
 async function scanRemoteTranscript(config, subscription) {
-    await catchUpSubscriptionEvents({ config, subscription });
+    const result = await catchUpSubscriptionEvents({ config, subscription });
+    if (result.status === "failed") {
+        console.error(`Remote orchestration transcript scan failed for ${subscription.taskId}`);
+    }
+    return result;
 }
 async function reconcileSubscriptionsOnce(config) {
     const store = await readOrchestrationStore();
@@ -76,7 +150,10 @@ async function reconcileSubscriptionsOnce(config) {
         if (latest?.eventTypes.some((type) => type === "terminal" || type === "failure"))
             continue;
         if (crewMachineId !== config.machineId) {
-            await reconcileRemoteSubscription(config, subscription);
+            // Remote reconcile shares the same bounded/fair scheduler as transcript
+            // catch-up. Startup recovery must not create an unbounded second fleet
+            // of sockets for one slow peer.
+            enqueueRemoteReconcile(config, subscription);
             continue;
         }
         await reconcileLocalSubscription(config, subscription, "startup");
@@ -428,6 +505,13 @@ function deliveryLeaseMs() {
     const parsed = Number(raw);
     return Number.isFinite(parsed) && parsed > 0 ? parsed : 5 * 60 * 1000;
 }
+function remoteScanIntervalMs() {
+    const raw = process.env.HAPPY_ELVES_ORCHESTRATION_REMOTE_SCAN_MS;
+    if (!raw)
+        return defaultRemoteScanIntervalMs;
+    const parsed = Number(raw);
+    return Number.isFinite(parsed) && parsed >= 1_000 ? parsed : defaultRemoteScanIntervalMs;
+}
 function internalSocket() {
     return {
         readyState: WebSocket.CLOSED,

package/apps/daemon/dist/orchestration/store.d.ts

@@ -89,6 +89,10 @@ export type OrchestrationSubscriptionRecord = {
     lastSeenAt?: string;
     lastRemoteScanAt?: string;
     remoteEventSeq?: number;
+    /** Last provider item used as the forward catch-up anchor for remote scans. */
+    remoteEventItemId?: string;
+    /** Append-only runtime event cursor; bound to the exact Crew session/turn. */
+    remoteEventCursor?: string;
     createdAt: string;
     updatedAt: string;
 };
@@ -236,7 +240,12 @@ export declare function upsertSubscription(input: Omit<OrchestrationSubscription
 export declare function activeSubscriptionsForTurn(crewSessionId: string, turnId: string): Promise<OrchestrationSubscriptionRecord[]>;
 export declare function markSubscriptionLiveness(crewSessionId: string, turnId: string, at?: string): Promise<void>;
 export declare function markSubscriptionRemoteScan(taskId: string, input: {
+    crewMachineId?: string;
+    crewSessionId?: string;
+    turnId?: string;
     remoteEventSeq?: number;
+    remoteEventItemId?: string;
+    remoteEventCursor?: string;
     at?: string;
 }): Promise<void>;
 export declare function appendAttention(input: AttentionInput): Promise<TaskAttentionRecord[]>;

package/apps/daemon/dist/orchestration/store.js

@@ -712,11 +712,18 @@ export async function upsertSubscription(input) {
 function upsertSubscriptionRecord(store, input, now) {
     const cursor = input.cursor ?? cursorFromNumber(store.nextCursor);
     const index = store.subscriptions.findIndex((record) => record.taskId === input.taskId);
+    const previous = index >= 0 ? store.subscriptions[index] : undefined;
+    const previousRemoteEventSeq = previous?.remoteEventSeq;
+    const crewMachineId = input.crewMachineId ?? previous?.crewMachineId;
+    const bindingUnchanged = Boolean(previous &&
+        previous.crewMachineId === crewMachineId &&
+        previous.crewSessionId === input.crewSessionId &&
+        previous.turnId === input.turnId);
     const next = {
-        ...(index >= 0 ? store.subscriptions[index] : { createdAt: now }),
+        ...(previous ? { createdAt: previous.createdAt } : { createdAt: now }),
         taskId: input.taskId,
         ...(input.consumerMachineId ? { consumerMachineId: input.consumerMachineId } : {}),
-        ...(input.crewMachineId ? { crewMachineId: input.crewMachineId } : {}),
+        ...(crewMachineId ? { crewMachineId } : {}),
         consumerSessionId: input.consumerSessionId,
         crewSessionId: input.crewSessionId,
         turnId: input.turnId,
@@ -725,9 +732,13 @@ function upsertSubscriptionRecord(store, input, now) {
         eventTypes: normalizeEventTypes(input.eventTypes),
         cursor,
         status: input.status ?? "active",
-        ...(input.lastSeenAt ? { lastSeenAt: input.lastSeenAt } : {}),
-        ...(input.lastRemoteScanAt ? { lastRemoteScanAt: input.lastRemoteScanAt } : {}),
-        ...(Number.isInteger(input.remoteEventSeq) ? { remoteEventSeq: input.remoteEventSeq } : {}),
+        ...(input.lastSeenAt ? { lastSeenAt: input.lastSeenAt } : bindingUnchanged && previous?.lastSeenAt ? { lastSeenAt: previous.lastSeenAt } : {}),
+        ...(bindingUnchanged && previous?.lastRemoteScanAt ? { lastRemoteScanAt: previous.lastRemoteScanAt } : input.lastRemoteScanAt ? { lastRemoteScanAt: input.lastRemoteScanAt } : {}),
+        ...(bindingUnchanged && Number.isInteger(previousRemoteEventSeq)
+            ? { remoteEventSeq: previousRemoteEventSeq }
+            : Number.isInteger(input.remoteEventSeq) ? { remoteEventSeq: input.remoteEventSeq } : {}),
+        ...(bindingUnchanged && previous?.remoteEventItemId ? { remoteEventItemId: previous.remoteEventItemId } : input.remoteEventItemId ? { remoteEventItemId: input.remoteEventItemId } : {}),
+        ...(bindingUnchanged && previous?.remoteEventCursor ? { remoteEventCursor: previous.remoteEventCursor } : input.remoteEventCursor ? { remoteEventCursor: input.remoteEventCursor } : {}),
         updatedAt: now,
     };
     if (index >= 0)
@@ -759,11 +770,21 @@ export async function markSubscriptionRemoteScan(taskId, input) {
         for (const record of current.subscriptions) {
             if (record.taskId !== taskId || record.status !== "active")
                 continue;
+            if (input.crewMachineId !== undefined && record.crewMachineId !== input.crewMachineId)
+                continue;
+            if (input.crewSessionId !== undefined && record.crewSessionId !== input.crewSessionId)
+                continue;
+            if (input.turnId !== undefined && record.turnId !== input.turnId)
+                continue;
             record.lastRemoteScanAt = at;
             const remoteEventSeq = input.remoteEventSeq;
             if (remoteEventSeq !== undefined && Number.isInteger(remoteEventSeq)) {
                 record.remoteEventSeq = Math.max(record.remoteEventSeq ?? 0, remoteEventSeq);
             }
+            if (input.remoteEventItemId)
+                record.remoteEventItemId = input.remoteEventItemId;
+            if (input.remoteEventCursor)
+                record.remoteEventCursor = input.remoteEventCursor;
             record.updatedAt = at;
         }
         return current;
@@ -1430,6 +1451,8 @@ function normalizeSubscriptionRecord(value, localMachineId) {
         ...(typeof value.providerTurnId === "string" && value.providerTurnId ? { providerTurnId: value.providerTurnId } : {}),
         ...(typeof value.lastRemoteScanAt === "string" && value.lastRemoteScanAt ? { lastRemoteScanAt: value.lastRemoteScanAt } : {}),
         ...(typeof remoteEventSeq === "number" && Number.isInteger(remoteEventSeq) && remoteEventSeq >= 0 ? { remoteEventSeq } : {}),
+        ...(typeof value.remoteEventItemId === "string" && value.remoteEventItemId ? { remoteEventItemId: value.remoteEventItemId } : {}),
+        ...(typeof value.remoteEventCursor === "string" && value.remoteEventCursor ? { remoteEventCursor: value.remoteEventCursor } : {}),
     };
 }
 function isTurnBindingRecord(value) {

package/apps/daemon/dist/orchestration/watcher.d.ts

@@ -12,10 +12,16 @@ export declare function observeTurnReconcile(input: {
     config: DaemonConfig;
     result: TurnReconcileResult;
 }): Promise<void>;
+export type SubscriptionCatchUpResult = {
+    status: "ok" | "unreachable" | "failed";
+    maxSeq?: number;
+    nextAfter?: string;
+    lastItemId?: string;
+};
 export declare function catchUpSubscriptionEvents(input: {
     config: DaemonConfig;
     subscription: OrchestrationSubscriptionRecord;
-}): Promise<void>;
+}): Promise<SubscriptionCatchUpResult>;
 export declare function attentionFromReconcile(result: TurnReconcileResult): {
     eventType: TaskAttentionEventType;
     reason: string;

package/apps/daemon/dist/orchestration/watcher.js

@@ -1,8 +1,13 @@
 import { randomId, } from "../../../../packages/shared/dist/index.js";
 import { appendAudit } from "../audit.js";
-import { requestRemoteMachineResult } from "../relay/remote-command.js";
+import { RemoteCommandError, requestRemoteMachineResult } from "../relay/remote-command.js";
 import { runtimeRead } from "../session/runtime-truth.js";
 import { activeSubscriptionsForTurn, appendAttention, markSubscriptionLiveness, markSubscriptionRemoteScan, recordTurnRuntimeBinding, } from "./store.js";
+import { scheduleRemoteOrchestrationWork } from "./remote-work-budget.js";
+const remoteCatchupsInFlight = new Map();
+const legacyRemoteCursorFallbackUntil = new Map();
+const legacyRemoteCursorFallbackMs = 5 * 60 * 1000;
+const initialRemoteEventCursor = "event_000000000000";
 export async function observeSessionEvent(input) {
     const attention = attentionFromEvent(input.payload);
     if (!attention) {
@@ -95,6 +100,27 @@ export async function observeTurnReconcile(input) {
 }
 export async function catchUpSubscriptionEvents(input) {
     const crewMachineId = input.subscription.crewMachineId ?? input.config.machineId;
+    if (crewMachineId !== input.config.machineId) {
+        const key = `${crewMachineId}:${input.subscription.taskId}:${input.subscription.crewSessionId}:${input.subscription.turnId}`;
+        const existing = remoteCatchupsInFlight.get(key);
+        if (existing)
+            return await existing;
+        const operation = scheduleRemoteOrchestrationWork(crewMachineId, () => catchUpSubscriptionEventsInternal(input)).finally(() => {
+            if (remoteCatchupsInFlight.get(key) === operation)
+                remoteCatchupsInFlight.delete(key);
+        });
+        remoteCatchupsInFlight.set(key, operation);
+        return await operation;
+    }
+    return await catchUpSubscriptionEventsInternal(input);
+}
+async function catchUpSubscriptionEventsInternal(input) {
+    const crewMachineId = input.subscription.crewMachineId ?? input.config.machineId;
+    const remoteCursorEnabled = crewMachineId !== input.config.machineId && canUseRemoteEventCursor(crewMachineId);
+    const remoteCursor = remoteCursorEnabled
+        ? input.subscription.remoteEventCursor
+            ?? (input.subscription.remoteEventSeq ? `event_${String(input.subscription.remoteEventSeq).padStart(12, "0")}` : initialRemoteEventCursor)
+        : undefined;
     try {
         const result = crewMachineId === input.config.machineId
             ? await runtimeRead(input.config, {
@@ -118,6 +144,8 @@ export async function catchUpSubscriptionEvents(input) {
                     action: "turn.eventsPage",
                     sessionId: input.subscription.crewSessionId,
                     turnId: input.subscription.turnId,
+                    ...(remoteCursor ? { after: remoteCursor } : {}),
+                    ...(input.subscription.remoteEventItemId && !remoteCursor ? { afterItemId: input.subscription.remoteEventItemId } : {}),
                     ...(input.subscription.runtimeTurnId ?? input.subscription.providerTurnId
                         ? { runtimeTurnId: input.subscription.runtimeTurnId ?? input.subscription.providerTurnId }
                         : {}),
@@ -137,7 +165,14 @@ export async function catchUpSubscriptionEvents(input) {
                 errorCode: "errorCode" in result ? result.errorCode : undefined,
                 reason: "reason" in result ? result.reason : undefined,
             });
-            return;
+            return { status: "unreachable" };
+        }
+        // A rolling-upgrade peer may accept the old runtimeRead command but omit
+        // the forward cursor in its response. Treat that as the legacy path for
+        // this machine; in particular, never advance a durable cursor from a
+        // page whose peer cannot prove append-only progress.
+        if (crewMachineId !== input.config.machineId && remoteCursorEnabled && !result.cursorSupported && !result.nextAfter) {
+            legacyRemoteCursorFallbackUntil.set(crewMachineId, Date.now() + legacyRemoteCursorFallbackMs);
         }
         const observed = await observeSubscriptionRuntimeEvents({
             config: input.config,
@@ -146,11 +181,28 @@ export async function catchUpSubscriptionEvents(input) {
             events: result.events,
             skipThroughSeq: crewMachineId === input.config.machineId ? 0 : input.subscription.remoteEventSeq ?? 0,
         });
-        if (crewMachineId !== input.config.machineId && observed.maxSeq > (input.subscription.remoteEventSeq ?? 0)) {
-            await markSubscriptionRemoteScan(input.subscription.taskId, { remoteEventSeq: observed.maxSeq });
+        if (crewMachineId !== input.config.machineId) {
+            const canAdvance = Boolean(result.nextAfter);
+            await markSubscriptionRemoteScan(input.subscription.taskId, {
+                crewMachineId,
+                crewSessionId: input.subscription.crewSessionId,
+                turnId: input.subscription.turnId,
+                ...(canAdvance && observed.maxSeq > (input.subscription.remoteEventSeq ?? 0) ? { remoteEventSeq: observed.maxSeq } : {}),
+                ...(canAdvance && observed.lastItemId ? { remoteEventItemId: observed.lastItemId } : {}),
+                ...(canAdvance && result.nextAfter ? { remoteEventCursor: result.nextAfter } : {}),
+            });
         }
+        return {
+            status: "ok",
+            maxSeq: observed.maxSeq,
+            ...(result.nextAfter ? { nextAfter: result.nextAfter } : {}),
+            ...(observed.lastItemId ? { lastItemId: observed.lastItemId } : {}),
+        };
     }
     catch (error) {
+        if (crewMachineId !== input.config.machineId && remoteCursorEnabled && isRemoteEventCursorCompatibilityFailure(error)) {
+            legacyRemoteCursorFallbackUntil.set(crewMachineId, Date.now() + legacyRemoteCursorFallbackMs);
+        }
         await appendOrchestrationAudit(input.config, "orchestration.subscription.catchup_failed", `Subscription catch-up failed for ${input.subscription.taskId}`, {
             taskId: input.subscription.taskId,
             crewMachineId,
@@ -158,17 +210,38 @@ export async function catchUpSubscriptionEvents(input) {
             turnId: input.subscription.turnId,
             error: error instanceof Error ? error.message : String(error),
         });
+        return { status: "failed" };
     }
 }
+function canUseRemoteEventCursor(crewMachineId) {
+    const until = legacyRemoteCursorFallbackUntil.get(crewMachineId);
+    if (until === undefined)
+        return true;
+    if (until <= Date.now()) {
+        legacyRemoteCursorFallbackUntil.delete(crewMachineId);
+        return true;
+    }
+    return false;
+}
+function isRemoteEventCursorCompatibilityFailure(error) {
+    if (!(error instanceof RemoteCommandError))
+        return false;
+    return error.code === "REMOTE_COMMAND_TIMEOUT" ||
+        error.code === "REMOTE_MACHINE_ERROR" ||
+        /INVALID|UNKNOWN|UNRECOGNIZED|SCHEMA|PROTOCOL|CURSOR|AFTER/u.test(error.code);
+}
 async function observeSubscriptionRuntimeEvents(input) {
     const crewMachineId = input.crewMachineId ?? input.subscription.crewMachineId ?? input.config.machineId;
     const skipThroughSeq = input.skipThroughSeq ?? 0;
     let maxSeq = skipThroughSeq;
     let observed = 0;
+    let lastItemId;
     for (const event of [...input.events].sort((left, right) => (left.seq ?? 0) - (right.seq ?? 0))) {
         const seq = event.seq ?? 0;
         if (seq > 0)
             maxSeq = Math.max(maxSeq, seq);
+        if (event.itemId && seq >= maxSeq)
+            lastItemId = event.itemId;
         if (seq > 0 && seq <= skipThroughSeq)
             continue;
         if (event.turnId !== input.subscription.turnId)
@@ -190,7 +263,7 @@ async function observeSubscriptionRuntimeEvents(input) {
         });
         observed += 1;
     }
-    return { maxSeq, observed };
+    return { maxSeq, observed, ...(lastItemId ? { lastItemId } : {}) };
 }
 function attentionFromEvent(payload) {
     if (payload.type === "option_request") {

package/apps/daemon/dist/relay/connection.js

@@ -84,6 +84,22 @@ export async function handleServerMessage(ws, config, message) {
                 capability: message.capability,
             },
         });
+        // A detached relay operation failure is recoverable only after the
+        // authenticated stream is recreated.  Keeping this socket open would
+        // leave durable packets stranded: the relay has already rejected the
+        // replay/live send, while the daemon's normal outbox drain is only
+        // kicked on a new socket.  Close the stream for storage-generation
+        // failures so start() performs its bounded reconnect loop.  Do not do
+        // this for ordinary command validation errors.
+        if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
+            try {
+                ws.close();
+            }
+            catch {
+                // The close event/error path still performs cleanup if the socket
+                // implementation rejects a redundant close.
+            }
+        }
         return;
     }
     if (message.type === "machine:createSession") {
@@ -268,6 +284,14 @@ async function handleRelayWireMessage(ws, config, message) {
                 packetId: message.packetId,
             },
         });
+        if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
+            try {
+                ws.close();
+            }
+            catch {
+                // Cleanup is also driven by the close/error listeners below.
+            }
+        }
         return;
     }
     if (message.type !== "server:packet")
@@ -338,6 +362,9 @@ async function handleRelayWireMessage(ws, config, message) {
         ackIncomingPacket(ws, message);
     }
 }
+function shouldReconnectAfterRelayError(code) {
+    return code === "RELAY_STORAGE_TIMEOUT" || code === "RELAY_STORAGE_UNAVAILABLE" || code === "RELAY_STORAGE_CLOSED";
+}
 function errorCode(error) {
     if (!error || typeof error !== "object")
         return undefined;

package/apps/daemon/dist/relay/remote-command.js

@@ -51,6 +51,10 @@ export async function publishRemoteControllerCommand(config, input) {
                 // durable daemon connection (with its own cursor) must remain the
                 // only consumer that can acknowledge those commands.
                 cursor: Number.MAX_SAFE_INTEGER,
+                // This ephemeral RPC only needs the session-to-machine routing table
+                // to publish one targeted command. Do not make every status/read call
+                // download the account's retained transcript and artifact payloads.
+                snapshotMode: "routing",
             }));
         });
         ws.addEventListener("error", () => {

package/apps/daemon/dist/runtime/headless-event-projector.d.ts

@@ -32,11 +32,22 @@ export declare function selectThreadEventsAfterItem(client: HeadlessRuntimeClien
     events: HeadlessRuntimeEvent[];
     anchorFound: boolean;
 }>;
+/**
+ * Read a forward page from the append-only runtime event sequence. Unlike an
+ * item anchor this cursor advances for every provider event, including
+ * repeated updates to one item and terminal events without an item id.
+ */
+export declare function selectThreadEventsAfterCursor(client: HeadlessRuntimeClient, threadId: string, afterCursor: string, limit: number): Promise<{
+    events: HeadlessRuntimeEvent[];
+    hasMoreBefore: boolean;
+    nextAfter?: string;
+}>;
 export declare function historicalEventsFromRuntimeEvents(runtimeEvents: HeadlessRuntimeEvent[]): RuntimeHistoricalEvent[];
-export declare function limitHistoricalEventsByBytes(events: RuntimeHistoricalEvent[], maxBytes: number | undefined): {
+export declare function limitHistoricalEventsByBytes(events: RuntimeHistoricalEvent[], maxBytes: number | undefined, direction?: "before" | "after"): {
     events: RuntimeHistoricalEvent[];
     truncated: boolean;
     nextBefore?: string;
+    nextAfter?: string;
 };
 export declare function terminalHistoricalEventFromTurn(turn: Turn & {
     status: "completed" | "interrupted" | "failed";

package/apps/daemon/dist/runtime/headless-event-projector.js

@@ -330,6 +330,26 @@ export async function selectThreadEventsAfterItem(client, threadId, afterItemId,
         events: sortRuntimeEventsByCursor([...selected, ...terminalEvents]),
     };
 }
+/**
+ * Read a forward page from the append-only runtime event sequence. Unlike an
+ * item anchor this cursor advances for every provider event, including
+ * repeated updates to one item and terminal events without an item id.
+ */
+export async function selectThreadEventsAfterCursor(client, threadId, afterCursor, limit) {
+    const page = await client.listThreadEvents({
+        threadId,
+        order: "asc",
+        afterCursor,
+        limit,
+    });
+    const events = sortRuntimeEventsByCursor(page.events);
+    const nextAfter = events.at(-1)?.cursor;
+    return {
+        events,
+        hasMoreBefore: false,
+        ...(nextAfter ? { nextAfter } : {}),
+    };
+}
 export function historicalEventsFromRuntimeEvents(runtimeEvents) {
     const deltaItemIds = runtimeEvents
         .filter((event) => event.method === "item/delta")
@@ -384,27 +404,57 @@ export function historicalEventsFromRuntimeEvents(runtimeEvents) {
         .sort((a, b) => (a.occurredAt ?? 0) - (b.occurredAt ?? 0) || (a.seq ?? 0) - (b.seq ?? 0))
         .map((event, index) => ({ ...event, seq: event.seq ?? index + 1 }));
 }
-export function limitHistoricalEventsByBytes(events, maxBytes) {
+export function limitHistoricalEventsByBytes(events, maxBytes, direction = "before") {
     if (maxBytes === undefined || !Number.isFinite(maxBytes) || maxBytes <= 0)
         return { events, truncated: false };
     const groups = groupHistoricalEvents(events);
     const selectedNewestFirst = [];
     let bytes = 0;
-    for (const group of [...groups].reverse()) {
+    const orderedGroups = direction === "after" ? groups : [...groups].reverse();
+    for (const group of orderedGroups) {
         const groupBytes = Buffer.byteLength(JSON.stringify(group.events), "utf8");
         if (bytes + groupBytes > maxBytes) {
             if (selectedNewestFirst.length === 0) {
+                if (direction === "after") {
+                    // A single item can be larger than the page budget. Emit a marker
+                    // for that item, then consume as many following complete groups as
+                    // fit. The cursor advances only through groups represented in the
+                    // response; it must never jump to the raw page tail.
+                    const marker = historyTruncatedMarker(group.events, maxBytes, group.events.length);
+                    const kept = [];
+                    let markerBytes = Buffer.byteLength(JSON.stringify([marker]), "utf8");
+                    for (const following of orderedGroups.slice(1)) {
+                        const followingBytes = Buffer.byteLength(JSON.stringify(following.events), "utf8");
+                        if (markerBytes + followingBytes > maxBytes)
+                            break;
+                        kept.push(following);
+                        markerBytes += followingBytes;
+                    }
+                    const consumed = [group, ...kept];
+                    const lastConsumedEvent = consumed.at(-1)?.events.at(-1);
+                    return {
+                        events: [marker, ...kept.flatMap((item) => item.events)],
+                        truncated: true,
+                        ...(lastConsumedEvent?.seq ? { nextAfter: headlessCursorFromSeq(lastConsumedEvent.seq) } : {}),
+                    };
+                }
                 return {
                     events: [historyTruncatedMarker(group.events, maxBytes, group.events.length)],
                     truncated: true,
-                    ...(group.oldestCursor ? { nextBefore: group.oldestCursor } : {}),
+                    ...(direction === "before"
+                        ? (group.oldestCursor ? { nextBefore: group.oldestCursor } : {})
+                        : {}),
                 };
             }
-            const keptGroups = [...selectedNewestFirst].reverse();
+            const keptGroups = direction === "after" ? selectedNewestFirst : [...selectedNewestFirst].reverse();
+            const lastKeptEvent = keptGroups.at(-1)?.events.at(-1);
             return {
                 events: keptGroups.flatMap((item) => item.events),
                 truncated: true,
-                ...(keptGroups[0]?.oldestCursor ? { nextBefore: keptGroups[0].oldestCursor } : {}),
+                ...(direction === "before" && keptGroups[0]?.oldestCursor ? { nextBefore: keptGroups[0].oldestCursor } : {}),
+                ...(direction === "after" && lastKeptEvent?.seq
+                    ? { nextAfter: headlessCursorFromSeq(lastKeptEvent.seq) }
+                    : {}),
             };
         }
         selectedNewestFirst.push(group);

package/apps/daemon/dist/runtime/headless-history-reader.d.ts

@@ -4,6 +4,7 @@ export type HeadlessHistoricalEventPageInput = {
     client: HeadlessRuntimeClient;
     threadId: string;
     turnId?: string;
+    after?: string;
     afterItemId?: string;
     before?: string;
     beforeItemId?: string;

package/apps/daemon/dist/runtime/headless-history-reader.js

@@ -1,20 +1,25 @@
-import { limitHistoricalEventsByBytes, historicalEventsFromThread, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
+import { limitHistoricalEventsByBytes, historicalEventsFromThread, mapHeadlessRuntimeEventsToHistoricalEvents, selectThreadEventsAfterCursor, selectThreadEventsAfterItem, selectThreadEventsBeforeCursor, selectThreadEventsBeforeItem, selectThreadEventsForTurn, } from "./headless-event-projector.js";
 const defaultEventLimit = 200;
 export async function readHeadlessHistoricalEventPage(input) {
+    assertValidHeadlessEventCursor(input.after);
     assertValidHeadlessEventCursor(input.before);
     const limitItems = clampLimit(input.limitItems);
     const selection = await selectHeadlessRuntimeEvents(input, limitItems);
     const runtimeEvents = selection.events;
     const allEvents = await authoritativeTurnEvents(input, runtimeEvents) ?? mapHeadlessRuntimeEventsToHistoricalEvents(runtimeEvents);
-    const limited = limitHistoricalEventsByBytes(allEvents, input.maxBytes);
+    const visibleEvents = input.turnId ? allEvents.filter((event) => event.turnId === input.turnId) : allEvents;
+    const limited = limitHistoricalEventsByBytes(visibleEvents, input.maxBytes, input.after ? "after" : "before");
     const includeSessionHead = !input.turnId && !input.before && !input.beforeItemId;
     const head = includeSessionHead ? latestHeadFromEvents(allEvents) : {};
     const nextBefore = limited.nextBefore ?? selection.nextBefore;
     const hasMoreBefore = Boolean((selection.hasMoreBefore || limited.truncated) && nextBefore);
+    const nextAfter = limited.truncated ? limited.nextAfter : selection.nextAfter;
     return {
         events: limited.events,
         hasMoreBefore,
         ...(hasMoreBefore && nextBefore ? { nextBefore } : {}),
+        ...(nextAfter ? { nextAfter } : {}),
+        ...(input.after ? { cursorSupported: true } : {}),
         ...(head.currentHead ? { currentHead: head.currentHead } : {}),
         ...(head.lastTurnId ? { lastTurnId: head.lastTurnId } : {}),
     };
@@ -28,7 +33,7 @@ async function authoritativeTurnEvents(input, runtimeEvents) {
     // plus a repeated terminal snapshot for the same item, or a late native
     // reconciliation. Identity is itemId/turnId, never text, so legitimate
     // repeated answers from distinct items remain visible.
-    if (input.before || input.beforeItemId || input.afterItemId)
+    if (input.after || input.before || input.beforeItemId || input.afterItemId)
         return undefined;
     // A few internal readers (and older provider adapters during rolling
     // upgrades) only expose the event reader.  Canonical rebuild is an
@@ -69,13 +74,20 @@ async function listAllThreadTurns(client, threadId) {
     return turns;
 }
 async function selectHeadlessRuntimeEvents(input, limitItems) {
-    if (input.turnId) {
-        return await selectThreadEventsForTurn(input.client, input.threadId, input.turnId, limitItems);
+    if (input.after) {
+        return await selectThreadEventsAfterCursor(input.client, input.threadId, input.after, limitItems);
     }
+    // An after-item anchor is a thread-level forward cursor. It must take
+    // precedence over turn filtering so callers can advance a subscription
+    // without re-reading the first page of a long turn on every poll. The
+    // runtime-truth layer filters the returned page back to the requested turn.
     if (input.afterItemId) {
         const selection = await selectThreadEventsAfterItem(input.client, input.threadId, input.afterItemId, limitItems);
         return { events: selection.events, hasMoreBefore: selection.anchorFound };
     }
+    if (input.turnId) {
+        return await selectThreadEventsForTurn(input.client, input.threadId, input.turnId, limitItems);
+    }
     if (input.beforeItemId) {
         const selection = await selectThreadEventsBeforeItem(input.client, input.threadId, input.beforeItemId, limitItems);
         return { events: selection.events, hasMoreBefore: selection.hasMoreBefore };

package/apps/daemon/dist/session/native-history-refresh.js

@@ -98,7 +98,21 @@ export async function refreshNativeHistoryForLoadedSession(session, config, requ
         const entry = { generation, promise: operation, startedAt: Date.now() };
         orchestrationRefreshes.set(refreshKey, entry);
         try {
-            const result = await operation;
+            const outcome = await Promise.race([
+                operation.then((result) => ({ kind: "result", result })),
+                new Promise((resolve) => setTimeout(() => resolve({ kind: "timeout" }), orchestrationRefreshMaxWaitMs)),
+            ]);
+            if (outcome.kind === "timeout") {
+                entry.abandoned = true;
+                if (orchestrationRefreshes.get(refreshKey) === entry)
+                    orchestrationRefreshes.delete(refreshKey);
+                orchestrationRefreshGenerations.set(refreshKey, (orchestrationRefreshGenerations.get(refreshKey) ?? 0) + 1);
+                // The first caller must have the same bounded wait as followers. Drop
+                // the provider client so the old IPC cannot poison the next refresh.
+                void getHeadlessRuntimeManager(session.cwd).invalidateClientForBinding(session.runtimeBinding, session.cwd);
+                return sessions.get(session.sessionId) ?? session;
+            }
+            const result = outcome.result;
             if (!entry.abandoned && generation !== (orchestrationRefreshGenerations.get(refreshKey) ?? 0)) {
                 if (orchestrationRefreshes.get(refreshKey) === entry)
                     orchestrationRefreshes.delete(refreshKey);

package/apps/daemon/dist/session/runtime-truth.d.ts

@@ -6,6 +6,7 @@ export type RuntimeReadInput = {
     sessionId: string;
     turnId?: string;
     runtimeTurnId?: string;
+    after?: string;
     before?: string;
     beforeItemId?: string;
     afterItemId?: string;

package/apps/daemon/dist/session/runtime-truth.js

@@ -17,7 +17,14 @@ export async function runtimeRead(config, input) {
         // Reconcile that history before projecting turn.result/status so an
         // upgrade/restart repairs old same-turn duplicates through the official
         // read path, without requiring an internal syncNativeHistory call.
-        session = await refreshNativeHistoryForLoadedSession(session, config, `runtime_truth_refresh_${input.requestId}`);
+        // Event catch-up reads the authoritative Headless event log and must stay
+        // independent from the provider-native history refresh. The latter can
+        // involve a slow/remote provider IPC call; putting it in every polling
+        // page made one stale Crew subscription block all other orchestration.
+        // Result/status/list retain the refresh for legacy native-history repair.
+        if (input.action !== "turn.eventsPage") {
+            session = await refreshNativeHistoryForLoadedSession(session, config, `runtime_truth_refresh_${input.requestId}`);
+        }
         runtimeSessionId = runtimeSessionIdForSession(session);
         if (!runtimeSessionId)
             return unreachable(input, `Session ${input.sessionId} does not have a runtime session binding`, undefined, "SESSION_BINDING_MISSING");
@@ -90,6 +97,7 @@ async function readTurnList(input, session, runtimeSessionId) {
         before: input.before,
         beforeItemId: input.beforeItemId,
         afterItemId: input.afterItemId,
+        after: input.after,
         limitItems: input.limitItems ?? 400,
         maxBytes: input.maxBytes ?? 240_000,
     });
@@ -207,6 +215,7 @@ async function readTurnEventsPage(input, session, runtimeSessionId) {
         before: input.before,
         beforeItemId: input.beforeItemId,
         afterItemId: input.afterItemId,
+        after: input.after,
         limitItems: input.limitItems ?? 300,
         maxBytes: input.maxBytes ?? 260_000,
     });
@@ -223,6 +232,8 @@ async function readTurnEventsPage(input, session, runtimeSessionId) {
         events: events.map((event) => runtimeReadEvent(event, input.turnId, runtimeTurnId)),
         hasMoreBefore: page.hasMoreBefore,
         nextBefore: page.nextBefore,
+        ...(page.nextAfter ? { nextAfter: page.nextAfter } : {}),
+        ...(page.cursorSupported ? { cursorSupported: true } : {}),
         ...(page.currentHead ? { currentHead: page.currentHead } : {}),
         ...(page.lastTurnId ? { lastTurnId: page.lastTurnId } : {}),
     };
@@ -252,6 +263,7 @@ async function readRuntimePage(session, input) {
         ...(input.turnId ? { turnId: input.turnId } : {}),
         ...(input.before ? { before: input.before } : {}),
         ...(input.beforeItemId ? { beforeItemId: input.beforeItemId } : {}),
+        ...(input.after ? { after: input.after } : {}),
         ...(input.afterItemId ? { afterItemId: input.afterItemId } : {}),
         limitItems: input.limitItems ?? 200,
         maxBytes: input.maxBytes ?? 200_000,

package/apps/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-daemon",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.495",
   "private": true,
   "type": "module"
 }

package/apps/relay/dist/packet-store.js

@@ -72,10 +72,21 @@ export function initPacketRelayTables(db) {
 
     CREATE INDEX IF NOT EXISTS idx_packets_to_node
       ON packets(space_id, to_node_id, id);
+    -- Replay admission must be able to reject already-acked packets without
+    -- touching the encrypted payload column.  The old query selected
+    -- packets.* while walking every packet addressed to a node, so a node
+    -- with a large, fully-acked history could read gigabytes of overflow
+    -- pages before returning an empty batch.
+    CREATE INDEX IF NOT EXISTS idx_packets_unacked_lookup
+      ON packets(space_id, to_node_id, id, expires_at, packet_id);
     CREATE INDEX IF NOT EXISTS idx_packets_channel
       ON packets(space_id, channel_id, stream_id, id);
+    CREATE INDEX IF NOT EXISTS idx_packets_channel_id
+      ON packets(space_id, channel_id, id);
     CREATE INDEX IF NOT EXISTS idx_packets_expires
       ON packets(expires_at);
+    CREATE INDEX IF NOT EXISTS idx_packets_received_at
+      ON packets(received_at);
     CREATE INDEX IF NOT EXISTS idx_retained_space_key
       ON retained_packets(space_id, retained_key);
   `);
@@ -192,24 +203,62 @@ export function listRetainedPackets(db, input) {
 export function listPacketsForSubscription(db, input) {
     const cursor = input.cursor ?? 0;
     const limit = input.limit ?? 500;
-    const rows = statement(db, `SELECT *
-     FROM packets
-     WHERE space_id = ?
-       AND id > ?
-       AND (expires_at IS NULL OR expires_at > ?)
-       AND (
-         to_node_id = ?
-         OR (? IS NOT NULL AND channel_id = ?)
-       )
-       AND (? IS NULL OR stream_id = ?)
-     ORDER BY id ASC
-     LIMIT ?`).all(input.spaceId, cursor, input.now, input.nodeId, input.subscription.channelId ?? null, input.subscription.channelId ?? null, input.subscription.streamId ?? null, input.subscription.streamId ?? null, limit);
-    return rows.map(packetFromRow);
+    const streamId = input.subscription.streamId;
+    const rows = [];
+    // Keep the historical OR semantics, but execute each arm independently so
+    // SQLite can seek through idx_packets_to_node or idx_packets_channel instead
+    // of scanning and sorting every packet in the account.  A packet can match
+    // both arms; the merge below de-duplicates by the stable packets.id cursor.
+    // The node arm is intentionally unconditional. `nodeId` is the authenticated
+    // connection identity and was always part of the old OR predicate, even for
+    // an empty/channel-only subscription. `subscription.toNodeId` is a snapshot
+    // label, not a permission to redirect this reader to another node.
+    {
+        const sql = streamId === undefined
+            ? `SELECT * FROM packets
+         WHERE space_id = ? AND to_node_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`
+            : `SELECT * FROM packets
+         WHERE space_id = ? AND to_node_id = ? AND stream_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`;
+        const values = streamId === undefined
+            ? [input.spaceId, input.nodeId, cursor, input.now, limit]
+            : [input.spaceId, input.nodeId, streamId, cursor, input.now, limit];
+        rows.push(...statement(db, sql).all(...values));
+    }
+    if (input.subscription.channelId !== undefined) {
+        const sql = streamId === undefined
+            ? `SELECT * FROM packets
+         WHERE space_id = ? AND channel_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`
+            : `SELECT * FROM packets
+         WHERE space_id = ? AND channel_id = ? AND stream_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`;
+        const values = streamId === undefined
+            ? [input.spaceId, input.subscription.channelId, cursor, input.now, limit]
+            : [input.spaceId, input.subscription.channelId, streamId, cursor, input.now, limit];
+        rows.push(...statement(db, sql).all(...values));
+    }
+    const uniqueRows = new Map();
+    for (const row of rows)
+        uniqueRows.set(row.id, row);
+    return [...uniqueRows.values()]
+        .sort((left, right) => left.id - right.id)
+        .slice(0, limit)
+        .map(packetFromRow);
 }
 export function listUnackedPacketsForNode(db, input) {
     const cursor = input.cursor ?? 0;
     const limit = input.limit ?? 500;
-    const rows = statement(db, `SELECT packets.*
+    // First find candidate row ids using a covering routing index.  Do not
+    // select packets.* in this anti-join: encrypted_payload is commonly a
+    // multi-kilobyte value stored in overflow pages, and loading it for every
+    // already-acked packet turns an empty replay into a large disk read.
+    const candidateRows = statement(db, `SELECT packets.id
      FROM packets
      LEFT JOIN packet_acks
        ON packet_acks.space_id = packets.space_id
@@ -222,13 +271,42 @@ export function listUnackedPacketsForNode(db, input) {
        AND packet_acks.packet_id IS NULL
      ORDER BY packets.id ASC
      LIMIT ?`).all(input.nodeId, input.spaceId, cursor, input.nodeId, input.now, limit);
+    if (candidateRows.length === 0)
+        return [];
+    // Fetch the encrypted rows only for the bounded set that really needs to
+    // be replayed.  The ids are already ordered, but retain an explicit ORDER
+    // BY so the FIFO/server-sequence contract remains clear to SQLite.
+    const placeholders = candidateRows.map(() => "?").join(", ");
+    const rows = statement(db, `SELECT * FROM packets
+     WHERE space_id = ? AND id IN (${placeholders})
+     ORDER BY id ASC`).all(input.spaceId, ...candidateRows.map((row) => row.id));
     return rows.map(packetFromRow);
 }
 export function prunePackets(db, input) {
     if (input.maxAgeMs <= 0)
         return;
     const oldestReceivedAt = input.now - input.maxAgeMs;
-    statement(db, "DELETE FROM packets WHERE (expires_at IS NOT NULL AND expires_at <= ?) OR received_at <= ?").run(input.now, oldestReceivedAt);
+    // Retention runs on the same serialized storage worker as packet replay.
+    // Bound each pass so a large historical database cannot occupy the worker
+    // past the operation deadline.  The two indexed arms avoid the old OR
+    // predicate's full packets-table scan when there is nothing to prune.
+    const candidates = statement(db, `WITH expired AS (
+       SELECT id FROM packets
+       WHERE expires_at IS NOT NULL AND expires_at <= ?
+       LIMIT 1000
+     ), stale AS (
+       SELECT id FROM packets
+       WHERE received_at <= ?
+       LIMIT 1000
+     )
+     SELECT id
+     FROM (SELECT id FROM expired UNION SELECT id FROM stale)
+     ORDER BY id ASC
+     LIMIT 1000`).all(input.now, oldestReceivedAt);
+    if (candidates.length === 0)
+        return;
+    const placeholders = candidates.map(() => "?").join(", ");
+    statement(db, `DELETE FROM packets WHERE id IN (${placeholders})`).run(...candidates.map((row) => row.id));
 }
 export function maxPacketSeq(db, spaceId) {
     const row = statement(db, "SELECT COALESCE(MAX(id), 0) AS seq FROM packets WHERE space_id = ?").get(spaceId);

package/apps/relay/dist/storage-worker.js

@@ -13,6 +13,7 @@ import { ackPacket, getPacketById, getRetainedPacket, insertPacket, listPacketsF
 if (parentPort && workerData?.relayStorageWorker === true) {
     const dbPath = String(workerData.dbPath);
     const testStorageDelayMs = Math.max(0, Number(workerData.testStorageDelayMs) || 0);
+    const testStorageDelayFile = typeof workerData.testStorageDelayFile === "string" ? workerData.testStorageDelayFile : undefined;
     mkdirSync(path.dirname(dbPath), { recursive: true });
     const db = new DatabaseSync(dbPath);
     db.exec("PRAGMA journal_mode = WAL");
@@ -26,8 +27,9 @@ if (parentPort && workerData?.relayStorageWorker === true) {
     parentPort.postMessage({ type: "ready" });
     parentPort.on("message", (request) => {
         try {
-            if (testStorageDelayMs > 0)
+            if (testStorageDelayMs > 0 && (!testStorageDelayFile || existsSync(testStorageDelayFile))) {
                 Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, testStorageDelayMs);
+            }
             const value = request.op === "controlPlane"
                 ? runControlPlaneAction(controlPlane, request.action, request.input)
                 : request.op === "directTicketCreate"
@@ -252,6 +254,7 @@ export function createRelayDurableStorage(dbPath) {
                 dbPath,
                 pairingTtlMs: 600_000,
                 testStorageDelayMs: process.env.HAPPY_ELVES_TEST_STORAGE_DELAY_MS,
+                testStorageDelayFile: process.env.HAPPY_ELVES_TEST_STORAGE_DELAY_FILE,
             },
         });
         nextWorker.unref();

package/apps/relay/dist/websocket.js

@@ -158,9 +158,12 @@ export function registerWebsocketRoute(app, context, originAllowed) {
                         });
                         await sendSnapshot(context, openedConnection, message.snapshotMode ?? "full");
                         broadcastPresence(context, openedConnection.spaceId, await presenceWithManifest(context, openedConnection.spaceId, presence));
-                        void replayNodePackets(context, openedConnection, message.cursor);
+                        void replayNodePackets(context, openedConnection, message.cursor).catch((error) => {
+                            reportDetachedOperationFailure(openedConnection, error);
+                        });
                     }).catch((error) => {
                         app.log.warn({ error }, "failed to persist relay presence");
+                        reportDetachedOperationFailure(openedConnection, error, true);
                     });
                 }).catch((error) => {
                     fail(error instanceof Error ? error.message : "Websocket authentication failed", "AUTH_TOKEN_INVALID");
@@ -210,10 +213,15 @@ export function registerWebsocketRoute(app, context, originAllowed) {
             if (!connection)
                 return;
             const closedConnection = connection;
+            closedConnection.closed = true;
             closedConnection.publishChainClosed = true;
             if (closedConnection.expiryTimer)
                 globalThis.clearTimeout(closedConnection.expiryTimer);
-            void flushPendingCursor(context, closedConnection);
+            if (closedConnection.cursorFlushTimer) {
+                globalThis.clearTimeout(closedConnection.cursorFlushTimer);
+                closedConnection.cursorFlushTimer = undefined;
+            }
+            void flushPendingCursor(context, closedConnection).catch(() => undefined);
             unregisterConnection(closedConnection);
             const remaining = connectionsByNode.get(nodeKey(closedConnection.spaceId, closedConnection.nodeId));
             const stillOnline = Boolean(remaining && remaining.size > 0);
@@ -544,7 +552,9 @@ function pumpSubscriptionReplays(context, connection) {
         if (!state || state.running)
             continue;
         state.queued = false;
-        void runSubscriptionReplay(context, connection, key, state);
+        void runSubscriptionReplay(context, connection, key, state).catch((error) => {
+            reportDetachedOperationFailure(connection, error);
+        });
     }
 }
 async function runSubscriptionReplay(context, connection, key, state) {
@@ -613,7 +623,9 @@ function deliverPacket(context, packet) {
     if (packet.toNodeId) {
         const connections = connectionsByNode.get(nodeKey(packet.spaceId, packet.toNodeId));
         for (const connection of connections ?? []) {
-            void sendPacket(context, connection, packet, "live");
+            void sendPacket(context, connection, packet, "live").catch((error) => {
+                reportDetachedOperationFailure(connection, error);
+            });
             delivered.add(connection);
         }
     }
@@ -623,7 +635,9 @@ function deliverPacket(context, packet) {
             continue;
         if (!connection.subscriptions.some((subscription) => subscriptionMatches(connection, subscription, packet)))
             continue;
-        void sendPacket(context, connection, packet, "live");
+        void sendPacket(context, connection, packet, "live").catch((error) => {
+            reportDetachedOperationFailure(connection, error);
+        });
     }
 }
 async function sendPacket(context, connection, packet, source) {
@@ -666,6 +680,8 @@ async function sendPacket(context, connection, packet, source) {
     return true;
 }
 function queueCursorUpdate(context, connection, serverSeq) {
+    if (connection.closed)
+        return;
     connection.pendingCursorServerSeq = Math.max(connection.pendingCursorServerSeq ?? 0, serverSeq);
     if (connection.cursorFlushTimer)
         return;
@@ -674,13 +690,35 @@ function queueCursorUpdate(context, connection, serverSeq) {
     // keeps high-rate replay from turning every packet into an fsync.
     connection.cursorFlushTimer = setTimeout(() => {
         connection.cursorFlushTimer = undefined;
-        void flushPendingCursor(context, connection);
+        void flushPendingCursor(context, connection).catch(() => undefined);
     }, 100);
     connection.cursorFlushTimer.unref?.();
 }
 async function flushPendingCursor(context, connection) {
+    if (connection.cursorFlushInFlight)
+        return await connection.cursorFlushInFlight;
+    const run = flushPendingCursorOnce(context, connection);
+    let trackedRun;
+    trackedRun = run.finally(() => {
+        if (connection.cursorFlushInFlight === trackedRun)
+            connection.cursorFlushInFlight = undefined;
+        // A newer packet can arrive while the write is in flight. Keep one
+        // coalesced follow-up timer, but never resurrect retries after the
+        // socket has closed.
+        if (!connection.closed && connection.pendingCursorServerSeq !== undefined && !connection.cursorFlushTimer) {
+            connection.cursorFlushTimer = setTimeout(() => {
+                connection.cursorFlushTimer = undefined;
+                void flushPendingCursor(context, connection).catch(() => undefined);
+            }, 250);
+            connection.cursorFlushTimer.unref?.();
+        }
+    });
+    connection.cursorFlushInFlight = trackedRun;
+    await trackedRun;
+}
+async function flushPendingCursorOnce(context, connection) {
     if (connection.cursorFlushTimer) {
-        clearTimeout(connection.cursorFlushTimer);
+        globalThis.clearTimeout(connection.cursorFlushTimer);
         connection.cursorFlushTimer = undefined;
     }
     const serverSeq = connection.pendingCursorServerSeq;
@@ -700,7 +738,8 @@ async function flushPendingCursor(context, connection) {
         connection.pendingCursorServerSeq = Math.max(connection.pendingCursorServerSeq ?? 0, serverSeq);
         // Cursor persistence is recoverable; do not terminate the relay process
         // because a worker/database write was temporarily unavailable.
-        setTimeout(() => { void flushPendingCursor(context, connection); }, 250).unref?.();
+        // The single-flight finalizer above owns the one retry timer. In
+        // particular, a close cannot leave an unbounded timer chain behind.
     }
 }
 function closeSlowReceiver(connection) {
@@ -874,7 +913,7 @@ function scheduleConnectionExpiry(context, connection, expiresAt) {
                 if (!current || current.revoked_at)
                     return;
                 if (current.expires_at && current.expires_at > context.now()) {
-                    void scheduleNext();
+                    void scheduleNext().catch(() => undefined);
                     return;
                 }
                 if (!current.expires_at)
@@ -888,7 +927,33 @@ function scheduleConnectionExpiry(context, connection, expiresAt) {
             }).catch(() => undefined);
         }, Math.min(delay, MAX_EXPIRY_TIMER_DELAY_MS));
     };
-    void scheduleNext();
+    void scheduleNext().catch(() => undefined);
+}
+function reportDetachedOperationFailure(connection, error, forceClose = false) {
+    if (connection.socket.readyState !== 1)
+        return;
+    const record = error && typeof error === "object" ? error : {};
+    const code = typeof record.code === "string" ? record.code : "RELAY_COMMAND_FAILED";
+    const message = typeof record.message === "string" ? record.message : String(error);
+    send(connection.socket, { type: "server:error", code, message });
+    // A detached replay/live send has already lost its in-memory continuation.
+    // Close the stream for storage-generation failures so old daemons and
+    // controllers, which only reconnect on close, replay their durable outbox
+    // and retained packets. The stored ACK boundary remains authoritative; no
+    // packet is acknowledged merely because the connection is being closed.
+    if (forceClose || isRecoverableStorageFailure(code)) {
+        connection.closed = true;
+        try {
+            connection.socket.close(1011, code);
+        }
+        catch {
+            // The close/error listener still performs cleanup for transports that
+            // reject a redundant close.
+        }
+    }
+}
+function isRecoverableStorageFailure(code) {
+    return code === "RELAY_STORAGE_TIMEOUT" || code === "RELAY_STORAGE_UNAVAILABLE" || code === "RELAY_STORAGE_CLOSED";
 }
 async function isScopedControllerConnection(context, connection) {
     const token = await context.tokenRecordAsync(connection.authToken);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.495",
   "description": "Remote controller for local coding agents with hosted or self-hosted relay support.",
   "type": "module",
   "bin": {

package/packages/runtime/dist/index.d.ts

@@ -250,6 +250,10 @@ export type RuntimeHistoricalEventPage = {
     events: RuntimeHistoricalEvent[];
     hasMoreBefore?: boolean;
     nextBefore?: string;
+    /** Forward event cursor for incremental readers. */
+    nextAfter?: string;
+    /** True when the reader understood a forward cursor, including an empty page. */
+    cursorSupported?: boolean;
     currentHead?: string;
     lastTurnId?: string;
 };

package/packages/shared/dist/protocol-schemas.d.ts

@@ -1113,6 +1113,8 @@ export declare const runtimeReadCommandResultSchema: z.ZodDiscriminatedUnion<[z.
     }, z.core.$strict>>;
     hasMoreBefore: z.ZodOptional<z.ZodBoolean>;
     nextBefore: z.ZodOptional<z.ZodString>;
+    nextAfter: z.ZodOptional<z.ZodString>;
+    cursorSupported: z.ZodOptional<z.ZodBoolean>;
     currentHead: z.ZodOptional<z.ZodString>;
     lastTurnId: z.ZodOptional<z.ZodString>;
     errorCode: z.ZodOptional<z.ZodString>;

package/packages/shared/dist/protocol-schemas.js

@@ -487,6 +487,8 @@ export const runtimeReadCommandResultSchema = z.discriminatedUnion("action", [
         events: z.array(runtimeReadEventSchema),
         hasMoreBefore: z.boolean().optional(),
         nextBefore: z.string().min(1).optional(),
+        nextAfter: z.string().min(1).optional(),
+        cursorSupported: z.boolean().optional(),
         currentHead: z.string().min(1).optional(),
         lastTurnId: z.string().min(1).optional(),
         errorCode: z.string().min(1).optional(),

package/packages/shared/dist/protocol-types.d.ts

@@ -349,6 +349,8 @@ export type RuntimeTurnEventsPageResult = {
     events: RuntimeReadEvent[];
     hasMoreBefore?: boolean;
     nextBefore?: string;
+    nextAfter?: string;
+    cursorSupported?: boolean;
     currentHead?: string;
     lastTurnId?: string;
     errorCode?: string;

package/packages/shared/dist/protocol.d.ts

@@ -138,6 +138,7 @@ export type ControllerClientMessage = {
     runtimeTurnId?: string;
     before?: string;
     beforeItemId?: string;
+    after?: string;
     afterItemId?: string;
     limitItems?: number;
     maxBytes?: number;
@@ -767,6 +768,7 @@ export type MachineCommand = {
     runtimeTurnId?: string;
     before?: string;
     beforeItemId?: string;
+    after?: string;
     afterItemId?: string;
     limitItems?: number;
     maxBytes?: number;

package/packages/shared/dist/protocol.js

@@ -155,6 +155,7 @@ const controllerMessageSchema = z.union([
         runtimeTurnId: z.string().min(1).optional(),
         before: z.string().min(1).optional(),
         beforeItemId: z.string().min(1).optional(),
+        after: z.string().min(1).optional(),
         afterItemId: z.string().min(1).optional(),
         limitItems: z.number().int().positive().max(1000).optional(),
         maxBytes: z.number().int().positive().max(2_000_000).optional(),
@@ -885,6 +886,7 @@ const machineCommandSchema = z.union([
         runtimeTurnId: z.string().min(1).optional(),
         before: z.string().min(1).optional(),
         beforeItemId: z.string().min(1).optional(),
+        after: z.string().min(1).optional(),
         afterItemId: z.string().min(1).optional(),
         limitItems: z.number().int().positive().max(1000).optional(),
         maxBytes: z.number().int().positive().max(2_000_000).optional(),