@cydm/happy-elves 0.1.0-beta.491 → 0.1.0-beta.494

package/apps/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-cli",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.494",
   "private": true,
   "type": "module"
 }

package/apps/daemon/dist/relay/connection.js

@@ -84,6 +84,22 @@ export async function handleServerMessage(ws, config, message) {
                 capability: message.capability,
             },
         });
+        // A detached relay operation failure is recoverable only after the
+        // authenticated stream is recreated.  Keeping this socket open would
+        // leave durable packets stranded: the relay has already rejected the
+        // replay/live send, while the daemon's normal outbox drain is only
+        // kicked on a new socket.  Close the stream for storage-generation
+        // failures so start() performs its bounded reconnect loop.  Do not do
+        // this for ordinary command validation errors.
+        if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
+            try {
+                ws.close();
+            }
+            catch {
+                // The close event/error path still performs cleanup if the socket
+                // implementation rejects a redundant close.
+            }
+        }
         return;
     }
     if (message.type === "machine:createSession") {
@@ -268,6 +284,14 @@ async function handleRelayWireMessage(ws, config, message) {
                 packetId: message.packetId,
             },
         });
+        if (shouldReconnectAfterRelayError(message.code) && ws.readyState === WebSocket.OPEN) {
+            try {
+                ws.close();
+            }
+            catch {
+                // Cleanup is also driven by the close/error listeners below.
+            }
+        }
         return;
     }
     if (message.type !== "server:packet")
@@ -338,6 +362,9 @@ async function handleRelayWireMessage(ws, config, message) {
         ackIncomingPacket(ws, message);
     }
 }
+function shouldReconnectAfterRelayError(code) {
+    return code === "RELAY_STORAGE_TIMEOUT" || code === "RELAY_STORAGE_UNAVAILABLE" || code === "RELAY_STORAGE_CLOSED";
+}
 function errorCode(error) {
     if (!error || typeof error !== "object")
         return undefined;

package/apps/daemon/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves-daemon",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.494",
   "private": true,
   "type": "module"
 }

package/apps/relay/dist/packet-store.js

@@ -72,10 +72,21 @@ export function initPacketRelayTables(db) {
 
     CREATE INDEX IF NOT EXISTS idx_packets_to_node
       ON packets(space_id, to_node_id, id);
+    -- Replay admission must be able to reject already-acked packets without
+    -- touching the encrypted payload column.  The old query selected
+    -- packets.* while walking every packet addressed to a node, so a node
+    -- with a large, fully-acked history could read gigabytes of overflow
+    -- pages before returning an empty batch.
+    CREATE INDEX IF NOT EXISTS idx_packets_unacked_lookup
+      ON packets(space_id, to_node_id, id, expires_at, packet_id);
     CREATE INDEX IF NOT EXISTS idx_packets_channel
       ON packets(space_id, channel_id, stream_id, id);
+    CREATE INDEX IF NOT EXISTS idx_packets_channel_id
+      ON packets(space_id, channel_id, id);
     CREATE INDEX IF NOT EXISTS idx_packets_expires
       ON packets(expires_at);
+    CREATE INDEX IF NOT EXISTS idx_packets_received_at
+      ON packets(received_at);
     CREATE INDEX IF NOT EXISTS idx_retained_space_key
       ON retained_packets(space_id, retained_key);
   `);
@@ -192,24 +203,62 @@ export function listRetainedPackets(db, input) {
 export function listPacketsForSubscription(db, input) {
     const cursor = input.cursor ?? 0;
     const limit = input.limit ?? 500;
-    const rows = statement(db, `SELECT *
-     FROM packets
-     WHERE space_id = ?
-       AND id > ?
-       AND (expires_at IS NULL OR expires_at > ?)
-       AND (
-         to_node_id = ?
-         OR (? IS NOT NULL AND channel_id = ?)
-       )
-       AND (? IS NULL OR stream_id = ?)
-     ORDER BY id ASC
-     LIMIT ?`).all(input.spaceId, cursor, input.now, input.nodeId, input.subscription.channelId ?? null, input.subscription.channelId ?? null, input.subscription.streamId ?? null, input.subscription.streamId ?? null, limit);
-    return rows.map(packetFromRow);
+    const streamId = input.subscription.streamId;
+    const rows = [];
+    // Keep the historical OR semantics, but execute each arm independently so
+    // SQLite can seek through idx_packets_to_node or idx_packets_channel instead
+    // of scanning and sorting every packet in the account.  A packet can match
+    // both arms; the merge below de-duplicates by the stable packets.id cursor.
+    // The node arm is intentionally unconditional. `nodeId` is the authenticated
+    // connection identity and was always part of the old OR predicate, even for
+    // an empty/channel-only subscription. `subscription.toNodeId` is a snapshot
+    // label, not a permission to redirect this reader to another node.
+    {
+        const sql = streamId === undefined
+            ? `SELECT * FROM packets
+         WHERE space_id = ? AND to_node_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`
+            : `SELECT * FROM packets
+         WHERE space_id = ? AND to_node_id = ? AND stream_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`;
+        const values = streamId === undefined
+            ? [input.spaceId, input.nodeId, cursor, input.now, limit]
+            : [input.spaceId, input.nodeId, streamId, cursor, input.now, limit];
+        rows.push(...statement(db, sql).all(...values));
+    }
+    if (input.subscription.channelId !== undefined) {
+        const sql = streamId === undefined
+            ? `SELECT * FROM packets
+         WHERE space_id = ? AND channel_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`
+            : `SELECT * FROM packets
+         WHERE space_id = ? AND channel_id = ? AND stream_id = ? AND id > ?
+           AND (expires_at IS NULL OR expires_at > ?)
+         ORDER BY id ASC LIMIT ?`;
+        const values = streamId === undefined
+            ? [input.spaceId, input.subscription.channelId, cursor, input.now, limit]
+            : [input.spaceId, input.subscription.channelId, streamId, cursor, input.now, limit];
+        rows.push(...statement(db, sql).all(...values));
+    }
+    const uniqueRows = new Map();
+    for (const row of rows)
+        uniqueRows.set(row.id, row);
+    return [...uniqueRows.values()]
+        .sort((left, right) => left.id - right.id)
+        .slice(0, limit)
+        .map(packetFromRow);
 }
 export function listUnackedPacketsForNode(db, input) {
     const cursor = input.cursor ?? 0;
     const limit = input.limit ?? 500;
-    const rows = statement(db, `SELECT packets.*
+    // First find candidate row ids using a covering routing index.  Do not
+    // select packets.* in this anti-join: encrypted_payload is commonly a
+    // multi-kilobyte value stored in overflow pages, and loading it for every
+    // already-acked packet turns an empty replay into a large disk read.
+    const candidateRows = statement(db, `SELECT packets.id
      FROM packets
      LEFT JOIN packet_acks
        ON packet_acks.space_id = packets.space_id
@@ -222,13 +271,42 @@ export function listUnackedPacketsForNode(db, input) {
        AND packet_acks.packet_id IS NULL
      ORDER BY packets.id ASC
      LIMIT ?`).all(input.nodeId, input.spaceId, cursor, input.nodeId, input.now, limit);
+    if (candidateRows.length === 0)
+        return [];
+    // Fetch the encrypted rows only for the bounded set that really needs to
+    // be replayed.  The ids are already ordered, but retain an explicit ORDER
+    // BY so the FIFO/server-sequence contract remains clear to SQLite.
+    const placeholders = candidateRows.map(() => "?").join(", ");
+    const rows = statement(db, `SELECT * FROM packets
+     WHERE space_id = ? AND id IN (${placeholders})
+     ORDER BY id ASC`).all(input.spaceId, ...candidateRows.map((row) => row.id));
     return rows.map(packetFromRow);
 }
 export function prunePackets(db, input) {
     if (input.maxAgeMs <= 0)
         return;
     const oldestReceivedAt = input.now - input.maxAgeMs;
-    statement(db, "DELETE FROM packets WHERE (expires_at IS NOT NULL AND expires_at <= ?) OR received_at <= ?").run(input.now, oldestReceivedAt);
+    // Retention runs on the same serialized storage worker as packet replay.
+    // Bound each pass so a large historical database cannot occupy the worker
+    // past the operation deadline.  The two indexed arms avoid the old OR
+    // predicate's full packets-table scan when there is nothing to prune.
+    const candidates = statement(db, `WITH expired AS (
+       SELECT id FROM packets
+       WHERE expires_at IS NOT NULL AND expires_at <= ?
+       LIMIT 1000
+     ), stale AS (
+       SELECT id FROM packets
+       WHERE received_at <= ?
+       LIMIT 1000
+     )
+     SELECT id
+     FROM (SELECT id FROM expired UNION SELECT id FROM stale)
+     ORDER BY id ASC
+     LIMIT 1000`).all(input.now, oldestReceivedAt);
+    if (candidates.length === 0)
+        return;
+    const placeholders = candidates.map(() => "?").join(", ");
+    statement(db, `DELETE FROM packets WHERE id IN (${placeholders})`).run(...candidates.map((row) => row.id));
 }
 export function maxPacketSeq(db, spaceId) {
     const row = statement(db, "SELECT COALESCE(MAX(id), 0) AS seq FROM packets WHERE space_id = ?").get(spaceId);

package/apps/relay/dist/storage-worker.js

@@ -13,6 +13,7 @@ import { ackPacket, getPacketById, getRetainedPacket, insertPacket, listPacketsF
 if (parentPort && workerData?.relayStorageWorker === true) {
     const dbPath = String(workerData.dbPath);
     const testStorageDelayMs = Math.max(0, Number(workerData.testStorageDelayMs) || 0);
+    const testStorageDelayFile = typeof workerData.testStorageDelayFile === "string" ? workerData.testStorageDelayFile : undefined;
     mkdirSync(path.dirname(dbPath), { recursive: true });
     const db = new DatabaseSync(dbPath);
     db.exec("PRAGMA journal_mode = WAL");
@@ -26,8 +27,9 @@ if (parentPort && workerData?.relayStorageWorker === true) {
     parentPort.postMessage({ type: "ready" });
     parentPort.on("message", (request) => {
         try {
-            if (testStorageDelayMs > 0)
+            if (testStorageDelayMs > 0 && (!testStorageDelayFile || existsSync(testStorageDelayFile))) {
                 Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, testStorageDelayMs);
+            }
             const value = request.op === "controlPlane"
                 ? runControlPlaneAction(controlPlane, request.action, request.input)
                 : request.op === "directTicketCreate"
@@ -252,6 +254,7 @@ export function createRelayDurableStorage(dbPath) {
                 dbPath,
                 pairingTtlMs: 600_000,
                 testStorageDelayMs: process.env.HAPPY_ELVES_TEST_STORAGE_DELAY_MS,
+                testStorageDelayFile: process.env.HAPPY_ELVES_TEST_STORAGE_DELAY_FILE,
             },
         });
         nextWorker.unref();

package/apps/relay/dist/websocket.js

@@ -158,9 +158,12 @@ export function registerWebsocketRoute(app, context, originAllowed) {
                         });
                         await sendSnapshot(context, openedConnection, message.snapshotMode ?? "full");
                         broadcastPresence(context, openedConnection.spaceId, await presenceWithManifest(context, openedConnection.spaceId, presence));
-                        void replayNodePackets(context, openedConnection, message.cursor);
+                        void replayNodePackets(context, openedConnection, message.cursor).catch((error) => {
+                            reportDetachedOperationFailure(openedConnection, error);
+                        });
                     }).catch((error) => {
                         app.log.warn({ error }, "failed to persist relay presence");
+                        reportDetachedOperationFailure(openedConnection, error, true);
                     });
                 }).catch((error) => {
                     fail(error instanceof Error ? error.message : "Websocket authentication failed", "AUTH_TOKEN_INVALID");
@@ -210,10 +213,15 @@ export function registerWebsocketRoute(app, context, originAllowed) {
             if (!connection)
                 return;
             const closedConnection = connection;
+            closedConnection.closed = true;
             closedConnection.publishChainClosed = true;
             if (closedConnection.expiryTimer)
                 globalThis.clearTimeout(closedConnection.expiryTimer);
-            void flushPendingCursor(context, closedConnection);
+            if (closedConnection.cursorFlushTimer) {
+                globalThis.clearTimeout(closedConnection.cursorFlushTimer);
+                closedConnection.cursorFlushTimer = undefined;
+            }
+            void flushPendingCursor(context, closedConnection).catch(() => undefined);
             unregisterConnection(closedConnection);
             const remaining = connectionsByNode.get(nodeKey(closedConnection.spaceId, closedConnection.nodeId));
             const stillOnline = Boolean(remaining && remaining.size > 0);
@@ -544,7 +552,9 @@ function pumpSubscriptionReplays(context, connection) {
         if (!state || state.running)
             continue;
         state.queued = false;
-        void runSubscriptionReplay(context, connection, key, state);
+        void runSubscriptionReplay(context, connection, key, state).catch((error) => {
+            reportDetachedOperationFailure(connection, error);
+        });
     }
 }
 async function runSubscriptionReplay(context, connection, key, state) {
@@ -613,7 +623,9 @@ function deliverPacket(context, packet) {
     if (packet.toNodeId) {
         const connections = connectionsByNode.get(nodeKey(packet.spaceId, packet.toNodeId));
         for (const connection of connections ?? []) {
-            void sendPacket(context, connection, packet, "live");
+            void sendPacket(context, connection, packet, "live").catch((error) => {
+                reportDetachedOperationFailure(connection, error);
+            });
             delivered.add(connection);
         }
     }
@@ -623,7 +635,9 @@ function deliverPacket(context, packet) {
             continue;
         if (!connection.subscriptions.some((subscription) => subscriptionMatches(connection, subscription, packet)))
             continue;
-        void sendPacket(context, connection, packet, "live");
+        void sendPacket(context, connection, packet, "live").catch((error) => {
+            reportDetachedOperationFailure(connection, error);
+        });
     }
 }
 async function sendPacket(context, connection, packet, source) {
@@ -666,6 +680,8 @@ async function sendPacket(context, connection, packet, source) {
     return true;
 }
 function queueCursorUpdate(context, connection, serverSeq) {
+    if (connection.closed)
+        return;
     connection.pendingCursorServerSeq = Math.max(connection.pendingCursorServerSeq ?? 0, serverSeq);
     if (connection.cursorFlushTimer)
         return;
@@ -674,13 +690,35 @@ function queueCursorUpdate(context, connection, serverSeq) {
     // keeps high-rate replay from turning every packet into an fsync.
     connection.cursorFlushTimer = setTimeout(() => {
         connection.cursorFlushTimer = undefined;
-        void flushPendingCursor(context, connection);
+        void flushPendingCursor(context, connection).catch(() => undefined);
     }, 100);
     connection.cursorFlushTimer.unref?.();
 }
 async function flushPendingCursor(context, connection) {
+    if (connection.cursorFlushInFlight)
+        return await connection.cursorFlushInFlight;
+    const run = flushPendingCursorOnce(context, connection);
+    let trackedRun;
+    trackedRun = run.finally(() => {
+        if (connection.cursorFlushInFlight === trackedRun)
+            connection.cursorFlushInFlight = undefined;
+        // A newer packet can arrive while the write is in flight. Keep one
+        // coalesced follow-up timer, but never resurrect retries after the
+        // socket has closed.
+        if (!connection.closed && connection.pendingCursorServerSeq !== undefined && !connection.cursorFlushTimer) {
+            connection.cursorFlushTimer = setTimeout(() => {
+                connection.cursorFlushTimer = undefined;
+                void flushPendingCursor(context, connection).catch(() => undefined);
+            }, 250);
+            connection.cursorFlushTimer.unref?.();
+        }
+    });
+    connection.cursorFlushInFlight = trackedRun;
+    await trackedRun;
+}
+async function flushPendingCursorOnce(context, connection) {
     if (connection.cursorFlushTimer) {
-        clearTimeout(connection.cursorFlushTimer);
+        globalThis.clearTimeout(connection.cursorFlushTimer);
         connection.cursorFlushTimer = undefined;
     }
     const serverSeq = connection.pendingCursorServerSeq;
@@ -700,7 +738,8 @@ async function flushPendingCursor(context, connection) {
         connection.pendingCursorServerSeq = Math.max(connection.pendingCursorServerSeq ?? 0, serverSeq);
         // Cursor persistence is recoverable; do not terminate the relay process
         // because a worker/database write was temporarily unavailable.
-        setTimeout(() => { void flushPendingCursor(context, connection); }, 250).unref?.();
+        // The single-flight finalizer above owns the one retry timer. In
+        // particular, a close cannot leave an unbounded timer chain behind.
     }
 }
 function closeSlowReceiver(connection) {
@@ -874,7 +913,7 @@ function scheduleConnectionExpiry(context, connection, expiresAt) {
                 if (!current || current.revoked_at)
                     return;
                 if (current.expires_at && current.expires_at > context.now()) {
-                    void scheduleNext();
+                    void scheduleNext().catch(() => undefined);
                     return;
                 }
                 if (!current.expires_at)
@@ -888,7 +927,33 @@ function scheduleConnectionExpiry(context, connection, expiresAt) {
             }).catch(() => undefined);
         }, Math.min(delay, MAX_EXPIRY_TIMER_DELAY_MS));
     };
-    void scheduleNext();
+    void scheduleNext().catch(() => undefined);
+}
+function reportDetachedOperationFailure(connection, error, forceClose = false) {
+    if (connection.socket.readyState !== 1)
+        return;
+    const record = error && typeof error === "object" ? error : {};
+    const code = typeof record.code === "string" ? record.code : "RELAY_COMMAND_FAILED";
+    const message = typeof record.message === "string" ? record.message : String(error);
+    send(connection.socket, { type: "server:error", code, message });
+    // A detached replay/live send has already lost its in-memory continuation.
+    // Close the stream for storage-generation failures so old daemons and
+    // controllers, which only reconnect on close, replay their durable outbox
+    // and retained packets. The stored ACK boundary remains authoritative; no
+    // packet is acknowledged merely because the connection is being closed.
+    if (forceClose || isRecoverableStorageFailure(code)) {
+        connection.closed = true;
+        try {
+            connection.socket.close(1011, code);
+        }
+        catch {
+            // The close/error listener still performs cleanup for transports that
+            // reject a redundant close.
+        }
+    }
+}
+function isRecoverableStorageFailure(code) {
+    return code === "RELAY_STORAGE_TIMEOUT" || code === "RELAY_STORAGE_UNAVAILABLE" || code === "RELAY_STORAGE_CLOSED";
 }
 async function isScopedControllerConnection(context, connection) {
     const token = await context.tokenRecordAsync(connection.authToken);

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@cydm/happy-elves",
-  "version": "0.1.0-beta.491",
+  "version": "0.1.0-beta.494",
   "description": "Remote controller for local coding agents with hosted or self-hosted relay support.",
   "type": "module",
   "bin": {