@tapflowio/relay 0.27.0 → 0.28.0

package/dist/RelayServer.js

@@ -5,12 +5,13 @@ import os from 'os';
 import path from 'path';
 import { randomUUID } from 'crypto';
 import { WebSocketServer, WebSocket } from 'ws';
-import { SessionManager } from './SessionManager.js';
+import { SessionManager, isAiClientKind } from './SessionManager.js';
+import { gestureAction, keyForWatchers } from './lib/watchActions.js';
 import { directionOf, parseInbound } from '@tapflowio/protocol/validate';
 import { Router, json } from './router.js';
 import { requireViewAuth, requireAuth, getAuth, verifyPat, touchPat, findPat, findUser } from './middleware/auth.js';
-import { AGENT_SCOPE, WS_ACCESS_CHANGED_REASON, WS_AGENT_OWNER_REASON, WS_SCOPE_REASON, classifyConnection, isAffectedBy, revalidatePrincipal } from './lib/connectionAuth.js';
-import { isTunnelIngress, markTunnelIngress, resolveClientAddress, resolveRequestClient } from './lib/clientAddress.js';
+import { AGENT_SCOPE, VIEW_SCOPE, WS_ACCESS_CHANGED_REASON, WS_AGENT_OWNER_REASON, WS_SCOPE_REASON, classifyConnection, isAffectedBy, revalidatePrincipal } from './lib/connectionAuth.js';
+import { isTunnelIngress, markTunnelIngress, proxiedWithoutForwardedFor, resolveClientAddress, resolveRequestClient } from './lib/clientAddress.js';
 import { BuildTicketStore } from './lib/buildTickets.js';
 import { resolveBuildFile } from './lib/buildFiles.js';
 import { resolveCorsHeaders } from './lib/cors.js';
@@ -23,6 +24,7 @@ import { getDb } from './db.js';
 import { handleLogin, handleLogout, handleMe, handleChangePassword, handleInit, handleAuthStatus } from './api/auth.js';
 import { handleVerify, handleAccept } from './api/invitations.js';
 import { createLogger, hasCapability } from '@tapflowio/agent-core';
+import { acquireLoopStallWatch } from '@tapflowio/agent-core/utils';
 import { createKeyframeAwareSender, createRateLimitedDropWarn, sendAudioYieldingToVideo, DEFAULT_BACKPRESSURE_BYTES, hasEnvelope, patchRelayedAt, readEnvelopeFlags, CODEC_JPEG, CODEC_AUDIO, } from '@tapflowio/agent-core/utils';
 const logger = createLogger('relay');
 // True when a remote IP is public — i.e. not loopback, private LAN, or link-local. Used to pick the
@@ -88,6 +90,8 @@ import { handleListTokens, handleCreateToken, handleRevokeToken } from './api/to
 import { handleGetSettings, handleUpdateSettings } from './api/settings.js';
 import { handleUpdateProfile } from './api/profile.js';
 import { handleUploadRecording, handleListRecordings, handleDownloadRecording, purgeExpiredRecordings } from './api/recordings.js';
+import { handleEndVideo, handleOpenVideo, handleVideoChunk, handleVideoData, handleVideoIndex, purgeRunVideos, releaseStaleVideoReservations, runVideosDirFor, videoSummaries } from './api/runVideos.js';
+import { abandonRunsOf, handleCreateRun, handleFinishRun, handleGetFlowScreenshot, handleGetRun, handleListRuns, handleReportFlow, handleUploadFlowScreenshot, purgeExpiredRuns, runScreenshotsDirFor } from './api/runs.js';
 import { handleListAgents, handleGetAgentResources } from './api/agents.js';
 const MIME_TYPES = {
     '.html': 'text/html',
@@ -209,11 +213,22 @@ export class RelayServer {
     resourceBuffers = new Map();
     logBuffer = [];
     recordingsDir = '';
+    runScreenshotsDir = '';
+    runVideosDir = '';
     purgeRecordingsTimer = null;
     purgeOldResourcesTimer = null;
     purgeBuildsTimer = null;
     flushResourcesTimer = null;
     heartbeatTimer = null;
+    /** When the last heartbeat sweep ran, by the same clock `isAlive` judges with. */
+    lastSweepAt = 0;
+    /** Names agents give at registration, kept apart from the session map so an agent with no sessions
+     *  can still be named when its socket goes. */
+    agentNames = new WeakMap();
+    releaseStallWatch = null;
+    /** Agent sockets whose close was already explained — ended by a sweep, or replaced by the same agent
+     *  registering again — so the close handler does not log them as a fresh drop. */
+    explainedCloses = new WeakMap();
     /**
      * When each socket last answered a ping. WeakMap → no manual cleanup on close (GC handles it).
      *
@@ -251,6 +266,19 @@ export class RelayServer {
     droppers = new Map();
     // Per-session throttled "request an IDR from the agent" callbacks (drop recovery).
     idrRequesters = new Map();
+    /**
+     * **One keyframe-aware sender per watcher, never the session's.** The sender's `dropping` state lives in
+     * its closure, so sharing the holder's would let a slow watcher put the holder into drop-to-keyframe
+     * too — every P-frame to the holder discarded until the next IDR because someone else's link is slow.
+     * Keyed by socket, then session, because one socket can watch several sessions.
+     */
+    watcherDroppers = new Map();
+    /**
+     * Whether this connection may watch: a cookie, or a remote PAT with `view` — decided once at the
+     * handshake, from the credential. **Not** an unauthenticated loopback socket: a page in any browser on
+     * the relay's Mac reaches loopback without a credential, and watching shows a device's screen.
+     */
+    mayWatch = new WeakMap();
     /** Unlike `idrRequesters` these hold a timer, which is why the value is not a bare closure — see
      *  `forgetSessionState`. */
     networkStateRequesters = new Map();
@@ -297,6 +325,7 @@ export class RelayServer {
     corsAllowed;
     // One-shot warning when XFF arrives on a loopback socket but TAPFLOW_TRUSTED_PROXIES is unset.
     warnedProxyMisconfig = false;
+    warnedTrustedProxyWithoutXff = false;
     /** Agent identities already warned about a missing `build-download`. See `warnLegacyInstaller`. */
     warnedLegacyInstaller = new Set();
     buildTickets = new BuildTicketStore();
@@ -367,7 +396,16 @@ export class RelayServer {
         if (options.agentGraceMs === undefined && graceRaw && !graceUsable) {
             logger.warn(`TAPFLOW_AGENT_GRACE_MS="${graceRaw}" is not a usable number of milliseconds — using ${DEFAULT_AGENT_GRACE_MS}`);
         }
-        this.sessions = new SessionManager({ idleTimeoutMs: options.idleTimeoutMs });
+        this.sessions = new SessionManager({
+            idleTimeoutMs: options.idleTimeoutMs,
+            // Before the sockets are forgotten, so each one is told why its picture stopped.
+            onWatchersEvicted: (sessionId, sockets, reason) => {
+                for (const w of sockets) {
+                    this.forgetWatcherDropper(w, sessionId);
+                    this.sendTo(w, { type: 'watch:ended', sessionId, reason });
+                }
+            },
+        });
         this.publicDir = options.publicDir ?? path.join(import.meta.dirname, '../public');
         this.uploadsDir = options.uploadsDir ?? path.join(import.meta.dirname, '../uploads');
         this.router = new Router();
@@ -420,7 +458,7 @@ export class RelayServer {
         this.router.get('/api/v1/apps', handleListApps);
         this.router.post('/api/v1/apps', handleCreateApp);
         this.router.patch('/api/v1/apps/:id', handleUpdateApp);
-        this.router.delete('/api/v1/apps/:id', handleDeleteApp);
+        this.router.delete('/api/v1/apps/:id', (req, res, params) => handleDeleteApp(req, res, params, this.buildFileDirs()));
         // builds
         this.router.get('/api/v1/builds', handleListBuilds);
         this.router.get('/api/v1/builds/:id', handleGetBuild);
@@ -454,7 +492,7 @@ export class RelayServer {
         this.router.post('/api/v1/tokens', handleCreateToken);
         this.router.delete('/api/v1/tokens/:id', (req, res, params) => handleRevokeToken(req, res, params, this.onAuthChanged));
         // settings
-        this.router.get('/api/v1/settings', handleGetSettings);
+        this.router.get('/api/v1/settings', (req, res) => handleGetSettings(req, res, this.runVideosDir));
         this.router.patch('/api/v1/settings', (req, res) => handleUpdateSettings(req, res, u));
         this.router.patch('/api/v1/profile', (req, res) => handleUpdateProfile(req, res, u));
         // recordings
@@ -462,6 +500,22 @@ export class RelayServer {
         this.router.post('/api/v1/recordings/upload', (req, res) => handleUploadRecording(req, res, this.recordingsDir));
         this.router.get('/api/v1/recordings', (req, res) => handleListRecordings(req, res));
         this.router.get('/api/v1/recordings/:filename', (req, res) => handleDownloadRecording(req, res, this.recordingsDir));
+        // flow run records — written by `tapflow flow run`, read by the runs page
+        this.runScreenshotsDir = runScreenshotsDirFor(u);
+        this.router.post('/api/v1/runs', handleCreateRun);
+        this.router.get('/api/v1/runs', (req, res) => handleListRuns(req, res, this.runHolders));
+        this.router.get('/api/v1/runs/:id', (req, res, params) => handleGetRun(req, res, params, this.runHolders, videoSummaries));
+        this.router.post('/api/v1/runs/:id/finish', handleFinishRun);
+        this.router.post('/api/v1/runs/:id/flows/:idx', handleReportFlow);
+        this.router.post('/api/v1/runs/:id/flows/:idx/screenshot', (req, res, params) => handleUploadFlowScreenshot(req, res, params, this.runScreenshotsDir));
+        this.router.get('/api/v1/runs/:id/flows/:idx/screenshot', (req, res, params) => handleGetFlowScreenshot(req, res, params, this.runScreenshotsDir));
+        // a flow's video (A4) — uploaded by the runner while the flow runs, kept by the retention policy
+        this.runVideosDir = runVideosDirFor(u);
+        this.router.post('/api/v1/runs/:id/flows/:idx/video/open', (req, res, params) => handleOpenVideo(req, res, params, this.runVideosDir));
+        this.router.post('/api/v1/runs/:id/flows/:idx/video/chunks/:seq', (req, res, params) => handleVideoChunk(req, res, params, this.runVideosDir, this.runHolders));
+        this.router.post('/api/v1/runs/:id/flows/:idx/video/end', handleEndVideo);
+        this.router.get('/api/v1/runs/:id/flows/:idx/video/index', (req, res, params) => handleVideoIndex(req, res, params, this.runHolders));
+        this.router.get('/api/v1/runs/:id/flows/:idx/video', (req, res, params) => handleVideoData(req, res, params, this.runHolders, this.runVideosDir));
         // logs — the relay host only. The buffer holds the addresses of refused connections, nothing
         // remote reads it (`tapflow logs` is its only consumer), and every line is also in the relay's own
         // output. Same locality rule and proxy list as `auth/init` and the WebSocket handshake, so the tunnel
@@ -521,13 +575,31 @@ export class RelayServer {
         purgeOldResources();
         this.purgeOldResourcesTimer = setInterval(purgeOldResources, 24 * 60 * 60 * 1000);
         this.purgeOldResourcesTimer.unref();
-        purgeExpiredBuilds(this.recordingsDir, this.uploadsDir);
-        this.purgeBuildsTimer = setInterval(() => purgeExpiredBuilds(this.recordingsDir, this.uploadsDir), 24 * 60 * 60 * 1000);
+        // Runs ride the builds' daily tick rather than a timer of their own: a build's runs go with it, and the
+        // runs left over (no build) expire on the same clock.
+        const purgeBuildsAndRuns = () => {
+            purgeExpiredBuilds(this.buildFileDirs());
+            purgeExpiredRuns(this.runScreenshotsDir, this.runVideosDir, this.runHolders);
+            purgeRunVideos(this.runVideosDir, this.runHolders);
+        };
+        // Before the first purge or write: no write in flight survived the last process.
+        releaseStaleVideoReservations();
+        purgeBuildsAndRuns();
+        this.purgeBuildsTimer = setInterval(purgeBuildsAndRuns, 24 * 60 * 60 * 1000);
         this.purgeBuildsTimer.unref();
         this.flushResourcesTimer = setInterval(() => this.flushResourceBuffers(), 60_000);
         this.flushResourcesTimer.unref();
-        this.heartbeatTimer = setInterval(() => this.runHeartbeat(), HEARTBEAT_MS);
+        this.lastSweepAt = Date.now();
+        this.heartbeatTimer = setInterval(() => { this.noteLateSweep(); this.runHeartbeat(); }, HEARTBEAT_MS);
         this.heartbeatTimer.unref();
+        this.releaseStallWatch ??= acquireLoopStallWatch();
+        // The rule a proxy has to follow, said once to whoever set the list: the case it covers (a proxy that
+        // sends no forwarding header at all) is one no request-time check can see.
+        const trustedProxies = this.options.trustedProxies ?? [];
+        if (trustedProxies.length > 0) {
+            logger.info(`Trusted proxies: ${trustedProxies.join(', ')}. A request from one of these without ` +
+                'X-Forwarded-For is treated as coming from that address itself, so the proxy must send X-Forwarded-For.');
+        }
         return new Promise((resolve, reject) => {
             this.httpServer.once('error', (err) => {
                 if (err.code === 'EADDRINUSE') {
@@ -586,6 +658,8 @@ export class RelayServer {
             clearInterval(this.heartbeatTimer);
             this.heartbeatTimer = null;
         }
+        this.releaseStallWatch?.();
+        this.releaseStallWatch = null;
         // `unref()` would not do: the test runner's process outlives the server, so an un-cleared hold
         // still fires — against a server that has already stopped.
         this.stopping = true;
@@ -664,14 +738,42 @@ export class RelayServer {
     // interval, whatever path the change took. `onAuthChanged` is the same check run early.
     runHeartbeat(clients = this.wss.clients) {
         this.revalidateSockets(clients);
+        // Agents one line each; everything else summed, because a relay waking from sleep finds every socket
+        // stale at once and one line per browser tab would push the agent lines out of the log buffer.
+        const others = new Map();
         for (const ws of clients) {
             if (!this.isAlive(ws)) {
+                const role = this.wsRoles.get(ws);
+                if (role === 'agent') {
+                    this.pushLog(`heartbeat terminated agent ${this.agentNames.get(ws) ?? 'unknown'} — last pong ${this.pongAgeSeconds(ws)}s ago`);
+                    this.explainedCloses.set(ws, 'terminated');
+                }
+                else {
+                    const key = role ?? 'unregistered';
+                    others.set(key, (others.get(key) ?? 0) + 1);
+                }
                 ws.terminate();
                 continue;
             }
             if (ws.readyState === WebSocket.OPEN)
                 ws.ping();
         }
+        if (others.size > 0) {
+            this.pushLog(`heartbeat terminated ${[...others].map(([role, n]) => `${n} ${role}`).join(', ')} socket(s)`);
+        }
+    }
+    /** A sweep that runs late means this process was held, or the machine slept: the pongs it judges were
+     *  never read, so its terminations say nothing about the clients. Same clock as `isAlive`. */
+    noteLateSweep(now = Date.now()) {
+        const late = now - this.lastSweepAt - HEARTBEAT_MS;
+        if (late > HEARTBEAT_MS / 2) {
+            this.pushLog(`heartbeat sweep ran ${Math.round(late / 1000)}s late — the relay was stalled or the system slept; this sweep may end sockets that were alive`);
+        }
+        this.lastSweepAt = now;
+    }
+    pongAgeSeconds(ws) {
+        const at = this.lastPongAt.get(ws);
+        return at === undefined ? '?' : Math.round((Date.now() - at) / 1000);
     }
     /**
      * Injected into the handlers whose write can take access away — member update and removal, token
@@ -767,6 +869,32 @@ export class RelayServer {
      * negative beats 30 with one every cycle**, because a false negative here does not merely delay: it
      * hands a live tester's session to whoever asked next.
      */
+    buildFileDirs() {
+        return { uploadsDir: this.uploadsDir, recordingsDir: this.recordingsDir, runScreenshotsDir: this.runScreenshotsDir, runVideosDir: this.runVideosDir };
+    }
+    /** Open sockets speaking for any of these owner keys. */
+    openSocketsOf(keys) {
+        return [...this.wss.clients].filter((ws) => ws.readyState === WebSocket.OPEN && keys.includes(this.ownerKey.get(ws)?.key ?? ''));
+    }
+    runHolders = {
+        connectedKeys: () => {
+            const keys = new Set();
+            for (const ws of this.wss.clients) {
+                const key = ws.readyState === WebSocket.OPEN ? this.ownerKey.get(ws)?.key : undefined;
+                if (key)
+                    keys.add(key);
+            }
+            return keys;
+        },
+        heldSession: (keys) => {
+            for (const ws of this.openSocketsOf(keys)) {
+                const held = this.sessions.getByBrowserSocket(ws);
+                if (held.length > 0)
+                    return held[0].id;
+            }
+            return null;
+        },
+    };
     isAlive(ws) {
         const at = this.lastPongAt.get(ws);
         // **No entry means just connected, not dead.** `handleConnection` seeds one, but a socket is in
@@ -809,6 +937,7 @@ export class RelayServer {
      */
     forgetSessionState(sessionId) {
         this.dropHandlers.delete(sessionId);
+        this.dropHandlers.delete(`${sessionId} watchers`);
         this.audioDropHandlers.delete(sessionId);
         this.droppers.delete(sessionId);
         this.idrRequesters.delete(sessionId);
@@ -897,6 +1026,24 @@ export class RelayServer {
     remoteAddressOf(request) {
         return request.socket.remoteAddress ?? '';
     }
+    /**
+     * Called on both doors — the WebSocket upgrade and every HTTP request — since `auth/init` and the logs
+     * route decide locality on the HTTP side. Not on the tunnel listener, where every request is remote
+     * whatever it carries.
+     */
+    warnTrustedProxyWithoutXffOnce(request) {
+        if (this.warnedTrustedProxyWithoutXff || isTunnelIngress(request))
+            return;
+        const header = proxiedWithoutForwardedFor(this.remoteAddressOf(request), request.headers, this.options.trustedProxies ?? []);
+        if (!header)
+            return;
+        logger.warn(`A request from a trusted proxy (TAPFLOW_TRUSTED_PROXIES) carried ${header} but no X-Forwarded-For. ` +
+            'Without that header the relay cannot see the real client, so it treats the request as coming from the ' +
+            'proxy itself: local, and so unauthenticated, when that address is loopback. Configure the proxy to ' +
+            'send X-Forwarded-For. A proxy on this host can instead use the tunnel port, which is open only when a ' +
+            'tunnel is configured or TAPFLOW_TUNNEL_PORT names a port.');
+        this.warnedTrustedProxyWithoutXff = true;
+    }
     warnProxyMisconfigOnce(socketAddr, forwardedFor) {
         if (this.warnedProxyMisconfig)
             return;
@@ -906,12 +1053,19 @@ export class RelayServer {
         if (a === '::1' || a.startsWith('127.')) {
             logger.warn('Received X-Forwarded-For from a loopback connection but TAPFLOW_TRUSTED_PROXIES is unset. ' +
                 'If a same-host reverse proxy or `tailscale serve` forwards to this port, every client it forwards is ' +
-                'treated as localhost (unauthenticated). Point it at the tunnel port instead (TAPFLOW_TUNNEL_PORT, ' +
-                'default 4001), or set TAPFLOW_TRUSTED_PROXIES so the real client IP is used.');
+                'treated as localhost (unauthenticated). Point it at the tunnel port instead (open only when a tunnel ' +
+                'is configured or TAPFLOW_TUNNEL_PORT names a port), or set TAPFLOW_TRUSTED_PROXIES so the real ' +
+                'client IP is used.');
             this.warnedProxyMisconfig = true;
         }
     }
     handleConnection(ws, request) {
+        // **First, before anything can return.** A frame that breaks the protocol makes `ws` emit `'error'` on
+        // this socket (a reset it handles itself), and an EventEmitter with no `'error'` listener throws: from
+        // inside `ws`'s receiver, with nothing above it to catch. One client could end the relay for everyone,
+        // signed in or not. `ws` closes the socket itself after emitting, so the `'close'` handler below still runs.
+        // Debug, not warn: any client can produce this at will, and the close is already logged where it matters.
+        ws.on('error', (err) => logger.debug(`WebSocket error: ${err.message}`));
         const socketAddr = this.remoteAddressOf(request);
         const xff = request.headers['x-forwarded-for'];
         const forwardedFor = Array.isArray(xff) ? xff[0] : xff;
@@ -919,6 +1073,7 @@ export class RelayServer {
         // A proxy in front of the tunnel port adds the header too, and there it changes nothing about auth.
         if (!viaTunnel)
             this.warnProxyMisconfigOnce(socketAddr, forwardedFor);
+        this.warnTrustedProxyWithoutXffOnce(request);
         const { addr, isLocal } = resolveClientAddress({
             socketAddr,
             forwardedFor,
@@ -952,6 +1107,10 @@ export class RelayServer {
             if (pat)
                 touchPat(pat.patId);
             userId = cookie?.userId ?? pat?.userId;
+            // A loopback cookie is not re-judged later — `principals` below records remote sockets only — so a
+            // watch on loopback lasts until the socket closes even if that user is removed. The same is true of
+            // everything else such a socket does; watching adds nothing to it.
+            this.mayWatch.set(ws, cookie !== null || (!isLocal && pat !== null && pat.scopes.includes(VIEW_SCOPE)));
             if (!isLocal) {
                 if (cookie)
                     this.principals.set(ws, { via: 'cookie', userId: cookie.userId, jwtExp: cookie.exp, pwv: cookie.pwv });
@@ -994,43 +1153,73 @@ export class RelayServer {
             if (this.revokedSockets.has(ws))
                 return;
             if (isBinary) {
-                // Binary frames arrive on the dedicated stream WS, route to the session's browser
+                // Binary frames arrive on the dedicated stream WS, route to the session's browser and watchers.
                 const session = this.sessions.getByStreamSocket(ws);
-                if (session?.browserSocket) {
-                    const frameBuf = data;
-                    // Audio rides the same socket (codec-tagged). Route it through a sender that YIELDS to
-                    // video — it drops audio unless the socket is near-empty, so audio never inflates
-                    // bufferedAmount enough to trip the video backpressure path. Must branch before the
-                    // video dropper, which would (wrongly) treat audio as a droppable P-frame.
-                    if (hasEnvelope(frameBuf) && readEnvelopeFlags(frameBuf).codec === CODEC_AUDIO) {
-                        patchRelayedAt(frameBuf, Date.now());
-                        let onAudioDrop = this.audioDropHandlers.get(session.id);
-                        if (!onAudioDrop) {
-                            onAudioDrop = createRateLimitedDropWarn(logger, `${session.id} audio`);
-                            this.audioDropHandlers.set(session.id, onAudioDrop);
-                        }
-                        sendAudioYieldingToVideo(session.browserSocket, frameBuf, onAudioDrop);
+                if (!session)
+                    return;
+                // **Not gated on the holder.** Watchers keep receiving frames while the holder is away inside its
+                // reconnect grace, which is when `browserSocket` is null.
+                const watchers = this.sessions.watchersOf(session.id);
+                // A holder that said it does not read the stream (`stream: false`) is sent nothing: the MCP server and a
+                // runner that is not recording discard every frame, and an iOS session is ~34 Mbit/s of JPEG to do it with.
+                const holder = session.holderStream ? session.browserSocket : null;
+                if (!holder && watchers.length === 0)
+                    return;
+                const frameBuf = data;
+                // Audio rides the same socket (codec-tagged). Route it through a sender that YIELDS to
+                // video — it drops audio unless the socket is near-empty, so audio never inflates
+                // bufferedAmount enough to trip the video backpressure path. Must branch before the
+                // video dropper, which would (wrongly) treat audio as a droppable P-frame.
+                //
+                // **The holder's alone.** A watch page has no audio control, and audio a teammate did not ask
+                // to hear should not start playing in their tab.
+                if (hasEnvelope(frameBuf) && readEnvelopeFlags(frameBuf).codec === CODEC_AUDIO) {
+                    if (!holder)
                         return;
+                    patchRelayedAt(frameBuf, Date.now());
+                    let onAudioDrop = this.audioDropHandlers.get(session.id);
+                    if (!onAudioDrop) {
+                        onAudioDrop = createRateLimitedDropWarn(logger, `${session.id} audio`);
+                        this.audioDropHandlers.set(session.id, onAudioDrop);
                     }
-                    let onDrop = this.dropHandlers.get(session.id);
-                    if (!onDrop) {
-                        onDrop = createRateLimitedDropWarn(logger, session.id);
-                        this.dropHandlers.set(session.id, onDrop);
-                    }
+                    sendAudioYieldingToVideo(holder, frameBuf, onAudioDrop);
+                    return;
+                }
+                let onDrop = this.dropHandlers.get(session.id);
+                if (!onDrop) {
+                    onDrop = createRateLimitedDropWarn(logger, session.id);
+                    this.dropHandlers.set(session.id, onDrop);
+                }
+                const requestIdr = this.idrRequester(session.id);
+                // JPEG and H.264 IDRs are resync points; only P-frames must wait for a keyframe after a drop.
+                // Patched once: every recipient is sent the same buffer, so it carries one relay timestamp.
+                let isKeyframe = true;
+                if (hasEnvelope(frameBuf)) {
+                    patchRelayedAt(frameBuf, Date.now());
+                    const flags = readEnvelopeFlags(frameBuf);
+                    isKeyframe = flags.codec === CODEC_JPEG || flags.keyframe;
+                }
+                if (holder) {
                     let dropper = this.droppers.get(session.id);
                     if (!dropper) {
                         dropper = createKeyframeAwareSender();
                         this.droppers.set(session.id, dropper);
                     }
-                    const requestIdr = this.idrRequester(session.id);
-                    // JPEG and H.264 IDRs are resync points; only P-frames must wait for a keyframe after a drop.
-                    let isKeyframe = true;
-                    if (hasEnvelope(frameBuf)) {
-                        patchRelayedAt(frameBuf, Date.now());
-                        const flags = readEnvelopeFlags(frameBuf);
-                        isKeyframe = flags.codec === CODEC_JPEG || flags.keyframe;
+                    dropper.send(holder, frameBuf, this.backpressureBytes, isKeyframe, onDrop, requestIdr);
+                }
+                // **No IDR request on a watcher's drop**, and its own drop warning. An IDR changes the encoder's
+                // output for everyone, so a watcher on a slow link asking for one on every drain would cost the
+                // holder up to two forced keyframes a second — the slow-watcher-hurts-holder case the separate
+                // sender exists to prevent, by another route. A watcher resyncs on the next periodic keyframe.
+                if (watchers.length > 0) {
+                    let onWatcherDrop = this.dropHandlers.get(`${session.id} watchers`);
+                    if (!onWatcherDrop) {
+                        onWatcherDrop = createRateLimitedDropWarn(logger, `${session.id} watchers`);
+                        this.dropHandlers.set(`${session.id} watchers`, onWatcherDrop);
+                    }
+                    for (const w of watchers) {
+                        this.watcherDropper(w, session.id).send(w, frameBuf, this.backpressureBytes, isKeyframe, onWatcherDrop);
                     }
-                    dropper.send(session.browserSocket, frameBuf, this.backpressureBytes, isKeyframe, onDrop, requestIdr);
                 }
                 return;
             }
@@ -1085,7 +1274,19 @@ export class RelayServer {
                 logger.error(`route failed for ${inbound.msg.type}:`, e);
             }
         });
-        ws.on('close', () => {
+        ws.on('close', (code, reason) => {
+            // Not while stopping: the relay ending every socket itself says nothing about the agents.
+            if (this.wsRoles.get(ws) === 'agent' && !this.stopping) {
+                const name = this.agentNames.get(ws) ?? 'unknown';
+                const explained = this.explainedCloses.get(ws);
+                if (explained === 'replaced') {
+                    this.pushLog(`agent socket replaced: ${name} registered again on a new connection`);
+                }
+                else if (!explained) {
+                    const why = reason.length > 0 ? ` ${reason.toString()}` : '';
+                    this.pushLog(`agent socket closed: ${name} (code ${code}${why}, last pong ${this.pongAgeSeconds(ws)}s ago)`);
+                }
+            }
             this.wsRoles.delete(ws);
             this.wsExternal.delete(ws);
             this.forgetShutdownRequesters(ws);
@@ -1102,6 +1303,9 @@ export class RelayServer {
             const streamSession = this.sessions.getByStreamSocket(ws);
             if (streamSession)
                 this.sessions.clearStreamSocket(streamSession.id);
+            // A watcher leaving changes nothing for the session it watched — no release, no timer, no IDR.
+            this.sessions.removeWatcherSocket(ws);
+            this.watcherDroppers.delete(ws);
             // Browser socket disconnected → release **every** session it held, each with its own idle timer.
             //
             // `getByBrowserSocket` used to answer with one session because the index held one, and a socket
@@ -1116,6 +1320,20 @@ export class RelayServer {
                 return;
             for (const held of this.sessions.getByBrowserSocket(ws)) {
                 this.sessions.clearBrowser(held.id, () => this.idleShutdown(held.id));
+                this.tellWatchers(held.id, { type: 'watch:holder-left', sessionId: held.id });
+            }
+            // A run is its runner's socket: the flow runner does not reconnect, so the last socket with its key
+            // closing is the end of that run, finished or not. Not done while stopping (above) — a relay going
+            // down has not seen the runner go, and the runner may still finish the run against the next one.
+            //
+            // Caught because this is an event listener: a database fault here (a full disk) would be uncaught and end
+            // the relay — the rule `handleConnection` states for its own reads. A run missed here is closed by
+            // `settleOrphanedRuns` on the next read.
+            try {
+                abandonRunsOf(this.ownerOf(ws), this.runHolders);
+            }
+            catch (e) {
+                logger.error('could not close the runs of a disconnected runner:', e);
             }
         });
     }
@@ -1284,6 +1502,26 @@ export class RelayServer {
             case 'session:start':
                 this.handleSessionStart(ws, msg);
                 break;
+            case 'session:activity': {
+                // A holder's name for what it is about to do (A2), for its watchers only. Unanswered either way, so an
+                // unowned one is simply dropped; the schema already capped its length. **Its content is trusted, not
+                // checked**: the rule that it carries nothing typed binds the sender (`activityLabel` is the only producer),
+                // and a holder that broke it could show no more than it already can on the screen its watchers see.
+                const session = this.sessions.get(msg.sessionId);
+                const label = msg.label.trim();
+                if (session && label && this.ownsSession(ws, session))
+                    this.tellWatchersAction(session.id, { kind: 'label', label });
+                break;
+            }
+            case 'watch:start':
+                this.handleWatchStart(ws, msg);
+                break;
+            // Unanswered, like `session:leave`: nobody waits on it, and closing the socket does the same.
+            case 'watch:stop': {
+                if (this.sessions.removeWatcher(msg.sessionId, ws))
+                    this.forgetWatcherDropper(ws, msg.sessionId);
+                break;
+            }
             // ── the two session commands, gated but not answered ───────────────────────────────────────
             //
             // Both destroy state a viewer depends on, and until L5c both acted on the strength of the session
@@ -1317,6 +1555,10 @@ export class RelayServer {
                 if (this.ownsSession(ws, this.sessions.get(msg.sessionId))) {
                     this.sessions.clearBrowser(msg.sessionId);
                     this.forgetSessionState(msg.sessionId);
+                    // **A leave is the end, not a blip.** The holder said it is done — a flow run finishing, an MCP
+                    // `disconnect_device` — so the watch ends. `watch:holder-left` is for a socket that dropped, which
+                    // may come back; sent here it left a watcher of every finished CI run waiting forever.
+                    this.sessions.endWatches(msg.sessionId, 'session-ended');
                 }
                 break;
             }
@@ -1352,6 +1594,7 @@ export class RelayServer {
                 if (session.browserSocket?.readyState === WebSocket.OPEN) {
                     session.browserSocket.send(JSON.stringify(raw));
                 }
+                this.forwardToWatchers(session.id, raw);
                 break;
             }
             case 'device:postures': {
@@ -1366,6 +1609,7 @@ export class RelayServer {
                 if (session.browserSocket?.readyState === WebSocket.OPEN) {
                     session.browserSocket.send(JSON.stringify(raw));
                 }
+                this.forwardToWatchers(session.id, raw);
                 break;
             }
             case 'session:deviceInfo': {
@@ -1377,6 +1621,7 @@ export class RelayServer {
                 if (session.browserSocket?.readyState === WebSocket.OPEN) {
                     session.browserSocket.send(JSON.stringify(raw));
                 }
+                this.forwardToWatchers(session.id, raw);
                 break;
             }
             case 'device:booting': {
@@ -1388,6 +1633,7 @@ export class RelayServer {
                 if (session.browserSocket?.readyState === WebSocket.OPEN) {
                     session.browserSocket.send(JSON.stringify(raw));
                 }
+                this.forwardToWatchers(session.id, raw);
                 break;
             }
             case 'device:boot-error': {
@@ -1441,6 +1687,7 @@ export class RelayServer {
                 if (session.browserSocket?.readyState === WebSocket.OPEN) {
                     session.browserSocket.send(JSON.stringify(raw));
                 }
+                this.forwardToWatchers(session.id, raw);
                 break;
             }
             case 'app:install-done':
@@ -1525,6 +1772,7 @@ export class RelayServer {
                 // presence check the old line carried is gone because the schema requires the payload — and
                 // `external` is deliberately not declared on `DeviceBoot`, since the browser never sends it.
                 msg.payload['external'] = this.wsExternal.get(ws) ?? false;
+                msg.payload['automation'] = isAiClientKind(boot.session.holderKind);
                 // Tracked by the socket it actually goes to (#885): a rebind invalidates only the boots
                 // tied to the replaced socket, so a boot dispatched after the rebind stays answerable.
                 this.rememberBoot(msg.sessionId, msg.requestId, boot.session.agentSocket);
@@ -1547,6 +1795,18 @@ export class RelayServer {
                 // command the relay never answers. `mcp-server`'s `shutdownDevice` waits 30s on `device:shutdown-done`
                 // and then reports `Request timed out` with no cause, which is the silence, not a diagnosis.
                 const session = this.sessions.get(msg.sessionId);
+                // A watcher's gate comes first: `mayShutDown` passes anyone while the session is unheld, and the
+                // holder's reconnect grace is exactly that — so a watch page closing at that moment would power
+                // off the device an agent is in the middle of testing.
+                if (session && this.sessions.isWatching(ws, session.id)) {
+                    this.sendTo(ws, {
+                        type: 'device:shutdown-error',
+                        sessionId: msg.sessionId,
+                        ...(msg.requestId === undefined ? {} : { requestId: msg.requestId }),
+                        message: 'This connection is watching the session and cannot shut its device down.',
+                    });
+                    break;
+                }
                 if (session && !this.mayShutDown(ws, session)) {
                     this.sendTo(ws, {
                         type: 'device:shutdown-error',
@@ -1605,6 +1865,8 @@ export class RelayServer {
                 const target = this.dispatchTarget(ws, msg.sessionId);
                 if (target.ok) {
                     target.session.agentSocket.send(JSON.stringify(msg));
+                    // No URL: a deep link can carry a token, and watchers are the whole team.
+                    this.tellWatchersAction(msg.sessionId, { kind: 'open-url' });
                 }
                 else {
                     this.sendTo(ws, {
@@ -1622,6 +1884,7 @@ export class RelayServer {
                 const target = this.dispatchTarget(ws, msg.sessionId);
                 if (target.ok) {
                     target.session.agentSocket.send(JSON.stringify(msg));
+                    this.tellWatchersAction(msg.sessionId, { kind: 'clear-state' });
                 }
                 else {
                     this.sendTo(ws, {
@@ -1920,6 +2183,7 @@ export class RelayServer {
         for (const s of sessions) {
             if (s.browserSocket)
                 this.sendTo(s.browserSocket, { type: 'session:agent-away', sessionId: s.id });
+            this.tellWatchers(s.id, { type: 'session:agent-away', sessionId: s.id });
         }
         const timer = setTimeout(() => {
             this.agentHolds.delete(ws);
@@ -2040,6 +2304,7 @@ export class RelayServer {
                 // gone and its in-flight screenshots would be undiscoverable — reject them here instead.
                 // The rebound sessions have already moved off `old`, so this no longer covers them.
                 this.evictAgentSocket(old, 'replaced');
+                this.explainedCloses.set(old, 'replaced');
                 old.terminate();
             }
         }
@@ -2084,6 +2349,7 @@ export class RelayServer {
             if (s?.browserSocket) {
                 this.sendTo(s.browserSocket, { type: 'session:rebound', sessionId, capabilities: msg.capabilities ?? [] });
             }
+            this.tellWatchers(sessionId, { type: 'session:rebound', sessionId, capabilities: msg.capabilities ?? [] });
         }
         // After the rebound frames, so the client records `needsReboot` before the error below
         // arrives and classifies it with the cause attached. Only boots tied to the replaced
@@ -2097,7 +2363,9 @@ export class RelayServer {
         // transition, matching the disconnect line in evictAgentSocket.
         // `||`, not `??`: the schema defaults both of these to `''` for an agent that omits them, so `??`
         // would print an empty name and an empty platform where this used to print `unknown`.
-        logger.info(`agent connected: ${msg.agentName || msg.agentId || 'unknown'} (${msg.platform || 'unknown'}) — ${registeredSessions.length} device(s)`);
+        this.agentNames.set(ws, msg.agentName || msg.agentId || 'unknown');
+        // Through `pushLog` too, so `tapflow logs` shows an agent coming back after the line about it leaving.
+        this.pushLog(`agent connected: ${msg.agentName || msg.agentId || 'unknown'} (${msg.platform || 'unknown'}) — ${registeredSessions.length} device(s)`);
     }
     /** The only producer of `error` — all five exits below, and nothing else in the repo sends that message.
      *
@@ -2112,6 +2380,17 @@ export class RelayServer {
             this.sendTo(ws, { type: 'error', sessionId: msg.sessionId, message: 'Session not found', reason: 'session-not-found' });
             return;
         }
+        // **A watcher may not take the session it watches.** Inside the holder's reconnect grace the session
+        // is unheld, so without this a watch page that re-sent `session:start` — a reused viewer path, a
+        // reconnect — would become the holder, and the agent's own re-join would then be refused as busy.
+        // `session-busy` because that is what the caller must act on: someone else's session, in use.
+        if (this.sessions.isWatching(ws, session.id)) {
+            this.sendTo(ws, {
+                type: 'error', sessionId: msg.sessionId, reason: 'session-busy',
+                message: 'This connection is watching the session. Stop watching to join it.',
+            });
+            return;
+        }
         // Occupancy first, because it is the more specific answer and both can be true at once. A tester
         // joining a device someone else has open on a loaded Mac was being told "this Mac is overloaded,
         // pick another" — advice for a problem they cannot act on, while the actual reason went unreported.
@@ -2143,7 +2422,9 @@ export class RelayServer {
             }
         }
         try {
-            const joined = this.sessions.join(msg.sessionId, ws, this.ownerRecord(ws), (h) => this.isAlive(h));
+            const joined = this.sessions.join(msg.sessionId, ws, this.ownerRecord(ws), (h) => this.isAlive(h), 
+            // Recorded by the join itself, so it holds before the agent-away return below and across a rebind.
+            { kind: msg.clientKind, label: this.holderLabel(ws), ...(msg.stream === false ? { stream: false } : {}) });
             if (!joined.ok) {
                 // Both of these are answered above, so arriving here means the state moved between that check and
                 // this call. They are **values now rather than throws** (#515), and that is what fixes the defect:
@@ -2174,6 +2455,7 @@ export class RelayServer {
         // deliberately instead of inferring anything from a timeout.
         this.sendTo(ws, {
             type: 'session:joined', sessionId: msg.sessionId, capabilities: session.agentCapabilities ?? [],
+            ...(isAiClientKind(msg.clientKind) ? this.watchUrlField(msg.sessionId) : {}),
         });
         if (agentAway) {
             // Joining into a held session. Refusing instead would be worse than it sounds: the viewer
@@ -2218,7 +2500,10 @@ export class RelayServer {
             // the P-frames around it, so the amplification lands on the tester who is actually watching. The
             // backpressure path already throttles this exact message per session; sharing that requester is
             // what keeps the two callers from having two policies.
-            this.idrRequester(session.id)();
+            // Not for a holder that takes no frames: the keyframe would go to watchers alone, and it would spend the
+            // throttle window a watcher's own `watch:start` request needs.
+            if (session.holderStream)
+                this.idrRequester(session.id)();
             // And ask what the device's network is doing (#614). `NetworkState` is agent-produced and is not
             // among the three replayed above, so without this a viewer that reconnects has no way to learn
             // whether the device is offline and its control renders in a guessed position.
@@ -2242,6 +2527,150 @@ export class RelayServer {
                 this.networkStateRequester(session.id)();
         }
     }
+    /**
+     * Admit `ws` as a read-only watcher, or say why not. The order of the refusals is the order a viewer
+     * can act on them: a connection that may never watch is told so before it learns anything about the
+     * session.
+     */
+    handleWatchStart(ws, msg) {
+        const refuse = (reason, message) => this.sendTo(ws, { type: 'watch:refused', sessionId: msg.sessionId, reason, message });
+        if (!this.mayWatch.get(ws))
+            return refuse('not-permitted', 'Sign in to watch a session.');
+        const session = this.sessions.get(msg.sessionId);
+        if (!session)
+            return refuse('session-not-found', 'Session not found');
+        if (session.owner === null || !isAiClientKind(session.holderKind)) {
+            return refuse('not-watchable', 'Only a session an AI client is driving can be watched.');
+        }
+        // The holder already receives everything. Watching too would deliver each frame twice to one socket,
+        // and the watcher gates would then refuse the holder's own shutdown and re-join.
+        if (this.ownsSession(ws, session))
+            return refuse('not-watchable', 'This connection already holds the session.');
+        if (this.sessions.addWatcher(session.id, ws, session.owner) === 'full') {
+            return refuse('watchers-full', 'This session already has as many watchers as it allows.');
+        }
+        this.sendTo(ws, { type: 'watch:started', sessionId: session.id });
+        // The same replay a re-joining viewer gets, for the same reason: a watch can begin mid-session.
+        if (session.agentSocket.readyState !== WebSocket.OPEN) {
+            this.sendTo(ws, { type: 'session:agent-away', sessionId: session.id });
+            return;
+        }
+        if (session.chromeData)
+            this.sendTo(ws, { type: 'session:chrome', sessionId: session.id, payload: session.chromeData });
+        if (session.deviceInfo)
+            this.sendTo(ws, { type: 'session:deviceInfo', sessionId: session.id, payload: session.deviceInfo });
+        if (session.postures)
+            this.sendTo(ws, { type: 'device:postures', sessionId: session.id, payload: session.postures });
+        if (session.readySent) {
+            this.sendTo(ws, { type: 'device:ready', payload: { deviceId: session.deviceId } });
+            // Through the session's throttled requester, as the re-join does: a watcher arriving mid-GOP needs
+            // a keyframe, and a socket starting watches repeatedly must not turn into a stream of IDRs.
+            this.idrRequester(session.id)();
+        }
+    }
+    /** The keyframe-aware sender for one watcher of one session — see `watcherDroppers`. */
+    watcherDropper(ws, sessionId) {
+        let bySession = this.watcherDroppers.get(ws);
+        if (!bySession) {
+            bySession = new Map();
+            this.watcherDroppers.set(ws, bySession);
+        }
+        let dropper = bySession.get(sessionId);
+        if (!dropper) {
+            dropper = createKeyframeAwareSender();
+            bySession.set(sessionId, dropper);
+        }
+        return dropper;
+    }
+    forgetWatcherDropper(ws, sessionId) {
+        const bySession = this.watcherDroppers.get(ws);
+        bySession?.delete(sessionId);
+        if (bySession?.size === 0)
+            this.watcherDroppers.delete(ws);
+    }
+    /**
+     * Forward an agent's frame to the session's watchers. **Only the messages that describe the device**
+     * call this — chrome, device info, postures, booting, ready — never a reply to something the holder
+     * asked. An ack, a clipboard payload or a network state is the holder's, and a watcher shown one would
+     * learn what the agent is doing on a channel it was never admitted to.
+     */
+    forwardToWatchers(sessionId, raw) {
+        const watchers = this.sessions.watchersOf(sessionId);
+        if (watchers.length === 0)
+            return;
+        const frame = JSON.stringify(raw);
+        for (const w of watchers)
+            if (w.readyState === WebSocket.OPEN)
+                w.send(frame);
+    }
+    /**
+     * What the holder just did, as its watchers are told it (A2): only for an input the relay passed on to the agent,
+     * so a refused one is never shown. **Built, never forwarded**: the raw `input:type` carries the text typed, and a
+     * watcher gets a count.
+     */
+    tellWatchersInput(session, msg) {
+        let action = null;
+        switch (msg.type) {
+            case 'input:touch:end': {
+                const g = session.gesture;
+                session.gesture = null;
+                action = gestureAction(g?.start ?? null, msg.payload ?? g?.last ?? null);
+                break;
+            }
+            case 'input:pinch:end':
+                action = { kind: 'pinch' };
+                break;
+            case 'input:key':
+                action = { kind: 'key', key: keyForWatchers(msg.payload.code) };
+                break;
+            // One press is a down and an up from a client that sends both; told once, on the up.
+            case 'input:button':
+                action = msg.payload.phase === 'down' ? null : { kind: 'button', button: msg.payload.name };
+                break;
+            case 'input:type':
+                action = { kind: 'type', chars: [...msg.payload.text].length };
+                break;
+        }
+        if (action)
+            this.tellWatchersAction(session.id, action);
+    }
+    tellWatchersAction(sessionId, action) {
+        const watchers = this.sessions.watchersOf(sessionId);
+        if (watchers.length === 0)
+            return;
+        for (const w of watchers)
+            this.sendTo(w, { type: 'watch:action', sessionId, at: Date.now(), action });
+    }
+    /** The relay-originated half of what watchers hear: holder left, agent away, agent rebound. */
+    tellWatchers(sessionId, msg) {
+        for (const w of this.sessions.watchersOf(sessionId))
+            this.sendTo(w, msg);
+    }
+    /** Who to show behind a holder: the signed-in user's email, or `local` for a loopback client with no
+     *  credential. A label for people only — nothing reads it back. */
+    holderLabel(ws) {
+        const user = this.userOf(ws);
+        const id = Number(user);
+        if (!Number.isInteger(id))
+            return 'local';
+        try {
+            const row = getDb().prepare('SELECT email FROM users WHERE id = ?').get(id);
+            return row?.email ?? 'unknown';
+        }
+        catch {
+            return 'unknown';
+        }
+    }
+    /**
+     * The dashboard page that watches a session — **only at an address a teammate's browser can open**, the
+     * gate invite links take (`forTeammates`). With no tunnel and no `relay.url` the only base this relay has
+     * is localhost, which is wrong for a client on another machine; the field is left out and the client
+     * builds the link from the address it dialled, which the person running it can always open.
+     */
+    watchUrlField(sessionId) {
+        const base = forTeammates(resolvePublicBaseUrl(config, this.options.tunnel));
+        return base === null ? {} : { watchUrl: `${base}/automation/sessions/${encodeURIComponent(sessionId)}` };
+    }
     /** Shut the device down because nobody is watching it any more. The idle timer's payload, hoisted out
      *  of the close handler so the release loop there reads as one line per session. */
     idleShutdown(sessionId) {
@@ -2391,6 +2820,12 @@ export class RelayServer {
         }
         if (session?.agentSocket.readyState === WebSocket.OPEN) {
             session.agentSocket.send(JSON.stringify(msg));
+            // Where a touch began and last was, for its end to be told to watchers (A2). Moves are not sent on: one
+            // action per gesture, at its end.
+            if (msg.type === 'input:touch:start')
+                session.gesture = { start: msg.payload, last: msg.payload };
+            else if (msg.type === 'input:touch:move' && session.gesture)
+                session.gesture.last = msg.payload;
         }
     }
     /** The five inputs an ack answers, so the five that carry a required correlator.
@@ -2418,6 +2853,7 @@ export class RelayServer {
         }
         if (session?.agentSocket.readyState === WebSocket.OPEN) {
             session.agentSocket.send(JSON.stringify(msg));
+            this.tellWatchersInput(session, msg);
             return;
         }
         // A request that cannot be dispatched. Two situations reach here and only one is the agent's fault:
@@ -2585,6 +3021,7 @@ export class RelayServer {
                 buildBytes: bytes,
             },
         });
+        this.tellWatchersAction(sessionId, { kind: 'install' });
     }
     /**
      * Says once, per agent, that this agent cannot fetch builds over HTTP.
@@ -2641,6 +3078,7 @@ export class RelayServer {
             requestId,
             payload: { bundleId: build.bundle_id },
         });
+        this.tellWatchersAction(sessionId, { kind: 'launch' });
     }
     async handleGetScreenshot(req, res, params) {
         if (!requireViewAuth(req, res))
@@ -2791,6 +3229,7 @@ export class RelayServer {
         this.resourceBuffers.clear();
     }
     async handleRequest(req, res) {
+        this.warnTrustedProxyWithoutXffOnce(req);
         const corsHeaders = resolveCorsHeaders(req.headers.origin, this.corsAllowed);
         if (corsHeaders) {
             for (const [k, v] of Object.entries(corsHeaders))