@yeaft/webchat-agent 1.0.428 → 1.0.430

package/browser-runtime/extension/offscreen.js

@@ -112,6 +112,9 @@ function peerRecord(record, message) {
 function closePeer(record, peer, notify = false) {
   if (!record || !peer || record.peers.get(peer.peerId) !== peer) return false;
   record.peers.delete(peer.peerId);
+  for (const channel of Object.values(peer.channels || {})) {
+    try { channel.close(); } catch {}
+  }
   peer.connection.close();
   if (notify) bridgeSend(record, {
     type: 'peer_closed',
@@ -186,6 +189,7 @@ async function createPeer(record, message) {
     pendingCandidates: [],
     offerSent: false,
     localCandidates: [],
+    channels: {},
   };
   record.peers.set(peerId, peer);
   const isCurrent = () => peerRecord(record, peer) === peer;
@@ -209,6 +213,30 @@ async function createPeer(record, message) {
       state: connection.connectionState,
     });
   };
+  if (message.role === 'interactive') {
+    const control = connection.createDataChannel('browser.control.v1', { ordered: true });
+    const pointer = connection.createDataChannel('browser.pointer.v1', {
+      ordered: false,
+      maxRetransmits: 0,
+    });
+    peer.channels.control = control;
+    peer.channels.pointer = pointer;
+    for (const [channel, dataChannel] of Object.entries(peer.channels)) {
+      dataChannel.onmessage = event => {
+        if (!isCurrent() || typeof event.data !== 'string' || event.data.length > 32 * 1024) return;
+        let envelope;
+        try { envelope = JSON.parse(event.data); } catch { return; }
+        if (envelope?.version !== 1 || envelope.connectionGeneration !== connectionGeneration) return;
+        bridgeSend(record, {
+          type: 'peer_input',
+          peerId,
+          connectionGeneration,
+          channel,
+          envelope,
+        });
+      };
+    }
+  }
   const sender = connection.addTrack(record.track, record.stream);
   preferVp8(connection, sender);
   const parameters = sender.getParameters();

package/browser-runtime/extension.js

@@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
 const moduleDir = dirname(fileURLToPath(import.meta.url));
 export const BROWSER_EXTENSION_DIR = join(moduleDir, 'extension');
 export const BROWSER_EXTENSION_NAME = 'Yeaft Browser Runtime';
-export const BROWSER_EXTENSION_SHA256 = '13279f6bb05975f9de9f92fdc13da28341ff99c161563cda25d27a14d5cdcdd5';
+export const BROWSER_EXTENSION_SHA256 = 'c17d5b60ade5526168bf543c177370ee2ff6170c67cb197ac33c4bc015d151c4';
 
 async function walkFiles(root, current = root) {
   const entries = await readdir(current, { withFileTypes: true });

package/browser-runtime/service.js

@@ -10,6 +10,7 @@ import { probeBrowserRuntime } from './probe.js';
 import { BrowserRuntimeError } from './errors.js';
 import { BrowserExtensionBridge } from './local-bridge.js';
 import { launchBrowserSession } from './chromium.js';
+import { ProducerSequenceState } from './protocol.js';
 
 const REQUEST_TTL_MS = 10 * 60_000;
 const MAX_INSTALL_PROGRESS_LISTENERS = 8;
@@ -389,8 +390,10 @@ export class BrowserRuntimeService {
       captureMode: session.captureMode,
       viewport: session.viewport,
       viewerCount: session.peers.size,
-      interactivePeerCount: 0,
-      authorizedProducerCount: 0,
+      interactivePeerCount: [...session.peers.values()].filter(peer => peer.role === 'interactive').length,
+      authorizedProducerCount: [...session.peers.values()].filter(peer => (
+        peer.role === 'interactive' && peer.state === 'connected'
+      )).length,
       expiresAt: session.expiresAt,
       terminalReason: session.terminalReason || null,
       safeError: session.safeError || null,
@@ -405,6 +408,17 @@ export class BrowserRuntimeService {
 
   #dropPeer(session, peer, reason = 'peer_closed') {
     if (!peer || session.peers.get(peer.peerId) !== peer) return false;
+    if (peer.role === 'interactive') {
+      void peer.actionChain.finally(async () => {
+        const page = session.runtime?.page;
+        for (const key of ['Alt', 'Control', 'Meta', 'Shift']) {
+          try { await page?.keyboard?.up?.(key); } catch {}
+        }
+        for (const button of ['left', 'middle', 'right']) {
+          try { await page?.mouse?.up?.({ button }); } catch {}
+        }
+      });
+    }
     clearTimeout(peer.expiryTimer);
     peer.expiryTimer = null;
     session.peers.delete(peer.peerId);
@@ -591,13 +605,23 @@ export class BrowserRuntimeService {
       return true;
     }
     if (session.peers.size >= this.config.maxPeersPerSession) throw new BrowserRuntimeError('browser_peer_limit');
+    const role = message?.role === 'interactive' ? 'interactive' : 'viewer';
+    if (role === 'interactive' && [...session.peers.values()].some(item => item.role === 'interactive')) {
+      throw new BrowserRuntimeError('browser_interactive_peer_limit');
+    }
     const peer = {
       peerId,
       connectionGeneration,
       identity,
+      role,
       state: 'preparing',
       webCandidateCount: 0,
       agentCandidateCount: 0,
+      sequences: new ProducerSequenceState({ producerId: peerId, producerGeneration: connectionGeneration }),
+      actionChain: Promise.resolve(),
+      pendingControlCount: 0,
+      pendingPointer: null,
+      pointerDrainScheduled: false,
       expiresAt: Number(message?.routeExpiresAt) || Date.now() + 10 * 60_000,
       expiryTimer: null,
     };
@@ -613,6 +637,7 @@ export class BrowserRuntimeService {
       type: 'peer_prepare',
       peerId,
       connectionGeneration,
+      role: peer.role,
       iceServers: publicIceServers(message?.agentIceServers),
       iceTransportPolicy: message?.iceTransportPolicy === 'relay' ? 'relay' : 'all',
       maxBitrate: this.config.maxBitrate,
@@ -743,8 +768,102 @@ export class BrowserRuntimeService {
     });
   }
 
+  async #executeInteractiveAction(session, peer, envelope, pointer = false) {
+    if (session.state !== 'ready' || peer.role !== 'interactive' || peer.state !== 'connected') return;
+    const sequenceEnvelope = {
+      producerId: peer.peerId,
+      producerGeneration: peer.connectionGeneration,
+      ...(pointer ? { pointerSeq: Number(envelope?.pointerSeq) } : { controlSeq: Number(envelope?.controlSeq) }),
+    };
+    const accepted = pointer
+      ? peer.sequences.acceptPointer(sequenceEnvelope)
+      : peer.sequences.acceptControl(sequenceEnvelope);
+    if (!accepted.accepted) return;
+    const action = envelope?.action;
+    if (!action || typeof action !== 'object') return;
+    const page = session.runtime?.page;
+    if (!page) return;
+    if (pointer) {
+      const x = Math.min(session.viewport?.width || 1920, Math.max(0, Number(action.x) || 0));
+      const y = Math.min(session.viewport?.height || 1080, Math.max(0, Number(action.y) || 0));
+      if (action.type === 'pointerMove') await page.mouse.move(x, y);
+      return;
+    }
+    if (action.type === 'navigate') {
+      const url = safeInitialUrl(action.url);
+      await page.goto(url, { waitUntil: 'domcontentloaded', timeout: this.config.maxActionRuntimeMs });
+    } else if (action.type === 'mouse') {
+      const hasPosition = Number.isFinite(Number(action.x)) && Number.isFinite(Number(action.y));
+      const x = Math.min(session.viewport?.width || 1920, Math.max(0, Number(action.x) || 0));
+      const y = Math.min(session.viewport?.height || 1080, Math.max(0, Number(action.y) || 0));
+      const button = ['left', 'middle', 'right'].includes(action.button) ? action.button : 'left';
+      if (hasPosition) await page.mouse.move(x, y);
+      if (action.event === 'down') await page.mouse.down({ button });
+      else if (action.event === 'up') await page.mouse.up({ button });
+      else if (action.event === 'click') await page.mouse.click(x, y, { button, clickCount: Math.min(3, Math.max(1, Number(action.clickCount) || 1)) });
+    } else if (action.type === 'wheel') {
+      await page.mouse.wheel({
+        deltaX: Math.max(-4000, Math.min(4000, Number(action.deltaX) || 0)),
+        deltaY: Math.max(-4000, Math.min(4000, Number(action.deltaY) || 0)),
+      });
+    } else if (action.type === 'key') {
+      const key = clean(action.key, 128);
+      if (!key) return;
+      if (action.event === 'down') await page.keyboard.down(key);
+      else if (action.event === 'up') await page.keyboard.up(key);
+      else await page.keyboard.press(key);
+    } else if (action.type === 'text') {
+      const text = typeof action.text === 'string' ? action.text.slice(0, 16 * 1024) : '';
+      if (text) await page.keyboard.type(text);
+    } else if (action.type === 'resetInput') {
+      for (const key of ['Alt', 'Control', 'Meta', 'Shift']) {
+        try { await page.keyboard.up(key); } catch {}
+      }
+      for (const button of ['left', 'middle', 'right']) {
+        try { await page.mouse.up({ button }); } catch {}
+      }
+    }
+  }
+
   #handleBridgeMessage(session, message) {
     if (this.sessions.get(session.browserSessionId) !== session) return;
+    if (message.type === 'peer_input') {
+      const peer = session.peers.get(clean(message.peerId));
+      if (!peer || peer.connectionGeneration !== Number(message.connectionGeneration)) return;
+      if (message.channel === 'pointer') {
+        peer.pendingPointer = message.envelope;
+        if (peer.pointerDrainScheduled) return;
+        peer.pointerDrainScheduled = true;
+        peer.actionChain = peer.actionChain.then(async () => {
+          while (peer.pendingPointer && session.peers.get(peer.peerId) === peer) {
+            const envelope = peer.pendingPointer;
+            peer.pendingPointer = null;
+            await this.#executeInteractiveAction(session, peer, envelope, true);
+          }
+        }).catch(() => {}).finally(() => { peer.pointerDrainScheduled = false; });
+        return;
+      }
+      const maxPending = Math.max(1, Number(this.config.maxQueuedActionsPerProducer) || 32);
+      if (peer.pendingControlCount >= maxPending) {
+        this.#dropPeer(session, peer, 'control_queue_saturated');
+        void this.#emit({
+          type: 'browser_peer_error',
+          browserSessionId: session.browserSessionId,
+          peerId: peer.peerId,
+          connectionGeneration: peer.connectionGeneration,
+          code: 'browser_control_queue_saturated',
+          safeError: 'Browser input queue saturated; reconnect required',
+        });
+        void this.#emitSnapshot(session);
+        return;
+      }
+      peer.pendingControlCount += 1;
+      peer.actionChain = peer.actionChain
+        .then(() => this.#executeInteractiveAction(session, peer, message.envelope, false))
+        .catch(() => {})
+        .finally(() => { peer.pendingControlCount = Math.max(0, peer.pendingControlCount - 1); });
+      return;
+    }
     if (message.type === 'peer_prepared' || message.type === 'peer_offer'
         || message.type === 'peer_ice_candidate' || message.type === 'peer_state'
         || message.type === 'peer_error') {

package/local-runtime/server/handlers/client-browser.js

@@ -205,7 +205,7 @@ export async function handleClientBrowser(client, msg, checkAgentAccess) {
       client,
       requestId,
       connectionGeneration,
-      role: 'viewer',
+      role: msg.role === 'interactive' ? 'interactive' : 'viewer',
     });
     if (reserved.error) return fail(client, msg, reserved.error);
     const peer = reserved.peer;
@@ -243,6 +243,7 @@ export async function handleClientBrowser(client, msg, checkAgentAccess) {
         connectionGeneration: peer.connectionGeneration,
         serverIdentity: identity,
         routeExpiresAt: peer.expiresAt,
+        role: peer.role,
         iceTransportPolicy: CONFIG.browserRuntime.iceTransportPolicy,
         agentIceServers: mintBrowserIceServers({ ...commonScope, endpointRole: 'agent' }),
       });

package/local-runtime/version.json

@@ -1 +1 @@
-{"version":"1.0.428"}
+{"version":"1.0.430"}