@yeaft/webchat-agent 1.0.415 → 1.0.417

package/browser-runtime/chromium.js

@@ -0,0 +1,192 @@
+import { mkdtemp, mkdir, rm } from 'node:fs/promises';
+import { join } from 'node:path';
+import {
+  BROWSER_RUNTIME_CHROME_BUILD,
+  readBrowserExecutableVersion,
+  resolveBrowserExecutable,
+} from './browser-install.js';
+import { BROWSER_EXTENSION_DIR } from './extension.js';
+import { BrowserRuntimeError } from './errors.js';
+
+const BROWSER_CLOSE_TIMEOUT_MS = 5_000;
+const BROWSER_FORCE_EXIT_TIMEOUT_MS = 1_000;
+
+function browserProcess(browser) {
+  try { return browser?.process?.() || null; } catch { return null; }
+}
+
+function forceKillBrowser(browser) {
+  const child = browserProcess(browser);
+  if (!child || child.exitCode !== null) return child;
+  try {
+    if (process.platform !== 'win32' && Number.isInteger(child.pid)) process.kill(-child.pid, 'SIGKILL');
+    else child.kill('SIGKILL');
+  } catch {
+    try { child.kill('SIGKILL'); } catch {}
+  }
+  return child;
+}
+
+async function settleWithin(promise, timeoutMs) {
+  let timer;
+  try {
+    return await Promise.race([
+      Promise.resolve(promise).then(() => true, () => false),
+      new Promise(resolve => { timer = setTimeout(() => resolve(false), timeoutMs); }),
+    ]);
+  } finally {
+    clearTimeout(timer);
+  }
+}
+
+async function waitForExit(child, timeoutMs) {
+  if (!child || child.exitCode !== null || typeof child.once !== 'function') return;
+  let timer;
+  await Promise.race([
+    new Promise(resolve => child.once('exit', resolve)),
+    new Promise(resolve => { timer = setTimeout(resolve, timeoutMs); }),
+  ]);
+  clearTimeout(timer);
+}
+
+async function cleanupBrowser(browser, profileDir) {
+  if (browser) {
+    let closePromise;
+    try { closePromise = browser.close(); } catch { closePromise = Promise.reject(new Error('Browser close failed')); }
+    if (!await settleWithin(closePromise, BROWSER_CLOSE_TIMEOUT_MS)) {
+      const child = forceKillBrowser(browser);
+      await waitForExit(child, BROWSER_FORCE_EXIT_TIMEOUT_MS);
+    }
+  }
+  await rm(profileDir, { recursive: true, force: true }).catch(() => {});
+}
+
+function actualChromeBuild(version) {
+  return String(version || '').match(/(?:Chrome(?:\s+for\s+Testing)?|Chromium)(?:\/|\s+)(\d+(?:\.\d+){0,3})/i)?.[1] || null;
+}
+
+function boundedViewport(value, config) {
+  const width = Math.min(config.maxWidth, Math.max(320, Math.floor(Number(value?.width) || 1280)));
+  const height = Math.min(config.maxHeight, Math.max(240, Math.floor(Number(value?.height) || 720)));
+  const deviceScaleFactor = Math.min(2, Math.max(1, Number(value?.deviceScaleFactor) || 1));
+  return Object.freeze({ width, height, deviceScaleFactor });
+}
+
+async function prepareExtensionRuntime(browser, installed, page, launchConfig, timeoutMs) {
+  // Loading the bundled popup in bootstrap mode starts the MV3 worker without
+  // consuming the extension-action gesture required by tabCapture.
+  const bootstrap = await browser.newPage();
+  try {
+    await bootstrap.goto(`chrome-extension://${installed.id}/popup.html?bootstrap=1`, {
+      waitUntil: 'domcontentloaded',
+      timeout: timeoutMs,
+    });
+    const target = await browser.waitForTarget(candidate => (
+      candidate.type() === 'service_worker'
+        && candidate.url().startsWith(`chrome-extension://${installed.id}/`)
+    ), { timeout: timeoutMs });
+    const worker = await target.worker();
+    if (!worker) throw new BrowserRuntimeError('browser_extension_worker_missing');
+    await worker.evaluate(async value => {
+      await chrome.storage.session.set({ browserRuntimeLaunch: value });
+    }, launchConfig);
+  } finally {
+    await bootstrap.close().catch(() => {});
+  }
+  await page.bringToFront();
+  await page.triggerExtensionAction(installed);
+}
+
+/** Launch one isolated Chromium process and page owned by one Browser Session. */
+export async function launchBrowserSession({
+  browserSessionId,
+  bridgeUrl,
+  config,
+  initialUrl = 'about:blank',
+  viewport,
+  locale = 'en-US',
+  launch = null,
+  signal = null,
+} = {}) {
+  if (!browserSessionId || !bridgeUrl) throw new BrowserRuntimeError('browser_session_invalid');
+  const cacheDir = config.cacheDir;
+  const profilesDir = `${cacheDir}-profiles`;
+  await mkdir(profilesDir, { recursive: true, mode: 0o700 });
+  const profileDir = await mkdtemp(join(profilesDir, 'session-'));
+  let browser = null;
+  try {
+    signal?.throwIfAborted();
+    const executablePath = await resolveBrowserExecutable({
+      executablePath: config.executablePath,
+      cacheDir,
+    });
+    if (!executablePath) throw new BrowserRuntimeError('browser_executable_missing');
+    signal?.throwIfAborted();
+    const version = await readBrowserExecutableVersion(executablePath, { signal });
+    if (actualChromeBuild(version) !== BROWSER_RUNTIME_CHROME_BUILD) {
+      throw new BrowserRuntimeError('browser_version_mismatch');
+    }
+    const launchBrowser = launch || (await import('puppeteer-core')).default.launch;
+    const resolvedViewport = boundedViewport(viewport, config);
+    browser = await launchBrowser({
+      executablePath,
+      headless: config.headless,
+      userDataDir: profileDir,
+      acceptInsecureCerts: false,
+      timeout: config.startupProbeTimeoutMs,
+      protocolTimeout: config.maxActionRuntimeMs,
+      defaultViewport: resolvedViewport,
+      signal,
+      args: [
+        `--disable-extensions-except=${BROWSER_EXTENSION_DIR}`,
+        `--load-extension=${BROWSER_EXTENSION_DIR}`,
+        '--disable-background-networking',
+        '--disable-component-update',
+        '--disable-default-apps',
+        '--disable-features=MediaRouter,Translate',
+        '--disable-sync',
+        '--no-first-run',
+        `--lang=${String(locale || 'en-US').slice(0, 32)}`,
+      ],
+    });
+    signal?.throwIfAborted();
+    const installed = [...(await browser.extensions()).values()]
+      .find(candidate => candidate.name === 'Yeaft Browser Runtime');
+    if (!installed?.id) throw new BrowserRuntimeError('browser_extension_missing');
+
+    const pages = await browser.pages();
+    const page = pages[0] || await browser.newPage();
+    await page.setViewport(resolvedViewport);
+    signal?.throwIfAborted();
+    await page.goto(String(initialUrl || 'about:blank').slice(0, 4096), {
+      waitUntil: 'domcontentloaded',
+      timeout: config.maxActionRuntimeMs,
+    });
+
+    signal?.throwIfAborted();
+    await prepareExtensionRuntime(browser, installed, page, {
+      browserSessionId,
+      bridgeUrl,
+      targetTabId: page.target()?._targetId || null,
+    }, config.maxActionRuntimeMs);
+
+    return {
+      browser,
+      page,
+      profileDir,
+      viewport: resolvedViewport,
+      captureMode: 'tab',
+      extensionId: installed.id,
+      async close() {
+        const ownedBrowser = browser;
+        browser = null;
+        await cleanupBrowser(ownedBrowser, profileDir);
+      },
+    };
+  } catch (error) {
+    const ownedBrowser = browser;
+    browser = null;
+    await cleanupBrowser(ownedBrowser, profileDir);
+    throw error;
+  }
+}

package/browser-runtime/cli.js

@@ -69,7 +69,7 @@ export async function handleBrowserCommand(args, dependencies = {}) {
     const result = configApi.updateBrowserRuntimeSettings({ enabled: action === 'enable' }, yeaftDir);
     if (result.error) throw new Error(result.error);
     log(JSON.stringify(result, null, 2));
-    log('Restart the selected Agent instance to run the startup probe. Phase 0 does not advertise Browser capability.');
+    log('Restart the selected Agent instance. A successful supported-platform probe advertises Browser viewer capability.');
     return;
   }
   if (action === 'status') {

package/browser-runtime/extension/manifest.json

@@ -14,5 +14,8 @@
     "offscreen",
     "storage",
     "tabCapture"
+  ],
+  "host_permissions": [
+    "http://127.0.0.1/*"
   ]
 }

package/browser-runtime/extension/offscreen.js

@@ -1,4 +1,5 @@
 const PROBE_TIMEOUT_MS = 5_000;
+const sessions = new Map();
 
 function sleep(ms) {
   return new Promise(resolve => setTimeout(resolve, ms));
@@ -22,44 +23,52 @@ async function inboundVideoStats(peer) {
   return null;
 }
 
+async function captureTab(streamId) {
+  const stream = await navigator.mediaDevices.getUserMedia({
+    audio: false,
+    video: {
+      mandatory: {
+        chromeMediaSource: 'tab',
+        chromeMediaSourceId: streamId,
+      },
+    },
+  });
+  const track = stream.getVideoTracks()[0];
+  if (!track) {
+    stream.getTracks().forEach(item => item.stop());
+    throw new Error('tab capture returned no video track');
+  }
+  return { stream, track };
+}
+
+function preferVp8(peer, sender) {
+  const vp8 = RTCRtpSender.getCapabilities('video')?.codecs
+    ?.filter(codec => codec.mimeType?.toLowerCase() === 'video/vp8') || [];
+  if (vp8.length === 0) throw new Error('VP8 encoder capability unavailable');
+  const transceiver = peer.getTransceivers().find(item => item.sender === sender);
+  transceiver?.setCodecPreferences(vp8);
+}
+
 async function runMediaProbe(streamId) {
   let stream = null;
   let offerer = null;
   let answerer = null;
   try {
-    stream = await navigator.mediaDevices.getUserMedia({
-      audio: false,
-      video: {
-        mandatory: {
-          chromeMediaSource: 'tab',
-          chromeMediaSourceId: streamId,
-        },
-      },
-    });
+    ({ stream } = await captureTab(streamId));
     const track = stream.getVideoTracks()[0];
-    if (!track) throw new Error('tab capture returned no video track');
-
     offerer = new RTCPeerConnection();
     answerer = new RTCPeerConnection();
     offerer.onicecandidate = event => event.candidate && answerer.addIceCandidate(event.candidate);
     answerer.onicecandidate = event => event.candidate && offerer.addIceCandidate(event.candidate);
-
     const video = document.querySelector('#probe-video');
-    answerer.ontrack = event => {
-      video.srcObject = event.streams[0] || new MediaStream([event.track]);
-    };
+    answerer.ontrack = event => { video.srcObject = event.streams[0] || new MediaStream([event.track]); };
     const sender = offerer.addTrack(track, stream);
-    const vp8 = RTCRtpSender.getCapabilities('video')?.codecs
-      ?.filter(codec => codec.mimeType?.toLowerCase() === 'video/vp8') || [];
-    if (vp8.length === 0) throw new Error('VP8 encoder capability unavailable');
-    const transceiver = offerer.getTransceivers().find(item => item.sender === sender);
-    transceiver?.setCodecPreferences(vp8);
+    preferVp8(offerer, sender);
     await offerer.setLocalDescription(await offerer.createOffer());
     await answerer.setRemoteDescription(offerer.localDescription);
     await answerer.setLocalDescription(await answerer.createAnswer());
     await offerer.setRemoteDescription(answerer.localDescription);
     await video.play().catch(() => {});
-
     const deadline = Date.now() + PROBE_TIMEOUT_MS;
     let media = null;
     while (Date.now() < deadline) {
@@ -89,12 +98,179 @@ async function runMediaProbe(streamId) {
   }
 }
 
+function bridgeSend(record, message) {
+  if (record.socket?.readyState === WebSocket.OPEN) {
+    record.socket.send(JSON.stringify({ ...message, browserSessionId: record.browserSessionId }));
+  }
+}
+
+function peerRecord(record, message) {
+  const peer = record?.peers.get(message?.peerId);
+  return peer?.connectionGeneration === message?.connectionGeneration ? peer : null;
+}
+
+function closePeer(record, peer, notify = false) {
+  if (!record || !peer || record.peers.get(peer.peerId) !== peer) return false;
+  record.peers.delete(peer.peerId);
+  peer.connection.close();
+  if (notify) bridgeSend(record, {
+    type: 'peer_closed',
+    peerId: peer.peerId,
+    connectionGeneration: peer.connectionGeneration,
+  });
+  return true;
+}
+
+function closeSession(record, notify = true) {
+  if (!record || record.closed) return;
+  record.closed = true;
+  if (sessions.get(record.browserSessionId) === record) sessions.delete(record.browserSessionId);
+  for (const peer of [...record.peers.values()]) closePeer(record, peer, notify);
+  record.stream?.getTracks().forEach(track => track.stop());
+  try { record.socket?.close(1000, 'Browser Session closed'); } catch {}
+}
+
+async function startRuntime({ browserSessionId, bridgeUrl, streamId }) {
+  closeSession(sessions.get(browserSessionId), false);
+  const { stream, track } = await captureTab(streamId);
+  const socket = new WebSocket(bridgeUrl);
+  const record = {
+    browserSessionId,
+    stream,
+    track,
+    socket,
+    peers: new Map(),
+    closed: false,
+  };
+  sessions.set(browserSessionId, record);
+  track.addEventListener('ended', () => bridgeSend(record, { type: 'capture_ended' }), { once: true });
+  socket.onopen = () => bridgeSend(record, {
+    type: 'runtime_ready',
+    captureMode: 'tab',
+    settings: track.getSettings?.() || {},
+  });
+  socket.onmessage = event => {
+    let message;
+    try { message = JSON.parse(event.data); } catch { return; }
+    if (message.browserSessionId !== browserSessionId) return;
+    handleBridgeMessage(record, message).catch(error => {
+      const peer = peerRecord(record, message);
+      if (peer) closePeer(record, peer);
+      bridgeSend(record, {
+        type: 'peer_error',
+        peerId: message.peerId || null,
+        connectionGeneration: message.connectionGeneration || null,
+        code: error?.name || 'peer_failed',
+        safeError: String(error?.message || error).slice(0, 500),
+      });
+    });
+  };
+  socket.onclose = () => closeSession(record, false);
+  return { ok: true };
+}
+
+async function createPeer(record, message) {
+  const peerId = message.peerId;
+  const connectionGeneration = message.connectionGeneration;
+  const existing = record.peers.get(peerId);
+  if (existing?.connectionGeneration === connectionGeneration) return;
+  if (existing) closePeer(record, existing);
+  const connection = new RTCPeerConnection({
+    iceServers: Array.isArray(message.iceServers) ? message.iceServers : [],
+    iceTransportPolicy: message.iceTransportPolicy === 'relay' ? 'relay' : 'all',
+  });
+  const peer = {
+    peerId,
+    connectionGeneration,
+    connection,
+    pendingCandidates: [],
+    offerSent: false,
+    localCandidates: [],
+  };
+  record.peers.set(peerId, peer);
+  const isCurrent = () => peerRecord(record, peer) === peer;
+  connection.onicecandidate = event => {
+    if (!isCurrent()) return;
+    const candidateMessage = {
+      type: 'peer_ice_candidate',
+      peerId,
+      connectionGeneration,
+      candidate: event.candidate ? event.candidate.toJSON() : null,
+    };
+    if (!peer.offerSent) peer.localCandidates.push(candidateMessage);
+    else bridgeSend(record, candidateMessage);
+  };
+  connection.onconnectionstatechange = () => {
+    if (!isCurrent()) return;
+    bridgeSend(record, {
+      type: 'peer_state',
+      peerId,
+      connectionGeneration,
+      state: connection.connectionState,
+    });
+  };
+  const sender = connection.addTrack(record.track, record.stream);
+  preferVp8(connection, sender);
+  const parameters = sender.getParameters();
+  if (parameters.encodings?.length) {
+    parameters.encodings[0].maxBitrate = Number(message.maxBitrate) || 4_000_000;
+    parameters.encodings[0].maxFramerate = Number(message.maxFps) || 30;
+    await sender.setParameters(parameters);
+  }
+  await connection.setLocalDescription(await connection.createOffer());
+  if (!isCurrent()) return;
+  bridgeSend(record, {
+    type: 'peer_prepared',
+    peerId,
+    connectionGeneration,
+  });
+  bridgeSend(record, {
+    type: 'peer_offer',
+    peerId,
+    connectionGeneration,
+    description: connection.localDescription,
+  });
+  peer.offerSent = true;
+  for (const candidate of peer.localCandidates.splice(0)) bridgeSend(record, candidate);
+}
+
+async function handleBridgeMessage(record, message) {
+  if (record.closed) return;
+  if (message.type === 'session_close') {
+    closeSession(record);
+    return;
+  }
+  if (message.type === 'peer_prepare') return createPeer(record, message);
+  const peer = peerRecord(record, message);
+  if (!peer) return;
+  const connection = peer.connection;
+  if (message.type === 'peer_answer') {
+    await connection.setRemoteDescription(message.description);
+    if (peerRecord(record, message) !== peer) return;
+    for (const candidate of peer.pendingCandidates.splice(0)) {
+      await connection.addIceCandidate(candidate);
+      if (peerRecord(record, message) !== peer) return;
+    }
+  } else if (message.type === 'peer_ice_candidate') {
+    if (!message.candidate) return;
+    if (!connection.remoteDescription) peer.pendingCandidates.push(message.candidate);
+    else await connection.addIceCandidate(message.candidate);
+  } else if (message.type === 'peer_close') {
+    closePeer(record, peer);
+  }
+}
+
 chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
-  if (message?.target !== 'browser_runtime_offscreen'
-      || message?.type !== 'browser_runtime_probe_media') return undefined;
-  runMediaProbe(message.streamId).then(sendResponse, error => sendResponse({
+  if (message?.target !== 'browser_runtime_offscreen') return undefined;
+  const operation = message.type === 'browser_runtime_probe_media'
+    ? () => runMediaProbe(message.streamId)
+    : message.type === 'browser_runtime_start'
+      ? () => startRuntime(message)
+      : null;
+  if (!operation) return undefined;
+  operation().then(sendResponse, error => sendResponse({
     ok: false,
-    code: 'media_probe_failed',
+    code: message.type === 'browser_runtime_probe_media' ? 'media_probe_failed' : 'browser_runtime_start_failed',
     safeError: String(error?.message || error).slice(0, 500),
   }));
   return true;

package/browser-runtime/extension/popup.js

@@ -1 +1,4 @@
-chrome.runtime.sendMessage({ type: 'browser_runtime_probe_start' }).catch(() => {});
+const bootstrap = new URLSearchParams(location.search).get('bootstrap') === '1';
+chrome.runtime.sendMessage({
+  type: bootstrap ? 'browser_runtime_bootstrap' : 'browser_runtime_action',
+}).catch(() => {});

package/browser-runtime/extension/service-worker.js

@@ -12,15 +12,20 @@ async function ensureOffscreenDocument() {
     offscreenCreation = chrome.offscreen.createDocument({
       url: OFFSCREEN_PATH,
       reasons: ['USER_MEDIA', 'WEB_RTC'],
-      justification: 'Capture the controlled tab and validate the Browser Runtime WebRTC media path',
+      justification: 'Capture the Agent-owned tab and provide the Browser Runtime WebRTC endpoint',
     }).finally(() => { offscreenCreation = null; });
   }
   await offscreenCreation;
 }
 
-async function runStartupProbe() {
+async function activeTab() {
   const [tab] = await chrome.tabs.query({ active: true, lastFocusedWindow: true });
   if (!tab?.id) throw new Error('active tab unavailable');
+  return tab;
+}
+
+async function runStartupProbe() {
+  const tab = await activeTab();
   const streamId = await chrome.tabCapture.getMediaStreamId({ targetTabId: tab.id });
   await ensureOffscreenDocument();
   return chrome.runtime.sendMessage({
@@ -30,18 +35,53 @@ async function runStartupProbe() {
   });
 }
 
+async function startRuntime(browserRuntimeLaunch) {
+  if (!browserRuntimeLaunch?.browserSessionId || !browserRuntimeLaunch?.bridgeUrl) {
+    throw new Error('Browser Runtime launch authorization missing');
+  }
+  const tab = await activeTab();
+  const streamId = await chrome.tabCapture.getMediaStreamId({ targetTabId: tab.id });
+  await ensureOffscreenDocument();
+  const response = await chrome.runtime.sendMessage({
+    target: 'browser_runtime_offscreen',
+    type: 'browser_runtime_start',
+    browserSessionId: browserRuntimeLaunch.browserSessionId,
+    bridgeUrl: browserRuntimeLaunch.bridgeUrl,
+    streamId,
+  });
+  await chrome.storage.session.remove('browserRuntimeLaunch');
+  return response;
+}
+
 chrome.runtime.onMessage.addListener((message, _sender, sendResponse) => {
-  if (message?.type !== 'browser_runtime_probe_start') return undefined;
-  runStartupProbe().then(async result => {
-    await chrome.storage.session.set({ browserRuntimeProbe: result });
+  if (message?.type === 'browser_runtime_bootstrap') {
+    sendResponse({ ok: true });
+    return undefined;
+  }
+  if (message?.type !== 'browser_runtime_probe_start'
+      && message?.type !== 'browser_runtime_action') return undefined;
+  let runningProbe = message.type === 'browser_runtime_probe_start';
+  const operation = async () => {
+    if (runningProbe) return runStartupProbe();
+    const { browserRuntimeLaunch } = await chrome.storage.session.get('browserRuntimeLaunch');
+    if (browserRuntimeLaunch) return startRuntime(browserRuntimeLaunch);
+    runningProbe = true;
+    return runStartupProbe();
+  };
+  operation().then(async result => {
+    if (runningProbe) {
+      await chrome.storage.session.set({ browserRuntimeProbe: result });
+    }
     sendResponse(result);
   }, async error => {
     const result = {
       ok: false,
-      code: 'capture_probe_failed',
+      code: message.type === 'browser_runtime_probe_start' ? 'capture_probe_failed' : 'browser_runtime_start_failed',
       safeError: String(error?.message || error).slice(0, 500),
     };
-    await chrome.storage.session.set({ browserRuntimeProbe: result });
+    if (runningProbe) {
+      await chrome.storage.session.set({ browserRuntimeProbe: result });
+    }
     sendResponse(result);
   });
   return true;

package/browser-runtime/extension.js

@@ -6,7 +6,7 @@ import { fileURLToPath } from 'node:url';
 const moduleDir = dirname(fileURLToPath(import.meta.url));
 export const BROWSER_EXTENSION_DIR = join(moduleDir, 'extension');
 export const BROWSER_EXTENSION_NAME = 'Yeaft Browser Runtime';
-export const BROWSER_EXTENSION_SHA256 = '51cc6519ec9f72f86af7a0a98c3511c32e0e8fde6e8e3406886e7048f0c5972e';
+export const BROWSER_EXTENSION_SHA256 = '13279f6bb05975f9de9f92fdc13da28341ff99c161563cda25d27a14d5cdcdd5';
 
 async function walkFiles(root, current = root) {
   const entries = await readdir(current, { withFileTypes: true });

package/browser-runtime/index.js

@@ -1,4 +1,7 @@
 export { BrowserRuntimeService, bootBrowserRuntime, getBrowserRuntime, shutdownBrowserRuntime } from './service.js';
+export { BrowserExtensionBridge } from './local-bridge.js';
+export { launchBrowserSession } from './chromium.js';
+export { handleBrowserRuntimeMessage, BROWSER_MESSAGE_TYPES } from './messages.js';
 export { probeBrowserRuntime } from './probe.js';
 export { installManagedBrowser, findManagedBrowser, defaultBrowserCacheDir, BROWSER_RUNTIME_CHROME_BUILD } from './browser-install.js';
 export { normaliseBrowserRuntimeSection, validateBrowserRuntimeUpdate } from './config.js';