@sanohiro/casty 1.1.2 → 1.2.1

package/README.ja.md

@@ -31,6 +31,12 @@ casty は w3m や lynx のようなテキストブラウザではありません
 
 SSH でヘッドレスサーバーに入っていて Web ページを確認したいとき、普通は `curl` か `lynx` か X11 転送しかない。casty ならターミナルを離れずに本物のブラウザが使えます。X11 も VNC も Wayland もいらない。Kitty 対応ターミナルさえあれば OK。
 
+### Google Meet でカメラ＆マイク（実験的）
+
+![casty で Google Meet](docs/screenshot-meet.png)
+
+ffmpeg 経由でカメラとマイクをキャプチャし、Google Meet や Zoom 等の WebRTC サイトで使えます。`ffmpeg` のインストールが必要です。映像はデバイスから直接取得するため、背景エフェクトは使えません。有効にするには[設定](#設定)を参照。
+
 ## インストール
 
 ```bash
@@ -124,6 +130,7 @@ casty   # ホームページを開く
 | `transport` | 画像転送方式: `auto`, `file`, `inline` | `auto` (bcon/kitty→file、他→inline) |
 | `format` | キャプチャ形式: `auto`, `png`, `jpeg` | `auto` (file→jpeg adaptive、inline→png) |
 | `mouseMode` | `1002` (ボタンイベント) or `1003` (全イベント) | 自動 (Ghostty→1003、他→1002) |
+| `media` | WebRTC 用カメラ/マイク有効化（実験的、`ffmpeg` 必要） | `false` |
 
 ## 比較
 

package/README.md

@@ -31,6 +31,12 @@ Since it's real Chrome, JavaScript, CSS, Canvas, and WebGL all work. Google logi
 
 If you're working over SSH on a headless server and need to check a web page, your options are usually `curl`, `lynx`, or forwarding X11. casty gives you an actual browser without leaving the terminal. No X11, no VNC, no Wayland — just a Kitty-compatible terminal.
 
+### Google Meet with camera & mic (experimental)
+
+![Google Meet on casty](docs/screenshot-meet.png)
+
+Camera and microphone can be streamed to WebRTC sites like Google Meet, Zoom, etc. via ffmpeg. Requires `ffmpeg` installed. Background effects are not available since the video is captured directly from the device. See [Configuration](#configuration) to enable.
+
 ## Installation
 
 ```bash
@@ -124,6 +130,7 @@ Create `~/.casty/bookmarks.json`:
 | `transport` | Image transfer: `auto`, `file`, `inline` | `auto` (bcon/kitty→file, others→inline) |
 | `format` | Capture format: `auto`, `png`, `jpeg` | `auto` (file→jpeg adaptive, inline→png) |
 | `mouseMode` | `1002` (button-event) or `1003` (any-event) | Auto (Ghostty→1003, others→1002) |
+| `media` | Enable camera/mic for WebRTC (experimental, requires `ffmpeg`) | `false` |
 
 ## Comparison
 

package/bin/casty

@@ -115,7 +115,12 @@ install_chromium() {
   mkdir -p "$BROWSERS_DIR"
   # zip 内は chrome-headless-shell-<platform>/ ディレクトリに入っている
   local tmp_dir=$(mktemp -d /tmp/casty-extract-XXXXXX)
-  unzip -q "$tmp_zip" -d "$tmp_dir" 2>/dev/null
+  if ! unzip -q "$tmp_zip" -d "$tmp_dir" 2>/dev/null; then
+    rm -f "$tmp_zip"
+    rm -rf "$tmp_dir"
+    echo "casty: Extraction failed (disk full?)" >&2
+    return 1
+  fi
   rm -f "$tmp_zip"
 
   # 展開されたディレクトリを見つけてリネーム
@@ -130,6 +135,13 @@ install_chromium() {
   rm -rf "$tmp_dir"
   chmod +x "$dest_dir/chrome-headless-shell"
 
+  # Verify binary is functional (catches truncated/corrupt downloads)
+  if ! "$dest_dir/chrome-headless-shell" --version &>/dev/null; then
+    echo "casty: Downloaded headless-shell is broken, removing..." >&2
+    rm -rf "$dest_dir"
+    return 1
+  fi
+
   touch "$STAMP"
   echo "casty: Installed Chrome Headless Shell ${version}" >&2
 }

package/bin/casty.js

@@ -60,6 +60,7 @@ import { sendFrame, resetFrameCache, clearScreen, hideCursor, showCursor, cleanu
 import { enableMouse, disableMouse, startInputHandling } from '../lib/input.js';
 import { loadKeyBindings } from '../lib/keys.js';
 import { loadConfig } from '../lib/config.js';
+import { startMedia } from '../lib/media.js';
 
 const config = loadConfig();
 const bindings = loadKeyBindings();
@@ -148,11 +149,13 @@ async function getTermInfo({ keepAlive = false } = {}) {
 }
 
 async function main() {
-  // Phase 1: Launch Chrome and get terminal info in parallel
+  // Phase 1: Launch Chrome, get terminal info, and start media in parallel
   // getTermInfo() must complete fully (prevent CSI 14t response leak)
   const browserP = startBrowser();
+  const mediaP = config.media ? startMedia(config) : null;
   const term = await getTermInfo();
   const browser = await browserP;
+  const media = mediaP ? await mediaP : null;
 
   // Reserve line 1 for URL bar, use the rest for browser display
   const barHeight = term.cellHeight;
@@ -160,7 +163,7 @@ async function main() {
   setDisplaySize(term.cols, term.rows - 1);
 
   // Phase 2: CDP connection + page setup
-  const { client, cssWidth, cssHeight } = await setupPage(browser, { ...term, height: viewHeight });
+  const { client, cssWidth, cssHeight } = await setupPage(browser, { ...term, height: viewHeight, mediaPort: media?.port || 0 });
   const chromeProcess = browser.proc;
 
   // Log WebSocket errors to stderr (prevent unhandled crash)
@@ -237,6 +240,7 @@ async function main() {
     } catch {}
     client.close();
     chromeProcess.kill();
+    media?.cleanup();
     disableMouse();
     showCursor();
     try { process.stdin.setRawMode(false); } catch {}

package/lib/browser.js

@@ -33,7 +33,7 @@ const CAPTURE_STUCK_RESET  = 5000; // Reset stuck capturing flag (ms)
 
 // Build stealth script with locale-dependent language settings
 // lang: primary language (e.g. "ja", "en-US")
-function buildStealthScript(lang, { fakeMedia = false } = {}) {
+function buildStealthScript(lang, { media = false, mediaPort = 0 } = {}) {
   // Build Accept-Language style list: primary, then en-US/en fallbacks
   const languages = [lang];
   if (lang !== 'en-US' && lang !== 'en') {
@@ -93,7 +93,7 @@ navigator.permissions.query = (params) => {
   if (params.name === 'notifications') {
     return Promise.resolve({ state: Notification.permission });
   }
-${fakeMedia ? `  if (params.name === 'camera' || params.name === 'microphone') {
+${media ? `  if (params.name === 'camera' || params.name === 'microphone') {
     return Promise.resolve({ state: 'granted' });
   }` : ''}
   return origQuery(params);
@@ -115,35 +115,81 @@ if (!navigator.connection) {
   });
 }
 
-${fakeMedia ? `
-// getUserMedia emulation (headless-shell lacks media capture)
+${media && mediaPort ? `
+// getUserMedia with real device capture via ffmpeg WebSocket relay
+if (!navigator.mediaDevices) navigator.mediaDevices = {};
 if (navigator.mediaDevices) {
-  const fakeStream = (constraints) => {
-    const tracks = [];
-    if (constraints.video) {
-      const w = constraints.video.width?.ideal || constraints.video.width?.max || 640;
-      const h = constraints.video.height?.ideal || constraints.video.height?.max || 480;
-      const canvas = document.createElement('canvas');
-      canvas.width = w;
-      canvas.height = h;
-      const ctx = canvas.getContext('2d');
-      const draw = () => { ctx.fillStyle = '#000'; ctx.fillRect(0, 0, w, h); requestAnimationFrame(draw); };
-      draw();
-      const stream = canvas.captureStream(30);
-      tracks.push(...stream.getVideoTracks());
-    }
-    if (constraints.audio) {
-      const actx = new AudioContext();
-      const gain = actx.createGain();
-      gain.gain.value = 0;
-      gain.connect(actx.destination);
-      const dest = actx.createMediaStreamDestination();
-      gain.connect(dest);
-      tracks.push(...dest.stream.getAudioTracks());
-    }
-    return Promise.resolve(new MediaStream(tracks));
+  navigator.mediaDevices.getUserMedia = (constraints) => {
+    return new Promise((resolve) => {
+      const tracks = [];
+      const ws = new WebSocket('ws://127.0.0.1:${mediaPort}');
+      ws.binaryType = 'arraybuffer';
+
+      let canvas, ctx, vstream;
+      if (constraints.video) {
+        const w = constraints.video.width?.ideal || constraints.video.width?.max || 640;
+        const h = constraints.video.height?.ideal || constraints.video.height?.max || 480;
+        canvas = document.createElement('canvas');
+        canvas.width = w; canvas.height = h;
+        ctx = canvas.getContext('2d');
+        vstream = canvas.captureStream(15);
+        tracks.push(...vstream.getVideoTracks());
+      }
+
+      let pcmQueue;
+      if (constraints.audio) {
+        const actx = new AudioContext({ sampleRate: 48000 });
+        if (actx.state === 'suspended') actx.resume();
+        pcmQueue = [];
+        // Use 1 input channel to ensure onaudioprocess fires reliably
+        const proc = actx.createScriptProcessor(4096, 1, 1);
+        // Feed silence into input to keep processor running
+        const silence = actx.createBufferSource();
+        const silenceBuf = actx.createBuffer(1, 4096, 48000);
+        silence.buffer = silenceBuf;
+        silence.loop = true;
+        silence.connect(proc);
+        silence.start();
+        proc.onaudioprocess = (e) => {
+          const out = e.outputBuffer.getChannelData(0);
+          const chunk = pcmQueue.shift();
+          if (chunk) {
+            const s16 = new Int16Array(chunk);
+            for (let i = 0; i < out.length && i < s16.length; i++) out[i] = s16[i] / 32768;
+            if (s16.length < out.length) out.fill(0, s16.length);
+          } else {
+            out.fill(0);
+          }
+        };
+        const dest = actx.createMediaStreamDestination();
+        proc.connect(dest);
+        tracks.push(...dest.stream.getAudioTracks());
+      }
+
+      ws.onmessage = (e) => {
+        const buf = new Uint8Array(e.data);
+        const type = buf[0];
+        const payload = buf.slice(1);
+        if (type === 1 && canvas) {
+          createImageBitmap(new Blob([payload], { type: 'image/jpeg' }))
+            .then(bmp => { ctx.drawImage(bmp, 0, 0, canvas.width, canvas.height); });
+        } else if (type === 2 && pcmQueue) {
+          pcmQueue.push(payload.buffer);
+        }
+      };
+
+      const stream = new MediaStream(tracks);
+      // Close WebSocket when all tracks are stopped (camera/mic off)
+      for (const track of tracks) {
+        track.addEventListener('ended', () => {
+          if (stream.getTracks().every(t => t.readyState === 'ended')) {
+            ws.close();
+          }
+        });
+      }
+      resolve(stream);
+    });
   };
-  navigator.mediaDevices.getUserMedia = fakeStream;
   navigator.mediaDevices.enumerateDevices = () => Promise.resolve([
     { deviceId: 'default', kind: 'audioinput', label: 'Default Microphone', groupId: 'default' },
     { deviceId: 'default', kind: 'videoinput', label: 'Default Camera', groupId: 'default' },
@@ -178,7 +224,7 @@ export async function startBrowser() {
 
 // Phase 2: CDP connection + page setup (navigation is done by the caller)
 // Tries /json/new first, then /json/list, then Target.createTarget via browser CDP
-export async function setupPage({ port, wsUrl }, { width, height, zoom = 1 } = {}) {
+export async function setupPage({ port, wsUrl }, { width, height, zoom = 1, mediaPort = 0 } = {}) {
   const config = loadConfig();
   const lang = config.language;
   const baseUrl = `http://127.0.0.1:${port}`;
@@ -240,7 +286,7 @@ export async function setupPage({ port, wsUrl }, { width, height, zoom = 1 } = {
         wow64: false,
       },
     }),
-    client.send('Page.addScriptToEvaluateOnNewDocument', { source: buildStealthScript(lang, { fakeMedia: config.fakeMedia }) }),
+    client.send('Page.addScriptToEvaluateOnNewDocument', { source: buildStealthScript(lang, { media: config.media, mediaPort }) }),
     client.send('Browser.setDownloadBehavior', {
       behavior: 'allowAndName', downloadPath: join(homedir(), 'Downloads'),
       eventsEnabled: true,

package/lib/config.js

@@ -32,7 +32,7 @@ const DEFAULTS = {
   searchUrl: 'https://www.google.com/search?q=',
   transport: 'auto', // 'auto' | 'file' | 'inline'
   format: 'auto',    // 'auto' | 'png' | 'jpeg'
-  fakeMedia: false,  // Emulate camera/mic for WebRTC (headless-shell lacks getUserMedia)
+  media: false,      // Enable camera/mic via ffmpeg for WebRTC (requires ffmpeg)
   language: detectLocale(), // System locale (override: "en-US", "ja", etc.)
 };
 

package/lib/input.js

@@ -302,6 +302,7 @@ export function startInputHandling(client, cellWidth, cellHeight, bindings, paus
 
   // Throttled mouse motion (mode 1003 generates events for every pixel)
   let pendingMotion = null;
+  let lastClickTime = 0, lastClickX = 0, lastClickY = 0, lastClickCount = 0;
   let _motionTimer = null;
   const MOTION_INTERVAL = 16; // ms (~60fps)
   function flushMotion() {
@@ -511,7 +512,21 @@ export function startInputHandling(client, cellWidth, cellHeight, bindings, paus
         if (release) {
           await dispatchMouse(client, 'mouseReleased', x, y, 'left');
         } else {
-          await dispatchMouse(client, 'mousePressed', x, y, 'left', 1);
+          // Detect double/triple click by timing and position
+          const now = Date.now();
+          const MULTI_CLICK_MS = 400;
+          const MULTI_CLICK_PX = 5;
+          if (now - lastClickTime < MULTI_CLICK_MS
+              && Math.abs(x - lastClickX) < MULTI_CLICK_PX
+              && Math.abs(y - lastClickY) < MULTI_CLICK_PX) {
+            lastClickCount = Math.min(lastClickCount + 1, 3);
+          } else {
+            lastClickCount = 1;
+          }
+          lastClickTime = now;
+          lastClickX = x;
+          lastClickY = y;
+          await dispatchMouse(client, 'mousePressed', x, y, 'left', lastClickCount);
         }
       } else if (cb === MOUSE_BTN_DRAG) {
         await dispatchMouse(client, 'mouseMoved', x, y, 'left');

package/lib/media.js

@@ -0,0 +1,202 @@
+// Real camera/mic capture via ffmpeg → WebSocket relay
+// Two ffmpeg processes (video MJPEG, audio PCM) piped to a local WS server
+
+import { spawn, execFileSync } from 'node:child_process';
+import { existsSync } from 'node:fs';
+import { WebSocketServer } from 'ws';
+
+const IS_MAC = process.platform === 'darwin';
+
+// Detect available devices via ffmpeg
+function listDevices() {
+  const video = detectVideoDevice();
+  const audio = detectAudioDevice();
+  return { video, audio };
+}
+
+function detectVideoDevice() {
+  if (IS_MAC) {
+    const out = ffmpegProbe(['-f', 'avfoundation', '-list_devices', 'true', '-i', '']);
+    // Parse first video device from "AVFoundation video devices:" section (language-independent)
+    const videoSection = out.split(/audio devices/i)[0];
+    const m = videoSection.match(/\[(\d+)\]/);
+    if (!m) return null;
+    const idx = m[1];
+    // Probe supported framerate by attempting to open with invalid fps
+    const probe = ffmpegProbe(['-f', 'avfoundation', '-framerate', '1', '-video_size', '640x480', '-i', idx]);
+    const fpsMatch = probe.match(/640x480@\[(\d+)/);
+    const fps = fpsMatch ? parseInt(fpsMatch[1]) : 30;
+    return { idx, fps };
+  }
+  return existsSync('/dev/video0') ? '/dev/video0' : null;
+}
+
+function detectAudioDevice() {
+  if (IS_MAC) {
+    const out = ffmpegProbe(['-f', 'avfoundation', '-list_devices', 'true', '-i', '']);
+    // Parse first audio device from "AVFoundation audio devices:" section (language-independent)
+    const audioSection = out.split(/audio devices/i)[1];
+    const m = audioSection && audioSection.match(/\[(\d+)\]/);
+    return m ? ':' + m[1] : null;
+  }
+  // Linux: find first PulseAudio input source (not monitor)
+  try {
+    const out = execFileSync('pactl', ['list', 'sources', 'short'], { encoding: 'utf8' });
+    for (const line of out.trim().split('\n')) {
+      const parts = line.split('\t');
+      if (parts[1] && !parts[1].includes('.monitor')) return parts[1];
+    }
+  } catch {}
+  return 'default';
+}
+
+function ffmpegProbe(args) {
+  try {
+    execFileSync('ffmpeg', args, { encoding: 'utf8', stdio: ['pipe', 'pipe', 'pipe'] });
+  } catch (e) {
+    return (e.stderr || '') + (e.stdout || '');
+  }
+  return '';
+}
+
+// Parse MJPEG stream: extract individual JPEG frames from continuous byte stream
+function createMjpegParser(onFrame) {
+  let buf = Buffer.alloc(0);
+  return (chunk) => {
+    buf = Buffer.concat([buf, chunk]);
+    for (;;) {
+      // Find SOI marker (0xFF 0xD8)
+      const soi = buf.indexOf(Buffer.from([0xff, 0xd8]));
+      if (soi < 0) { buf = Buffer.alloc(0); return; }
+      if (soi > 0) buf = buf.subarray(soi);
+      // Find EOI marker (0xFF 0xD9) after SOI
+      const eoi = buf.indexOf(Buffer.from([0xff, 0xd9]), 2);
+      if (eoi < 0) return; // Incomplete frame
+      const frame = buf.subarray(0, eoi + 2);
+      onFrame(frame);
+      buf = buf.subarray(eoi + 2);
+    }
+  };
+}
+
+// Start media capture and WebSocket relay
+export async function startMedia(config = {}) {
+  // Check ffmpeg availability
+  try {
+    execFileSync('ffmpeg', ['-version'], { stdio: 'pipe' });
+  } catch {
+    console.error('casty: ffmpeg is required for media capture.');
+    if (IS_MAC) console.error('casty: Install with: brew install ffmpeg');
+    else console.error('casty: Install with: sudo apt install ffmpeg');
+    return { port: 0, cleanup: () => {} };
+  }
+
+  const devices = listDevices();
+  const videoRaw = config.videoDevice || devices.video;
+  const audioDevice = config.audioDevice || devices.audio;
+
+  // Normalize video device: { idx, fps } on macOS, string on Linux
+  const videoDevice = videoRaw && typeof videoRaw === 'object' ? videoRaw.idx : videoRaw;
+  const videoFps = videoRaw && typeof videoRaw === 'object' ? videoRaw.fps : 15;
+
+  if (!videoDevice && !audioDevice) {
+    console.error('casty: no media devices found');
+    return { port: 0, cleanup: () => {} };
+  }
+
+  // Start WebSocket server on random port
+  const wss = new WebSocketServer({ host: '127.0.0.1', port: 0 });
+  const port = await new Promise(resolve => {
+    wss.on('listening', () => resolve(wss.address().port));
+  });
+
+  const clients = new Set();
+  const procs = [];
+  let capturing = false;
+
+  function broadcast(prefix, data) {
+    const msg = Buffer.concat([Buffer.from([prefix]), data]);
+    for (const ws of clients) {
+      if (ws.readyState === 1) ws.send(msg);
+    }
+  }
+
+  // Start ffmpeg capture on demand (first WebSocket connection = getUserMedia called)
+  function startCapture() {
+    if (capturing) return;
+    capturing = true;
+    console.error('casty: media requested, starting capture');
+
+    // Video capture
+    if (videoDevice) {
+      const vargs = IS_MAC
+        ? ['-f', 'avfoundation', '-framerate', String(videoFps), '-video_size', '640x480', '-i', videoDevice]
+        : ['-f', 'v4l2', '-framerate', '15', '-video_size', '640x480', '-i', videoDevice];
+      vargs.push('-f', 'mjpeg', '-q:v', '5', '-r', '15', 'pipe:1');
+
+      const vproc = spawn('ffmpeg', vargs, { stdio: ['pipe', 'pipe', 'pipe'] });
+      const parse = createMjpegParser((frame) => broadcast(0x01, frame));
+      vproc.stdout.on('data', parse);
+      vproc.stderr.on('data', () => {}); // Suppress
+      vproc.on('error', (e) => console.error('casty: video capture error:', e.message));
+      procs.push(vproc);
+      console.error(`casty: video capture started (${videoDevice})`);
+    }
+
+    // Audio capture
+    if (audioDevice) {
+      const aargs = IS_MAC
+        ? ['-f', 'avfoundation', '-i', audioDevice]
+        : ['-f', 'pulse', '-i', audioDevice];
+      aargs.push('-f', 's16le', '-ar', '48000', '-ac', '1', 'pipe:1');
+
+      const env = { ...process.env };
+      if (!env.XDG_RUNTIME_DIR) env.XDG_RUNTIME_DIR = `/run/user/${process.getuid()}`;
+      const aproc = spawn('ffmpeg', aargs, { stdio: ['pipe', 'pipe', 'pipe'], env });
+      let pcmBuf = Buffer.alloc(0);
+      const PCM_CHUNK = 9600;
+      aproc.stdout.on('data', (chunk) => {
+        pcmBuf = Buffer.concat([pcmBuf, chunk]);
+        while (pcmBuf.length >= PCM_CHUNK) {
+          broadcast(0x02, pcmBuf.subarray(0, PCM_CHUNK));
+          pcmBuf = pcmBuf.subarray(PCM_CHUNK);
+        }
+      });
+      aproc.stderr.on('data', () => {}); // Suppress
+      aproc.on('error', (e) => console.error('casty: audio capture error:', e.message));
+      procs.push(aproc);
+      console.error(`casty: audio capture started (${audioDevice})`);
+    }
+  }
+
+  function stopCapture() {
+    if (!capturing) return;
+    for (const p of procs) {
+      try { p.kill(); } catch {}
+    }
+    procs.length = 0;
+    capturing = false;
+    console.error('casty: media stopped (no active sessions)');
+  }
+
+  // Start capture when first client connects, stop when all disconnect
+  wss.on('connection', (ws) => {
+    clients.add(ws);
+    startCapture();
+    ws.on('close', () => {
+      clients.delete(ws);
+      if (clients.size === 0) stopCapture();
+    });
+  });
+
+  console.error('casty: media ready (capture starts on demand)');
+
+  function cleanup() {
+    for (const p of procs) {
+      try { p.kill(); } catch {}
+    }
+    wss.close();
+  }
+
+  return { port, cleanup };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sanohiro/casty",
-  "version": "1.1.2",
+  "version": "1.2.1",
   "description": "TTY web browser using raw CDP and Kitty graphics protocol",
   "main": "bin/casty.js",
   "bin": {