linkshell-cli 0.1.15 → 0.2.0

package/src/runtime/bridge-session.ts

@@ -11,6 +11,8 @@ import {
 import type { Envelope } from "@linkshell/protocol";
 import type { ProviderConfig } from "../providers.js";
 import { ScrollbackBuffer } from "./scrollback.js";
+import { ScreenFallback } from "./screen-fallback.js";
+import { ScreenShare } from "./screen-share.js";
 import { getLanIp } from "../utils/lan-ip.js";
 
 export interface BridgeSessionOptions {
@@ -21,7 +23,9 @@ export interface BridgeSessionOptions {
   cols: number;
   rows: number;
   clientName: string;
+  hostname?: string;
   verbose?: boolean;
+  screen?: boolean;
   providerConfig: ProviderConfig;
 }
 
@@ -100,6 +104,8 @@ export class BridgeSession {
   private sessionId = "";
   private exited = false;
   private stopped = false;
+  private screenCapture: ScreenFallback | undefined;
+  private screenShare: ScreenShare | undefined;
 
   constructor(options: BridgeSessionOptions) {
     this.options = options;
@@ -208,7 +214,7 @@ export class BridgeSession {
             clientName: this.options.clientName,
             provider: this.options.providerConfig.provider,
             protocolVersion: PROTOCOL_VERSION,
-            hostname: hostname(),
+            hostname: this.options.hostname || hostname(),
             platform: platform(),
           },
         }),
@@ -266,6 +272,25 @@ export class BridgeSession {
       }
       case "session.heartbeat":
         break;
+      case "screen.start": {
+        const p = parseTypedPayload("screen.start", envelope.payload);
+        this.startScreenCapture(p.fps, p.quality, p.scale);
+        break;
+      }
+      case "screen.stop": {
+        this.stopScreenCapture();
+        break;
+      }
+      case "screen.answer": {
+        const p = parseTypedPayload("screen.answer", envelope.payload);
+        this.screenShare?.handleAnswer(p.sdp);
+        break;
+      }
+      case "screen.ice": {
+        const p = parseTypedPayload("screen.ice", envelope.payload);
+        this.screenShare?.handleIceCandidate(p.candidate, p.sdpMid, p.sdpMLineIndex);
+        break;
+      }
       default:
         break;
     }
@@ -372,6 +397,68 @@ export class BridgeSession {
     }
   }
 
+  private startScreenCapture(fps: number, quality: number, scale: number): void {
+    if (!this.options.screen) {
+      this.log("screen sharing not enabled (use --screen)");
+      this.send(
+        createEnvelope({
+          type: "screen.status",
+          sessionId: this.sessionId,
+          payload: { active: false, mode: "off" as const, error: "Screen sharing not enabled on host. Start CLI with --screen flag." },
+        }),
+      );
+      return;
+    }
+    this.stopScreenCapture();
+    this.log(`starting screen capture (fps=${fps}, quality=${quality}, scale=${scale})`);
+
+    // Try WebRTC first, fall back to screenshot stream
+    if (ScreenShare.isAvailable()) {
+      this.log("WebRTC available, starting screen share");
+      this.screenShare = new ScreenShare({
+        sessionId: this.sessionId,
+        fps,
+        quality,
+        scale,
+        onSignal: (envelope) => this.send(envelope),
+        onStatus: (envelope) => this.send(envelope),
+      });
+      this.screenShare.start().catch((err) => {
+        this.log(`WebRTC failed, falling back to screenshot stream: ${err}`);
+        this.screenShare = undefined;
+        this.startFallbackCapture(fps, quality, scale);
+      });
+    } else {
+      this.log("WebRTC not available (missing werift or ffmpeg), using screenshot fallback");
+      this.startFallbackCapture(fps, quality, scale);
+    }
+  }
+
+  private startFallbackCapture(fps: number, quality: number, scale: number): void {
+    this.screenCapture = new ScreenFallback({
+      fps,
+      quality,
+      scale,
+      sessionId: this.sessionId,
+      onFrame: (envelope) => this.send(envelope),
+      onStatus: (envelope) => this.send(envelope),
+    });
+    this.screenCapture.start();
+  }
+
+  private stopScreenCapture(): void {
+    if (this.screenShare) {
+      this.log("stopping WebRTC screen share");
+      this.screenShare.stop();
+      this.screenShare = undefined;
+    }
+    if (this.screenCapture) {
+      this.log("stopping screenshot capture");
+      this.screenCapture.stop();
+      this.screenCapture = undefined;
+    }
+  }
+
   private scheduleReconnect(): void {
     if (this.reconnecting || this.reconnectAttempts >= RECONNECT_MAX_ATTEMPTS) {
       process.stderr.write(
@@ -407,6 +494,7 @@ export class BridgeSession {
     this.stopped = true;
     this.exited = true;
     this.stopHeartbeat();
+    this.stopScreenCapture();
     if (this.reconnectTimer) {
       clearTimeout(this.reconnectTimer);
       this.reconnectTimer = undefined;

package/src/runtime/screen-fallback.ts

@@ -0,0 +1,235 @@
+import { execSync, spawn } from "node:child_process";
+import { readFileSync, unlinkSync, existsSync } from "node:fs";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { platform } from "node:os";
+import { createEnvelope, serializeEnvelope } from "@linkshell/protocol";
+import type { Envelope } from "@linkshell/protocol";
+
+export interface ScreenFallbackOptions {
+  fps: number;
+  quality: number;
+  scale: number;
+  sessionId: string;
+  onFrame: (envelope: Envelope) => void;
+  onStatus: (envelope: Envelope) => void;
+}
+
+const CHUNK_SIZE = 48 * 1024; // 48KB per chunk, leave room for envelope overhead
+const TMP_FILE = join(tmpdir(), `linkshell-screen-${process.pid}.jpg`);
+
+export class ScreenFallback {
+  private timer: ReturnType<typeof setInterval> | undefined;
+  private frameId = 0;
+  private active = false;
+  private readonly options: ScreenFallbackOptions;
+
+  constructor(options: ScreenFallbackOptions) {
+    this.options = options;
+  }
+
+  start(): void {
+    if (this.active) return;
+    this.active = true;
+    this.frameId = 0;
+
+    const interval = Math.max(50, Math.floor(1000 / this.options.fps));
+
+    this.options.onStatus(
+      createEnvelope({
+        type: "screen.status",
+        sessionId: this.options.sessionId,
+        payload: { active: true, mode: "fallback" as const },
+      }),
+    );
+
+    // Capture first frame immediately
+    this.captureAndSend();
+
+    this.timer = setInterval(() => {
+      this.captureAndSend();
+    }, interval);
+  }
+
+  stop(): void {
+    if (!this.active) return;
+    this.active = false;
+    if (this.timer) {
+      clearInterval(this.timer);
+      this.timer = undefined;
+    }
+    // Clean up temp file
+    try { if (existsSync(TMP_FILE)) unlinkSync(TMP_FILE); } catch {}
+
+    this.options.onStatus(
+      createEnvelope({
+        type: "screen.status",
+        sessionId: this.options.sessionId,
+        payload: { active: false, mode: "off" as const },
+      }),
+    );
+  }
+
+  private captureAndSend(): void {
+    if (!this.active) return;
+
+    try {
+      const imgBuffer = this.captureScreen();
+      if (!imgBuffer || imgBuffer.length === 0) return;
+
+      const base64 = imgBuffer.toString("base64");
+      const frameId = this.frameId++;
+
+      // Get dimensions (approximate from JPEG header or use defaults)
+      const dims = this.getJpegDimensions(imgBuffer);
+
+      if (base64.length <= CHUNK_SIZE) {
+        // Single chunk
+        this.options.onFrame(
+          createEnvelope({
+            type: "screen.frame",
+            sessionId: this.options.sessionId,
+            payload: {
+              data: base64,
+              width: dims.width,
+              height: dims.height,
+              frameId,
+              chunkIndex: 0,
+              chunkTotal: 1,
+            },
+          }),
+        );
+      } else {
+        // Split into chunks
+        const chunkTotal = Math.ceil(base64.length / CHUNK_SIZE);
+        for (let i = 0; i < chunkTotal; i++) {
+          if (!this.active) return;
+          const chunk = base64.slice(i * CHUNK_SIZE, (i + 1) * CHUNK_SIZE);
+          this.options.onFrame(
+            createEnvelope({
+              type: "screen.frame",
+              sessionId: this.options.sessionId,
+              payload: {
+                data: chunk,
+                width: dims.width,
+                height: dims.height,
+                frameId,
+                chunkIndex: i,
+                chunkTotal,
+              },
+            }),
+          );
+        }
+      }
+    } catch {
+      // Silently skip failed frames
+    }
+  }
+
+  private captureScreen(): Buffer | null {
+    const os = platform();
+    const q = this.options.quality;
+    const s = this.options.scale;
+
+    try {
+      if (os === "darwin") {
+        // macOS: screencapture
+        execSync(
+          `screencapture -x -t jpg -C "${TMP_FILE}"`,
+          { timeout: 3000, stdio: "pipe" },
+        );
+        if (!existsSync(TMP_FILE)) return null;
+        let buf = readFileSync(TMP_FILE);
+
+        // Resize if scale < 1 and sips is available
+        if (s < 1) {
+          try {
+            const w = Math.floor(this.getJpegDimensions(buf).width * s);
+            execSync(
+              `sips --resampleWidth ${w} -s formatOptions ${q} "${TMP_FILE}" --out "${TMP_FILE}"`,
+              { timeout: 3000, stdio: "pipe" },
+            );
+            buf = readFileSync(TMP_FILE);
+          } catch {
+            // Use original size if resize fails
+          }
+        }
+        return buf;
+      }
+
+      if (os === "linux") {
+        // Linux: try import (ImageMagick) or scrot
+        try {
+          const resizeArg = s < 1 ? `-resize ${Math.floor(s * 100)}%` : "";
+          execSync(
+            `import -window root -quality ${q} ${resizeArg} jpeg:"${TMP_FILE}"`,
+            { timeout: 3000, stdio: "pipe" },
+          );
+        } catch {
+          // Fallback to scrot
+          execSync(
+            `scrot -q ${q} "${TMP_FILE}"`,
+            { timeout: 3000, stdio: "pipe" },
+          );
+        }
+        if (!existsSync(TMP_FILE)) return null;
+        return readFileSync(TMP_FILE);
+      }
+
+      // Windows: use PowerShell
+      if (os === "win32") {
+        execSync(
+          `powershell -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.Screen]::PrimaryScreen | ForEach-Object { $bmp = New-Object System.Drawing.Bitmap($_.Bounds.Width, $_.Bounds.Height); $g = [System.Drawing.Graphics]::FromImage($bmp); $g.CopyFromScreen($_.Bounds.Location, [System.Drawing.Point]::Empty, $_.Bounds.Size); $bmp.Save('${TMP_FILE.replace(/\\/g, "\\\\")}', [System.Drawing.Imaging.ImageFormat]::Jpeg) }"`,
+          { timeout: 5000, stdio: "pipe" },
+        );
+        if (!existsSync(TMP_FILE)) return null;
+        return readFileSync(TMP_FILE);
+      }
+
+      return null;
+    } catch {
+      return null;
+    }
+  }
+
+  private getJpegDimensions(buf: Buffer): { width: number; height: number } {
+    // Parse JPEG SOF marker for dimensions
+    try {
+      let i = 2; // Skip SOI
+      while (i < buf.length - 8) {
+        if (buf[i] !== 0xff) break;
+        const marker = buf[i + 1]!;
+        // SOF0, SOF1, SOF2
+        if (marker >= 0xc0 && marker <= 0xc2) {
+          const height = buf.readUInt16BE(i + 5);
+          const width = buf.readUInt16BE(i + 7);
+          return { width, height };
+        }
+        const len = buf.readUInt16BE(i + 2);
+        i += 2 + len;
+      }
+    } catch {}
+    return { width: 1920, height: 1080 }; // Default fallback
+  }
+
+  static isAvailable(): boolean {
+    const os = platform();
+    try {
+      if (os === "darwin") {
+        execSync("which screencapture", { stdio: "pipe" });
+        return true;
+      }
+      if (os === "linux") {
+        try {
+          execSync("which import", { stdio: "pipe" });
+          return true;
+        } catch {
+          execSync("which scrot", { stdio: "pipe" });
+          return true;
+        }
+      }
+      if (os === "win32") return true; // PowerShell always available
+    } catch {}
+    return false;
+  }
+}

package/src/runtime/screen-share.ts

@@ -0,0 +1,283 @@
+import { spawn, execSync } from "node:child_process";
+import type { ChildProcess } from "node:child_process";
+import { platform } from "node:os";
+import { createEnvelope } from "@linkshell/protocol";
+import type { Envelope } from "@linkshell/protocol";
+
+export interface ScreenShareOptions {
+  sessionId: string;
+  fps: number;
+  quality: number;
+  scale: number;
+  turnUrl?: string;
+  turnUser?: string;
+  turnPass?: string;
+  onSignal: (envelope: Envelope) => void;
+  onStatus: (envelope: Envelope) => void;
+}
+
+/**
+ * WebRTC screen sharing using werift (pure TS WebRTC) + ffmpeg.
+ *
+ * Flow:
+ * 1. ffmpeg captures screen → H.264 → writes to stdout as raw annexb
+ * 2. We read H.264 NAL units and push them into werift video track
+ * 3. werift handles ICE/DTLS/SRTP and sends to remote peer
+ * 4. Signaling (SDP offer/answer, ICE candidates) goes through existing WebSocket
+ */
+export class ScreenShare {
+  private pc: any; // werift.RTCPeerConnection
+  private ffmpeg: ChildProcess | undefined;
+  private active = false;
+  private readonly options: ScreenShareOptions;
+
+  constructor(options: ScreenShareOptions) {
+    this.options = options;
+  }
+
+  static isAvailable(): boolean {
+    // Check if werift can be imported and ffmpeg exists
+    try {
+      execSync("which ffmpeg", { stdio: "pipe" });
+    } catch {
+      return false;
+    }
+    try {
+      require.resolve("werift");
+      return true;
+    } catch {
+      return false;
+    }
+  }
+
+  async start(): Promise<void> {
+    if (this.active) return;
+    this.active = true;
+
+    try {
+      const werift = await import("werift");
+
+      // ICE servers
+      const iceServers: any[] = [
+        { urls: "stun:stun.l.google.com:19302" },
+      ];
+      if (this.options.turnUrl) {
+        iceServers.push({
+          urls: this.options.turnUrl,
+          username: this.options.turnUser ?? "",
+          credential: this.options.turnPass ?? "",
+        });
+      }
+
+      this.pc = new werift.RTCPeerConnection({
+        iceServers,
+        bundlePolicy: "max-bundle",
+      });
+
+      // Create video track
+      const videoTrack = new werift.MediaStreamTrack({ kind: "video" });
+      this.pc.addTrack(videoTrack);
+
+      // ICE candidate → send to remote
+      this.pc.onIceCandidate.subscribe((candidate: any) => {
+        if (candidate) {
+          this.options.onSignal(
+            createEnvelope({
+              type: "screen.ice",
+              sessionId: this.options.sessionId,
+              payload: {
+                candidate: candidate.candidate,
+                sdpMid: candidate.sdpMid ?? null,
+                sdpMLineIndex: candidate.sdpMLineIndex ?? null,
+              },
+            }),
+          );
+        }
+      });
+
+      // Connection state changes
+      this.pc.connectionStateChange.subscribe((state: string) => {
+        if (state === "connected") {
+          this.options.onStatus(
+            createEnvelope({
+              type: "screen.status",
+              sessionId: this.options.sessionId,
+              payload: { active: true, mode: "webrtc" as const },
+            }),
+          );
+        } else if (state === "failed" || state === "disconnected" || state === "closed") {
+          this.options.onStatus(
+            createEnvelope({
+              type: "screen.status",
+              sessionId: this.options.sessionId,
+              payload: { active: false, mode: "off" as const, error: `WebRTC ${state}` },
+            }),
+          );
+        }
+      });
+
+      // Create offer
+      const offer = await this.pc.createOffer();
+      await this.pc.setLocalDescription(offer);
+
+      this.options.onSignal(
+        createEnvelope({
+          type: "screen.offer",
+          sessionId: this.options.sessionId,
+          payload: { sdp: offer.sdp },
+        }),
+      );
+
+      // Start ffmpeg capture (will begin sending once ICE connects)
+      this.startFfmpeg(videoTrack);
+
+    } catch (err) {
+      this.options.onStatus(
+        createEnvelope({
+          type: "screen.status",
+          sessionId: this.options.sessionId,
+          payload: {
+            active: false,
+            mode: "off" as const,
+            error: `WebRTC init failed: ${err instanceof Error ? err.message : String(err)}`,
+          },
+        }),
+      );
+      this.active = false;
+    }
+  }
+
+  async handleAnswer(sdp: string): Promise<void> {
+    if (!this.pc) return;
+    try {
+      const werift = await import("werift");
+      await this.pc.setRemoteDescription(
+        new werift.RTCSessionDescription(sdp, "answer"),
+      );
+    } catch (err) {
+      process.stderr.write(`[screen-share] failed to set answer: ${err}\n`);
+    }
+  }
+
+  async handleIceCandidate(candidate: string, sdpMid?: string | null, sdpMLineIndex?: number | null): Promise<void> {
+    if (!this.pc) return;
+    try {
+      const werift = await import("werift");
+      await this.pc.addIceCandidate(
+        new werift.RTCIceCandidate({ candidate, sdpMid: sdpMid ?? undefined, sdpMLineIndex: sdpMLineIndex ?? undefined }),
+      );
+    } catch (err) {
+      process.stderr.write(`[screen-share] failed to add ICE candidate: ${err}\n`);
+    }
+  }
+
+  stop(): void {
+    this.active = false;
+    if (this.ffmpeg) {
+      this.ffmpeg.kill("SIGTERM");
+      this.ffmpeg = undefined;
+    }
+    if (this.pc) {
+      try { this.pc.close(); } catch {}
+      this.pc = undefined;
+    }
+    this.options.onStatus(
+      createEnvelope({
+        type: "screen.status",
+        sessionId: this.options.sessionId,
+        payload: { active: false, mode: "off" as const },
+      }),
+    );
+  }
+
+  private startFfmpeg(videoTrack: any): void {
+    const os = platform();
+    const fps = this.options.fps;
+    const scale = this.options.scale;
+
+    let inputArgs: string[];
+    if (os === "darwin") {
+      // macOS: AVFoundation screen capture
+      inputArgs = ["-f", "avfoundation", "-framerate", String(fps), "-i", "1:none"];
+    } else if (os === "linux") {
+      // Linux: X11 screen grab
+      inputArgs = ["-f", "x11grab", "-framerate", String(fps), "-i", ":0"];
+    } else {
+      // Windows: GDI screen grab
+      inputArgs = ["-f", "gdigrab", "-framerate", String(fps), "-i", "desktop"];
+    }
+
+    const scaleFilter = scale < 1 ? ["-vf", `scale=iw*${scale}:ih*${scale}`] : [];
+
+    this.ffmpeg = spawn("ffmpeg", [
+      ...inputArgs,
+      ...scaleFilter,
+      "-c:v", "libx264",
+      "-preset", "ultrafast",
+      "-tune", "zerolatency",
+      "-profile:v", "baseline",
+      "-level", "3.1",
+      "-pix_fmt", "yuv420p",
+      "-g", String(fps * 2), // keyframe every 2 seconds
+      "-b:v", "1500k",
+      "-maxrate", "2000k",
+      "-bufsize", "4000k",
+      "-f", "h264",
+      "-an", // no audio
+      "pipe:1",
+    ], {
+      stdio: ["pipe", "pipe", "pipe"],
+    });
+
+    // Read H.264 NAL units from stdout and push to video track
+    let nalBuffer = Buffer.alloc(0);
+
+    this.ffmpeg.stdout?.on("data", (chunk: Buffer) => {
+      if (!this.active) return;
+
+      nalBuffer = Buffer.concat([nalBuffer, chunk]);
+
+      // Split on NAL unit start codes (0x00000001 or 0x000001)
+      let offset = 0;
+      while (offset < nalBuffer.length - 4) {
+        let startCodeLen = 0;
+        if (nalBuffer[offset] === 0 && nalBuffer[offset + 1] === 0) {
+          if (nalBuffer[offset + 2] === 0 && nalBuffer[offset + 3] === 1) {
+            startCodeLen = 4;
+          } else if (nalBuffer[offset + 2] === 1) {
+            startCodeLen = 3;
+          }
+        }
+
+        if (startCodeLen > 0 && offset > 0) {
+          // Found a NAL unit boundary
+          const nalUnit = nalBuffer.subarray(0, offset);
+          try {
+            videoTrack.writeRtp(nalUnit);
+          } catch {
+            // Track might not be ready yet
+          }
+          nalBuffer = nalBuffer.subarray(offset);
+          offset = 0;
+          continue;
+        }
+        offset++;
+      }
+    });
+
+    this.ffmpeg.stderr?.on("data", (data: Buffer) => {
+      // ffmpeg logs to stderr, ignore unless debugging
+      const msg = data.toString();
+      if (msg.includes("Error") || msg.includes("error")) {
+        process.stderr.write(`[screen-share:ffmpeg] ${msg}\n`);
+      }
+    });
+
+    this.ffmpeg.on("exit", (code) => {
+      if (this.active) {
+        process.stderr.write(`[screen-share] ffmpeg exited with code ${code}\n`);
+        this.stop();
+      }
+    });
+  }
+}

package/src/types/werift.d.ts

@@ -0,0 +1,23 @@
+declare module "werift" {
+  export class RTCPeerConnection {
+    constructor(config?: any);
+    onIceCandidate: { subscribe: (cb: (candidate: any) => void) => void };
+    connectionStateChange: { subscribe: (cb: (state: string) => void) => void };
+    addTrack(track: MediaStreamTrack): void;
+    createOffer(): Promise<{ sdp: string; type: string }>;
+    setLocalDescription(desc: any): Promise<void>;
+    setRemoteDescription(desc: any): Promise<void>;
+    addIceCandidate(candidate: any): Promise<void>;
+    close(): void;
+  }
+  export class RTCSessionDescription {
+    constructor(sdp: string, type: string);
+  }
+  export class RTCIceCandidate {
+    constructor(init: { candidate: string; sdpMid?: string; sdpMLineIndex?: number });
+  }
+  export class MediaStreamTrack {
+    constructor(init: { kind: string });
+    writeRtp(data: Buffer): void;
+  }
+}