@agentconnect.md/computer 2.8.0-rc.23 → 2.8.0-rc.38

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -82903,6 +82903,8 @@ var MAC_COMMAND_KEYS = /* @__PURE__ */ new Set(["cmd", "command", "meta"]);
82903
82903
  var HUMAN_POLL_MS = 500;
82904
82904
  var MAX_TYPE_CHARS = 4096;
82905
82905
  var TYPE_REMAP_PAUSE_MS = 40;
82906
+ var VIEWER_SESSION = "viewer";
82907
+ var VIEWER_SESSION_REFRESH_MS = 4 * 6e4;
82906
82908
  var Computer = class {
82907
82909
  driver;
82908
82910
  config;
@@ -82929,6 +82931,8 @@ var Computer = class {
82929
82931
  // `by` is set when a remote human took over through the web viewer.
82930
82932
  paused;
82931
82933
  capture;
82934
+ // `cursor` is false where the driver refuses cursor settings (macOS has no overlay).
82935
+ viewerCapture;
82932
82936
  queue = Promise.resolve();
82933
82937
  busy = false;
82934
82938
  actions = 0;
@@ -83065,18 +83069,20 @@ var Computer = class {
83065
83069
  this.assertViewerControl(client);
83066
83070
  await this.exclusive(async () => {
83067
83071
  this.assertViewerControl(client);
83068
- if (!this.captureIsCurrent()) await this.captureScreen();
83069
- const { width, height } = this.capture;
83072
+ const { width, height } = await this.viewerSessionCapture();
83070
83073
  const px = (point) => ({
83071
83074
  x: Math.round(Math.min(Math.max(point.x, 0), 1) * (width - 1)),
83072
83075
  y: Math.round(Math.min(Math.max(point.y, 0), 1) * (height - 1))
83073
83076
  });
83074
- const call = viewerCall(input2, px);
83077
+ const viewer = viewerCall(input2, px);
83078
+ const call = { ...viewer, args: { ...viewer.args, session: VIEWER_SESSION } };
83075
83079
  this.assertViewerControl(client);
83076
83080
  await this.callDriver(call, () => {
83077
83081
  this.assertAuthorized();
83078
83082
  this.assertViewerControl(client);
83079
83083
  });
83084
+ if (this.viewerCapture?.cursor)
83085
+ await this.driver.call("set_agent_cursor_enabled", { session: VIEWER_SESSION, enabled: false });
83080
83086
  });
83081
83087
  if (input2.type !== "move")
83082
83088
  this.audit({ client, tool: `viewer_${input2.type}`, args, outcome: "ok", ms: this.now() - started });
@@ -83585,6 +83591,48 @@ var Computer = class {
83585
83591
  }
83586
83592
  return result;
83587
83593
  }
83594
+ // Sets up the viewer session (no glide, then a capture that maps its coordinates) whenever the driver may have lost it.
83595
+ async viewerSessionCapture() {
83596
+ const now = this.now();
83597
+ const current = this.viewerCapture;
83598
+ if (current?.generation === this.driver.generation() && now - current.usedAt < VIEWER_SESSION_REFRESH_MS) {
83599
+ current.usedAt = now;
83600
+ return current;
83601
+ }
83602
+ const session = VIEWER_SESSION;
83603
+ await this.driver.call("start_session", { session });
83604
+ let cursor = true;
83605
+ try {
83606
+ await this.driver.call("set_agent_cursor_motion", {
83607
+ session,
83608
+ timing: "fixed",
83609
+ glide_duration_ms: 1,
83610
+ dwell_after_click_ms: 0,
83611
+ effects: { glow: "off", ripple: "off", squish: "off", trail: "off", magnet: "off" }
83612
+ });
83613
+ } catch (error63) {
83614
+ const refusal = error63 instanceof DriverToolError ? error63.result.structuredContent?.refusal : void 0;
83615
+ if (refusal?.code !== "facility_unavailable") throw error63;
83616
+ cursor = false;
83617
+ }
83618
+ const result = await this.driver.call("get_desktop_state", {
83619
+ max_image_dimension: this.config.maxImageDimension,
83620
+ session
83621
+ });
83622
+ const meta3 = result.structuredContent ?? {};
83623
+ const num = (key) => typeof meta3[key] === "number" ? meta3[key] : 0;
83624
+ this.viewerCapture = {
83625
+ generation: this.driver.generation(),
83626
+ width: num("screenshot_width"),
83627
+ height: num("screenshot_height"),
83628
+ screenWidth: num("screen_width"),
83629
+ screenHeight: num("screen_height"),
83630
+ scale: num("scale_factor") || 1,
83631
+ usedAt: now,
83632
+ cursor
83633
+ };
83634
+ return this.viewerCapture;
83635
+ }
83588
83636
  captureIsCurrent() {
83589
83637
  return this.capture !== void 0 && this.capture.generation === this.driver.generation();
83590
83638
  }
@@ -87124,7 +87172,7 @@ var serviceEntry = () => managedEntry() ?? resolve2(process.argv[1]);
87124
87172
 
87125
87173
  // src/version.ts
87126
87174
  init_define_AGENTCONNECT_COMPUTER_WINDOWS_SOURCES();
87127
- var COMPUTER_DAEMON_VERSION = true ? "2.8.0-rc.23" : JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
87175
+ var COMPUTER_DAEMON_VERSION = true ? "2.8.0-rc.38" : JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
87128
87176
 
87129
87177
  // src/http-server.ts
87130
87178
  var CONTROL_HEADER = "x-agentconnect-computer";
@@ -87266,7 +87314,7 @@ import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "node:f
87266
87314
  import { join as join6 } from "node:path";
87267
87315
  var waylandSession = () => process.platform === "linux" && Boolean(process.env.WAYLAND_DISPLAY);
87268
87316
  var HELPER = String.raw`#!/usr/bin/env python3
87269
- # Usage: screencast.py TOKEN_FILE FPS [FRAMES]. Asks the ScreenCast portal for the monitor (persisting a restore
87317
+ # Usage: screencast.py TOKEN_FILE FPS CURSOR(1|0) [FRAMES]. Asks the ScreenCast portal for the monitor (persisting a restore
87270
87318
  # token so consent is asked once), then streams it as Y4M on stdout until killed.
87271
87319
  import os, signal, subprocess, sys
87272
87320
  import gi
@@ -87274,7 +87322,8 @@ gi.require_version('Gio', '2.0')
87274
87322
  from gi.repository import Gio, GLib
87275
87323
 
87276
87324
  token_path, fps = sys.argv[1], int(sys.argv[2])
87277
- frames = int(sys.argv[3]) if len(sys.argv) > 3 else 0
87325
+ show_cursor = sys.argv[3] == '1'
87326
+ frames = int(sys.argv[4]) if len(sys.argv) > 4 else 0
87278
87327
  NAME, PATH = 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop'
87279
87328
  IFACE = 'org.freedesktop.portal.ScreenCast'
87280
87329
  bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
@@ -87306,7 +87355,8 @@ def request(method, build):
87306
87355
  modes = bus.call_sync(NAME, PATH, 'org.freedesktop.DBus.Properties', 'Get',
87307
87356
  GLib.Variant('(ss)', (IFACE, 'AvailableCursorModes')), None, Gio.DBusCallFlags.NONE, -1,
87308
87357
  None).unpack()[0]
87309
- cursor = 2 if modes & 2 else 1
87358
+ # 2 embeds the pointer in the frames, 1 hides it.
87359
+ cursor = 2 if show_cursor and modes & 2 else 1
87310
87360
  session = request('CreateSession', lambda t: GLib.Variant('(a{sv})', (
87311
87361
  {'handle_token': t, 'session_handle_token': GLib.Variant('s', 'acd%d' % os.getpid())},)))['session_handle']
87312
87362
  options = {'types': GLib.Variant('u', 1), 'multiple': GLib.Variant('b', False),
@@ -87440,19 +87490,20 @@ async function downloadFfmpeg(dataDir2, target, log40) {
87440
87490
  log40(`Installed ffmpeg ${build3.version} at ${target}`);
87441
87491
  return target;
87442
87492
  }
87443
- function captureInputs(fps, dataDir2, frames = 0) {
87493
+ function captureInputs(fps, dataDir2, frames = 0, cursor = true) {
87494
+ const mouse = cursor ? "1" : "0";
87444
87495
  const override = process.env.AGENTCONNECT_COMPUTER_CAPTURE;
87445
87496
  if (override) return [{ name: "override", args: override.split(" ").filter(Boolean), filters: [] }];
87446
87497
  if (process.platform === "win32") {
87447
87498
  return [
87448
87499
  {
87449
87500
  name: "ddagrab",
87450
- args: ["-f", "lavfi", "-i", `ddagrab=output_idx=0:framerate=${fps}:draw_mouse=1`],
87501
+ args: ["-f", "lavfi", "-i", `ddagrab=output_idx=0:framerate=${fps}:draw_mouse=${mouse}`],
87451
87502
  filters: ["hwdownload", "format=bgra"]
87452
87503
  },
87453
87504
  {
87454
87505
  name: "gdigrab",
87455
- args: ["-f", "gdigrab", "-framerate", String(fps), "-draw_mouse", "1", "-i", "desktop"],
87506
+ args: ["-f", "gdigrab", "-framerate", String(fps), "-draw_mouse", mouse, "-i", "desktop"],
87456
87507
  filters: []
87457
87508
  }
87458
87509
  ];
@@ -87461,7 +87512,16 @@ function captureInputs(fps, dataDir2, frames = 0) {
87461
87512
  return [
87462
87513
  {
87463
87514
  name: "avfoundation",
87464
- args: ["-f", "avfoundation", "-capture_cursor", "1", "-framerate", String(fps), "-i", "Capture screen 0:none"],
87515
+ args: [
87516
+ "-f",
87517
+ "avfoundation",
87518
+ "-capture_cursor",
87519
+ mouse,
87520
+ "-framerate",
87521
+ String(fps),
87522
+ "-i",
87523
+ "Capture screen 0:none"
87524
+ ],
87465
87525
  filters: []
87466
87526
  }
87467
87527
  ];
@@ -87473,7 +87533,7 @@ function captureInputs(fps, dataDir2, frames = 0) {
87473
87533
  name: "pipewire",
87474
87534
  producer: {
87475
87535
  command: "python3",
87476
- args: [helper, screencastTokenPath(dataDir2), String(fps), ...frames ? [String(frames)] : []]
87536
+ args: [helper, screencastTokenPath(dataDir2), String(fps), mouse, ...frames ? [String(frames)] : []]
87477
87537
  },
87478
87538
  args: ["-f", "yuv4mpegpipe", "-i", "pipe:0"],
87479
87539
  filters: []
@@ -87483,7 +87543,7 @@ function captureInputs(fps, dataDir2, frames = 0) {
87483
87543
  return [
87484
87544
  {
87485
87545
  name: "x11grab",
87486
- args: ["-f", "x11grab", "-framerate", String(fps), "-i", process.env.DISPLAY ?? ":0"],
87546
+ args: ["-f", "x11grab", "-draw_mouse", mouse, "-framerate", String(fps), "-i", process.env.DISPLAY ?? ":0"],
87487
87547
  filters: []
87488
87548
  }
87489
87549
  ];
@@ -109989,7 +110049,7 @@ var RtcSessions = class {
109989
110049
  session.connected = true;
109990
110050
  session.lastSeen = Date.now();
109991
110051
  this.options.log(`viewer ${client()}: WebRTC connected (${transport}${selectedPairType(pc)})`);
109992
- session.unsubscribe ??= this.options.stream.subscribe((packet) => track.writeRtp(packet));
110052
+ session.unsubscribe ??= this.options.stream.subscribe(continuous(track));
109993
110053
  }
109994
110054
  if (state === "failed" || state === "closed" || state === "disconnected") this.close(session.id);
109995
110055
  });
@@ -110096,6 +110156,15 @@ function selectedPairType(pc) {
110096
110156
  }
110097
110157
  var PAIR_STATES = ["frozen", "waiting", "in-progress", "succeeded", "failed"];
110098
110158
  var endpoint = (c) => `${c.type} ${c.host}:${c.port}`;
110159
+ function continuous(track) {
110160
+ let source;
110161
+ return (packet) => {
110162
+ const ssrc = packet.readUInt32BE(8);
110163
+ if (source !== void 0 && ssrc !== source) track.notifySourceChanged(RtpPacket.deSerialize(packet).header);
110164
+ source = ssrc;
110165
+ track.writeRtp(packet);
110166
+ };
110167
+ }
110099
110168
  function iceReport(pc, trickled) {
110100
110169
  const connection = pc.iceTransports[0]?.connection;
110101
110170
  if (!connection) return "no ICE transport";
@@ -110135,6 +110204,9 @@ var ScreenStream = class {
110135
110204
  socket;
110136
110205
  process;
110137
110206
  restart;
110207
+ // The running capture's pointer setting, and the process being replaced because that setting changed.
110208
+ cursor = true;
110209
+ swapping;
110138
110210
  inputIndex = 0;
110139
110211
  running = false;
110140
110212
  runs = 0;
@@ -110170,7 +110242,8 @@ var ScreenStream = class {
110170
110242
  }
110171
110243
  spawn() {
110172
110244
  if (!this.running || !this.socket) return;
110173
- const inputs = captureInputs(this.options.fps, this.options.dataDir);
110245
+ this.cursor = this.options.cursor?.() ?? true;
110246
+ const inputs = captureInputs(this.options.fps, this.options.dataDir, 0, this.cursor);
110174
110247
  const input2 = inputs[Math.min(this.inputIndex, inputs.length - 1)];
110175
110248
  const args = [
110176
110249
  "-hide_banner",
@@ -110203,7 +110276,9 @@ var ScreenStream = class {
110203
110276
  `)));
110204
110277
  feeder.once("exit", () => this.children.delete(feeder));
110205
110278
  }
110206
- this.options.log(`screen stream: ffmpeg started (${input2.name}, ${this.options.fps} fps)`);
110279
+ this.options.log(
110280
+ `screen stream: ffmpeg started (${input2.name}, ${this.options.fps} fps${this.cursor ? "" : ", no pointer"})`
110281
+ );
110207
110282
  child2.once("error", (error63) => {
110208
110283
  this.lastError = `ffmpeg could not start: ${error63.message}`;
110209
110284
  this.options.log(`screen stream: ${this.lastError}`);
@@ -110213,6 +110288,10 @@ var ScreenStream = class {
110213
110288
  producer?.kill();
110214
110289
  if (this.process === child2) this.process = void 0;
110215
110290
  if (!this.running) return;
110291
+ if (this.swapping === child2) {
110292
+ this.swapping = void 0;
110293
+ return this.spawn();
110294
+ }
110216
110295
  const detail = stderr.trim().split("\n").slice(-3).join(" | ");
110217
110296
  this.lastError = `ffmpeg (${input2.name}) exited with ${code}: ${detail}`;
110218
110297
  this.options.log(`screen stream: ${this.lastError}`);
@@ -110220,6 +110299,13 @@ var ScreenStream = class {
110220
110299
  this.restart = setTimeout(() => this.spawn(), RESTART_DELAY_MS);
110221
110300
  });
110222
110301
  }
110302
+ // Restarts the capture when the pointer setting changed; the next keyframe carries the new frames.
110303
+ refresh() {
110304
+ const child2 = this.process;
110305
+ if (!child2 || this.swapping || (this.options.cursor?.() ?? true) === this.cursor) return;
110306
+ this.swapping = child2;
110307
+ terminate(child2);
110308
+ }
110223
110309
  get bytesPerMs() {
110224
110310
  return this.options.bitrateKbps * PACE_FACTOR / 8;
110225
110311
  }
@@ -110252,16 +110338,19 @@ var ScreenStream = class {
110252
110338
  this.queue.length = 0;
110253
110339
  this.queuedBytes = 0;
110254
110340
  clearTimeout(this.restart);
110341
+ this.swapping = void 0;
110255
110342
  const child2 = this.process;
110256
110343
  this.process = void 0;
110257
- child2?.kill();
110258
- if (child2)
110259
- setTimeout(() => child2.exitCode === null && child2.signalCode === null && child2.kill("SIGKILL"), 2e3).unref();
110344
+ if (child2) terminate(child2);
110260
110345
  this.socket?.close();
110261
110346
  this.socket = void 0;
110262
110347
  this.options.log("screen stream: stopped (no viewers)");
110263
110348
  }
110264
110349
  };
110350
+ function terminate(child2) {
110351
+ child2.kill();
110352
+ setTimeout(() => child2.exitCode === null && child2.signalCode === null && child2.kill("SIGKILL"), 2e3).unref();
110353
+ }
110265
110354
 
110266
110355
  // src/tools.ts
110267
110356
  init_define_AGENTCONNECT_COMPUTER_WINDOWS_SOURCES();
@@ -110620,7 +110709,7 @@ var VIEWER_PAGE = `<!doctype html>
110620
110709
  #screen.ctl { outline: 2px solid #4ade80; }
110621
110710
  video, img { max-width: 100vw; max-height: calc(100vh - 48px); display: block; }
110622
110711
  #surface { position: absolute; inset: 0; outline: none; }
110623
- #screen.ctl #surface { cursor: crosshair; }
110712
+ #screen.ctl #surface { cursor: default; }
110624
110713
  #login { display: none; gap: 8px; }
110625
110714
  #error { color: #f87171; }
110626
110715
  </style>
@@ -114528,6 +114617,8 @@ function buildViewer(dir, config2, computer, refusal, consoleViewer, advertiseAd
114528
114617
  fps: config2.streamFps,
114529
114618
  maxWidth: config2.streamMaxWidth,
114530
114619
  bitrateKbps: config2.streamBitrateKbps,
114620
+ // A person driving sees their browser's pointer at once, so the delayed one in the frames would be a second cursor.
114621
+ cursor: () => computer.liveStatus() !== "human_in_control",
114531
114622
  log: log39
114532
114623
  });
114533
114624
  const rtc = config2.webrtc ? new RtcSessions({
@@ -114550,7 +114641,7 @@ function buildViewer(dir, config2, computer, refusal, consoleViewer, advertiseAd
114550
114641
  streamError: () => stream.lastError,
114551
114642
  ...refusal ? { refusal } : {}
114552
114643
  });
114553
- return viewer;
114644
+ return { ...viewer, statusChanged: () => stream.refresh() };
114554
114645
  }
114555
114646
  async function loadDriverTools(driver, config2) {
114556
114647
  if (!config2.exposeDriverTools) return [];
@@ -114656,6 +114747,7 @@ async function run3(dir, initial, clientName) {
114656
114747
  setTitle(status);
114657
114748
  live2.channel?.client.beat();
114658
114749
  live2.local?.update();
114750
+ live2.viewer?.statusChanged();
114659
114751
  void live2.computer?.adminPromptChanged();
114660
114752
  },
114661
114753
  () => authority.refusal(),
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@agentconnect.md/computer",
3
- "version": "2.8.0-rc.23",
3
+ "version": "2.8.0-rc.38",
4
4
  "description": "AgentConnect computer daemon: drives a desktop through Cua Driver and serves the computer_* tools over MCP; installed and upgraded by agentconnect computer",
5
5
  "license": "Apache-2.0",
6
6
  "repository": {