@agentconnect.md/computer 2.8.0-rc.23 → 2.8.0-rc.38
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agentconnect-computer.mjs +113 -21
- package/package.json +1 -1
|
@@ -82903,6 +82903,8 @@ var MAC_COMMAND_KEYS = /* @__PURE__ */ new Set(["cmd", "command", "meta"]);
|
|
|
82903
82903
|
var HUMAN_POLL_MS = 500;
|
|
82904
82904
|
var MAX_TYPE_CHARS = 4096;
|
|
82905
82905
|
var TYPE_REMAP_PAUSE_MS = 40;
|
|
82906
|
+
var VIEWER_SESSION = "viewer";
|
|
82907
|
+
var VIEWER_SESSION_REFRESH_MS = 4 * 6e4;
|
|
82906
82908
|
var Computer = class {
|
|
82907
82909
|
driver;
|
|
82908
82910
|
config;
|
|
@@ -82929,6 +82931,8 @@ var Computer = class {
|
|
|
82929
82931
|
// `by` is set when a remote human took over through the web viewer.
|
|
82930
82932
|
paused;
|
|
82931
82933
|
capture;
|
|
82934
|
+
// `cursor` is false where the driver refuses cursor settings (macOS has no overlay).
|
|
82935
|
+
viewerCapture;
|
|
82932
82936
|
queue = Promise.resolve();
|
|
82933
82937
|
busy = false;
|
|
82934
82938
|
actions = 0;
|
|
@@ -83065,18 +83069,20 @@ var Computer = class {
|
|
|
83065
83069
|
this.assertViewerControl(client);
|
|
83066
83070
|
await this.exclusive(async () => {
|
|
83067
83071
|
this.assertViewerControl(client);
|
|
83068
|
-
|
|
83069
|
-
const { width, height } = this.capture;
|
|
83072
|
+
const { width, height } = await this.viewerSessionCapture();
|
|
83070
83073
|
const px = (point) => ({
|
|
83071
83074
|
x: Math.round(Math.min(Math.max(point.x, 0), 1) * (width - 1)),
|
|
83072
83075
|
y: Math.round(Math.min(Math.max(point.y, 0), 1) * (height - 1))
|
|
83073
83076
|
});
|
|
83074
|
-
const
|
|
83077
|
+
const viewer = viewerCall(input2, px);
|
|
83078
|
+
const call = { ...viewer, args: { ...viewer.args, session: VIEWER_SESSION } };
|
|
83075
83079
|
this.assertViewerControl(client);
|
|
83076
83080
|
await this.callDriver(call, () => {
|
|
83077
83081
|
this.assertAuthorized();
|
|
83078
83082
|
this.assertViewerControl(client);
|
|
83079
83083
|
});
|
|
83084
|
+
if (this.viewerCapture?.cursor)
|
|
83085
|
+
await this.driver.call("set_agent_cursor_enabled", { session: VIEWER_SESSION, enabled: false });
|
|
83080
83086
|
});
|
|
83081
83087
|
if (input2.type !== "move")
|
|
83082
83088
|
this.audit({ client, tool: `viewer_${input2.type}`, args, outcome: "ok", ms: this.now() - started });
|
|
@@ -83585,6 +83591,48 @@ var Computer = class {
|
|
|
83585
83591
|
}
|
|
83586
83592
|
return result;
|
|
83587
83593
|
}
|
|
83594
|
+
// Sets up the viewer session (no glide, then a capture that maps its coordinates) whenever the driver may have lost it.
|
|
83595
|
+
async viewerSessionCapture() {
|
|
83596
|
+
const now = this.now();
|
|
83597
|
+
const current = this.viewerCapture;
|
|
83598
|
+
if (current?.generation === this.driver.generation() && now - current.usedAt < VIEWER_SESSION_REFRESH_MS) {
|
|
83599
|
+
current.usedAt = now;
|
|
83600
|
+
return current;
|
|
83601
|
+
}
|
|
83602
|
+
const session = VIEWER_SESSION;
|
|
83603
|
+
await this.driver.call("start_session", { session });
|
|
83604
|
+
let cursor = true;
|
|
83605
|
+
try {
|
|
83606
|
+
await this.driver.call("set_agent_cursor_motion", {
|
|
83607
|
+
session,
|
|
83608
|
+
timing: "fixed",
|
|
83609
|
+
glide_duration_ms: 1,
|
|
83610
|
+
dwell_after_click_ms: 0,
|
|
83611
|
+
effects: { glow: "off", ripple: "off", squish: "off", trail: "off", magnet: "off" }
|
|
83612
|
+
});
|
|
83613
|
+
} catch (error63) {
|
|
83614
|
+
const refusal = error63 instanceof DriverToolError ? error63.result.structuredContent?.refusal : void 0;
|
|
83615
|
+
if (refusal?.code !== "facility_unavailable") throw error63;
|
|
83616
|
+
cursor = false;
|
|
83617
|
+
}
|
|
83618
|
+
const result = await this.driver.call("get_desktop_state", {
|
|
83619
|
+
max_image_dimension: this.config.maxImageDimension,
|
|
83620
|
+
session
|
|
83621
|
+
});
|
|
83622
|
+
const meta3 = result.structuredContent ?? {};
|
|
83623
|
+
const num = (key) => typeof meta3[key] === "number" ? meta3[key] : 0;
|
|
83624
|
+
this.viewerCapture = {
|
|
83625
|
+
generation: this.driver.generation(),
|
|
83626
|
+
width: num("screenshot_width"),
|
|
83627
|
+
height: num("screenshot_height"),
|
|
83628
|
+
screenWidth: num("screen_width"),
|
|
83629
|
+
screenHeight: num("screen_height"),
|
|
83630
|
+
scale: num("scale_factor") || 1,
|
|
83631
|
+
usedAt: now,
|
|
83632
|
+
cursor
|
|
83633
|
+
};
|
|
83634
|
+
return this.viewerCapture;
|
|
83635
|
+
}
|
|
83588
83636
|
captureIsCurrent() {
|
|
83589
83637
|
return this.capture !== void 0 && this.capture.generation === this.driver.generation();
|
|
83590
83638
|
}
|
|
@@ -87124,7 +87172,7 @@ var serviceEntry = () => managedEntry() ?? resolve2(process.argv[1]);
|
|
|
87124
87172
|
|
|
87125
87173
|
// src/version.ts
|
|
87126
87174
|
init_define_AGENTCONNECT_COMPUTER_WINDOWS_SOURCES();
|
|
87127
|
-
var COMPUTER_DAEMON_VERSION = true ? "2.8.0-rc.
|
|
87175
|
+
var COMPUTER_DAEMON_VERSION = true ? "2.8.0-rc.38" : JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8")).version;
|
|
87128
87176
|
|
|
87129
87177
|
// src/http-server.ts
|
|
87130
87178
|
var CONTROL_HEADER = "x-agentconnect-computer";
|
|
@@ -87266,7 +87314,7 @@ import { mkdirSync as mkdirSync4, writeFileSync as writeFileSync4 } from "node:f
|
|
|
87266
87314
|
import { join as join6 } from "node:path";
|
|
87267
87315
|
var waylandSession = () => process.platform === "linux" && Boolean(process.env.WAYLAND_DISPLAY);
|
|
87268
87316
|
var HELPER = String.raw`#!/usr/bin/env python3
|
|
87269
|
-
# Usage: screencast.py TOKEN_FILE FPS [FRAMES]. Asks the ScreenCast portal for the monitor (persisting a restore
|
|
87317
|
+
# Usage: screencast.py TOKEN_FILE FPS CURSOR(1|0) [FRAMES]. Asks the ScreenCast portal for the monitor (persisting a restore
|
|
87270
87318
|
# token so consent is asked once), then streams it as Y4M on stdout until killed.
|
|
87271
87319
|
import os, signal, subprocess, sys
|
|
87272
87320
|
import gi
|
|
@@ -87274,7 +87322,8 @@ gi.require_version('Gio', '2.0')
|
|
|
87274
87322
|
from gi.repository import Gio, GLib
|
|
87275
87323
|
|
|
87276
87324
|
token_path, fps = sys.argv[1], int(sys.argv[2])
|
|
87277
|
-
|
|
87325
|
+
show_cursor = sys.argv[3] == '1'
|
|
87326
|
+
frames = int(sys.argv[4]) if len(sys.argv) > 4 else 0
|
|
87278
87327
|
NAME, PATH = 'org.freedesktop.portal.Desktop', '/org/freedesktop/portal/desktop'
|
|
87279
87328
|
IFACE = 'org.freedesktop.portal.ScreenCast'
|
|
87280
87329
|
bus = Gio.bus_get_sync(Gio.BusType.SESSION, None)
|
|
@@ -87306,7 +87355,8 @@ def request(method, build):
|
|
|
87306
87355
|
modes = bus.call_sync(NAME, PATH, 'org.freedesktop.DBus.Properties', 'Get',
|
|
87307
87356
|
GLib.Variant('(ss)', (IFACE, 'AvailableCursorModes')), None, Gio.DBusCallFlags.NONE, -1,
|
|
87308
87357
|
None).unpack()[0]
|
|
87309
|
-
|
|
87358
|
+
# 2 embeds the pointer in the frames, 1 hides it.
|
|
87359
|
+
cursor = 2 if show_cursor and modes & 2 else 1
|
|
87310
87360
|
session = request('CreateSession', lambda t: GLib.Variant('(a{sv})', (
|
|
87311
87361
|
{'handle_token': t, 'session_handle_token': GLib.Variant('s', 'acd%d' % os.getpid())},)))['session_handle']
|
|
87312
87362
|
options = {'types': GLib.Variant('u', 1), 'multiple': GLib.Variant('b', False),
|
|
@@ -87440,19 +87490,20 @@ async function downloadFfmpeg(dataDir2, target, log40) {
|
|
|
87440
87490
|
log40(`Installed ffmpeg ${build3.version} at ${target}`);
|
|
87441
87491
|
return target;
|
|
87442
87492
|
}
|
|
87443
|
-
function captureInputs(fps, dataDir2, frames = 0) {
|
|
87493
|
+
function captureInputs(fps, dataDir2, frames = 0, cursor = true) {
|
|
87494
|
+
const mouse = cursor ? "1" : "0";
|
|
87444
87495
|
const override = process.env.AGENTCONNECT_COMPUTER_CAPTURE;
|
|
87445
87496
|
if (override) return [{ name: "override", args: override.split(" ").filter(Boolean), filters: [] }];
|
|
87446
87497
|
if (process.platform === "win32") {
|
|
87447
87498
|
return [
|
|
87448
87499
|
{
|
|
87449
87500
|
name: "ddagrab",
|
|
87450
|
-
args: ["-f", "lavfi", "-i", `ddagrab=output_idx=0:framerate=${fps}:draw_mouse
|
|
87501
|
+
args: ["-f", "lavfi", "-i", `ddagrab=output_idx=0:framerate=${fps}:draw_mouse=${mouse}`],
|
|
87451
87502
|
filters: ["hwdownload", "format=bgra"]
|
|
87452
87503
|
},
|
|
87453
87504
|
{
|
|
87454
87505
|
name: "gdigrab",
|
|
87455
|
-
args: ["-f", "gdigrab", "-framerate", String(fps), "-draw_mouse",
|
|
87506
|
+
args: ["-f", "gdigrab", "-framerate", String(fps), "-draw_mouse", mouse, "-i", "desktop"],
|
|
87456
87507
|
filters: []
|
|
87457
87508
|
}
|
|
87458
87509
|
];
|
|
@@ -87461,7 +87512,16 @@ function captureInputs(fps, dataDir2, frames = 0) {
|
|
|
87461
87512
|
return [
|
|
87462
87513
|
{
|
|
87463
87514
|
name: "avfoundation",
|
|
87464
|
-
args: [
|
|
87515
|
+
args: [
|
|
87516
|
+
"-f",
|
|
87517
|
+
"avfoundation",
|
|
87518
|
+
"-capture_cursor",
|
|
87519
|
+
mouse,
|
|
87520
|
+
"-framerate",
|
|
87521
|
+
String(fps),
|
|
87522
|
+
"-i",
|
|
87523
|
+
"Capture screen 0:none"
|
|
87524
|
+
],
|
|
87465
87525
|
filters: []
|
|
87466
87526
|
}
|
|
87467
87527
|
];
|
|
@@ -87473,7 +87533,7 @@ function captureInputs(fps, dataDir2, frames = 0) {
|
|
|
87473
87533
|
name: "pipewire",
|
|
87474
87534
|
producer: {
|
|
87475
87535
|
command: "python3",
|
|
87476
|
-
args: [helper, screencastTokenPath(dataDir2), String(fps), ...frames ? [String(frames)] : []]
|
|
87536
|
+
args: [helper, screencastTokenPath(dataDir2), String(fps), mouse, ...frames ? [String(frames)] : []]
|
|
87477
87537
|
},
|
|
87478
87538
|
args: ["-f", "yuv4mpegpipe", "-i", "pipe:0"],
|
|
87479
87539
|
filters: []
|
|
@@ -87483,7 +87543,7 @@ function captureInputs(fps, dataDir2, frames = 0) {
|
|
|
87483
87543
|
return [
|
|
87484
87544
|
{
|
|
87485
87545
|
name: "x11grab",
|
|
87486
|
-
args: ["-f", "x11grab", "-framerate", String(fps), "-i", process.env.DISPLAY ?? ":0"],
|
|
87546
|
+
args: ["-f", "x11grab", "-draw_mouse", mouse, "-framerate", String(fps), "-i", process.env.DISPLAY ?? ":0"],
|
|
87487
87547
|
filters: []
|
|
87488
87548
|
}
|
|
87489
87549
|
];
|
|
@@ -109989,7 +110049,7 @@ var RtcSessions = class {
|
|
|
109989
110049
|
session.connected = true;
|
|
109990
110050
|
session.lastSeen = Date.now();
|
|
109991
110051
|
this.options.log(`viewer ${client()}: WebRTC connected (${transport}${selectedPairType(pc)})`);
|
|
109992
|
-
session.unsubscribe ??= this.options.stream.subscribe((
|
|
110052
|
+
session.unsubscribe ??= this.options.stream.subscribe(continuous(track));
|
|
109993
110053
|
}
|
|
109994
110054
|
if (state === "failed" || state === "closed" || state === "disconnected") this.close(session.id);
|
|
109995
110055
|
});
|
|
@@ -110096,6 +110156,15 @@ function selectedPairType(pc) {
|
|
|
110096
110156
|
}
|
|
110097
110157
|
var PAIR_STATES = ["frozen", "waiting", "in-progress", "succeeded", "failed"];
|
|
110098
110158
|
var endpoint = (c) => `${c.type} ${c.host}:${c.port}`;
|
|
110159
|
+
function continuous(track) {
|
|
110160
|
+
let source;
|
|
110161
|
+
return (packet) => {
|
|
110162
|
+
const ssrc = packet.readUInt32BE(8);
|
|
110163
|
+
if (source !== void 0 && ssrc !== source) track.notifySourceChanged(RtpPacket.deSerialize(packet).header);
|
|
110164
|
+
source = ssrc;
|
|
110165
|
+
track.writeRtp(packet);
|
|
110166
|
+
};
|
|
110167
|
+
}
|
|
110099
110168
|
function iceReport(pc, trickled) {
|
|
110100
110169
|
const connection = pc.iceTransports[0]?.connection;
|
|
110101
110170
|
if (!connection) return "no ICE transport";
|
|
@@ -110135,6 +110204,9 @@ var ScreenStream = class {
|
|
|
110135
110204
|
socket;
|
|
110136
110205
|
process;
|
|
110137
110206
|
restart;
|
|
110207
|
+
// The running capture's pointer setting, and the process being replaced because that setting changed.
|
|
110208
|
+
cursor = true;
|
|
110209
|
+
swapping;
|
|
110138
110210
|
inputIndex = 0;
|
|
110139
110211
|
running = false;
|
|
110140
110212
|
runs = 0;
|
|
@@ -110170,7 +110242,8 @@ var ScreenStream = class {
|
|
|
110170
110242
|
}
|
|
110171
110243
|
spawn() {
|
|
110172
110244
|
if (!this.running || !this.socket) return;
|
|
110173
|
-
|
|
110245
|
+
this.cursor = this.options.cursor?.() ?? true;
|
|
110246
|
+
const inputs = captureInputs(this.options.fps, this.options.dataDir, 0, this.cursor);
|
|
110174
110247
|
const input2 = inputs[Math.min(this.inputIndex, inputs.length - 1)];
|
|
110175
110248
|
const args = [
|
|
110176
110249
|
"-hide_banner",
|
|
@@ -110203,7 +110276,9 @@ var ScreenStream = class {
|
|
|
110203
110276
|
`)));
|
|
110204
110277
|
feeder.once("exit", () => this.children.delete(feeder));
|
|
110205
110278
|
}
|
|
110206
|
-
this.options.log(
|
|
110279
|
+
this.options.log(
|
|
110280
|
+
`screen stream: ffmpeg started (${input2.name}, ${this.options.fps} fps${this.cursor ? "" : ", no pointer"})`
|
|
110281
|
+
);
|
|
110207
110282
|
child2.once("error", (error63) => {
|
|
110208
110283
|
this.lastError = `ffmpeg could not start: ${error63.message}`;
|
|
110209
110284
|
this.options.log(`screen stream: ${this.lastError}`);
|
|
@@ -110213,6 +110288,10 @@ var ScreenStream = class {
|
|
|
110213
110288
|
producer?.kill();
|
|
110214
110289
|
if (this.process === child2) this.process = void 0;
|
|
110215
110290
|
if (!this.running) return;
|
|
110291
|
+
if (this.swapping === child2) {
|
|
110292
|
+
this.swapping = void 0;
|
|
110293
|
+
return this.spawn();
|
|
110294
|
+
}
|
|
110216
110295
|
const detail = stderr.trim().split("\n").slice(-3).join(" | ");
|
|
110217
110296
|
this.lastError = `ffmpeg (${input2.name}) exited with ${code}: ${detail}`;
|
|
110218
110297
|
this.options.log(`screen stream: ${this.lastError}`);
|
|
@@ -110220,6 +110299,13 @@ var ScreenStream = class {
|
|
|
110220
110299
|
this.restart = setTimeout(() => this.spawn(), RESTART_DELAY_MS);
|
|
110221
110300
|
});
|
|
110222
110301
|
}
|
|
110302
|
+
// Restarts the capture when the pointer setting changed; the next keyframe carries the new frames.
|
|
110303
|
+
refresh() {
|
|
110304
|
+
const child2 = this.process;
|
|
110305
|
+
if (!child2 || this.swapping || (this.options.cursor?.() ?? true) === this.cursor) return;
|
|
110306
|
+
this.swapping = child2;
|
|
110307
|
+
terminate(child2);
|
|
110308
|
+
}
|
|
110223
110309
|
get bytesPerMs() {
|
|
110224
110310
|
return this.options.bitrateKbps * PACE_FACTOR / 8;
|
|
110225
110311
|
}
|
|
@@ -110252,16 +110338,19 @@ var ScreenStream = class {
|
|
|
110252
110338
|
this.queue.length = 0;
|
|
110253
110339
|
this.queuedBytes = 0;
|
|
110254
110340
|
clearTimeout(this.restart);
|
|
110341
|
+
this.swapping = void 0;
|
|
110255
110342
|
const child2 = this.process;
|
|
110256
110343
|
this.process = void 0;
|
|
110257
|
-
child2
|
|
110258
|
-
if (child2)
|
|
110259
|
-
setTimeout(() => child2.exitCode === null && child2.signalCode === null && child2.kill("SIGKILL"), 2e3).unref();
|
|
110344
|
+
if (child2) terminate(child2);
|
|
110260
110345
|
this.socket?.close();
|
|
110261
110346
|
this.socket = void 0;
|
|
110262
110347
|
this.options.log("screen stream: stopped (no viewers)");
|
|
110263
110348
|
}
|
|
110264
110349
|
};
|
|
110350
|
+
function terminate(child2) {
|
|
110351
|
+
child2.kill();
|
|
110352
|
+
setTimeout(() => child2.exitCode === null && child2.signalCode === null && child2.kill("SIGKILL"), 2e3).unref();
|
|
110353
|
+
}
|
|
110265
110354
|
|
|
110266
110355
|
// src/tools.ts
|
|
110267
110356
|
init_define_AGENTCONNECT_COMPUTER_WINDOWS_SOURCES();
|
|
@@ -110620,7 +110709,7 @@ var VIEWER_PAGE = `<!doctype html>
|
|
|
110620
110709
|
#screen.ctl { outline: 2px solid #4ade80; }
|
|
110621
110710
|
video, img { max-width: 100vw; max-height: calc(100vh - 48px); display: block; }
|
|
110622
110711
|
#surface { position: absolute; inset: 0; outline: none; }
|
|
110623
|
-
#screen.ctl #surface { cursor:
|
|
110712
|
+
#screen.ctl #surface { cursor: default; }
|
|
110624
110713
|
#login { display: none; gap: 8px; }
|
|
110625
110714
|
#error { color: #f87171; }
|
|
110626
110715
|
</style>
|
|
@@ -114528,6 +114617,8 @@ function buildViewer(dir, config2, computer, refusal, consoleViewer, advertiseAd
|
|
|
114528
114617
|
fps: config2.streamFps,
|
|
114529
114618
|
maxWidth: config2.streamMaxWidth,
|
|
114530
114619
|
bitrateKbps: config2.streamBitrateKbps,
|
|
114620
|
+
// A person driving sees their browser's pointer at once, so the delayed one in the frames would be a second cursor.
|
|
114621
|
+
cursor: () => computer.liveStatus() !== "human_in_control",
|
|
114531
114622
|
log: log39
|
|
114532
114623
|
});
|
|
114533
114624
|
const rtc = config2.webrtc ? new RtcSessions({
|
|
@@ -114550,7 +114641,7 @@ function buildViewer(dir, config2, computer, refusal, consoleViewer, advertiseAd
|
|
|
114550
114641
|
streamError: () => stream.lastError,
|
|
114551
114642
|
...refusal ? { refusal } : {}
|
|
114552
114643
|
});
|
|
114553
|
-
return viewer;
|
|
114644
|
+
return { ...viewer, statusChanged: () => stream.refresh() };
|
|
114554
114645
|
}
|
|
114555
114646
|
async function loadDriverTools(driver, config2) {
|
|
114556
114647
|
if (!config2.exposeDriverTools) return [];
|
|
@@ -114656,6 +114747,7 @@ async function run3(dir, initial, clientName) {
|
|
|
114656
114747
|
setTitle(status);
|
|
114657
114748
|
live2.channel?.client.beat();
|
|
114658
114749
|
live2.local?.update();
|
|
114750
|
+
live2.viewer?.statusChanged();
|
|
114659
114751
|
void live2.computer?.adminPromptChanged();
|
|
114660
114752
|
},
|
|
114661
114753
|
() => authority.refusal(),
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@agentconnect.md/computer",
|
|
3
|
-
"version": "2.8.0-rc.
|
|
3
|
+
"version": "2.8.0-rc.38",
|
|
4
4
|
"description": "AgentConnect computer daemon: drives a desktop through Cua Driver and serves the computer_* tools over MCP; installed and upgraded by agentconnect computer",
|
|
5
5
|
"license": "Apache-2.0",
|
|
6
6
|
"repository": {
|