@hasna/computer 0.1.1 → 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/agent/loop.d.ts.map +1 -1
- package/dist/cli/index.js +442 -15
- package/dist/db/agents.d.ts +26 -0
- package/dist/db/agents.d.ts.map +1 -0
- package/dist/db/index.d.ts +1 -0
- package/dist/db/index.d.ts.map +1 -1
- package/dist/drivers/mac/accessibility.d.ts +33 -0
- package/dist/drivers/mac/accessibility.d.ts.map +1 -0
- package/dist/drivers/mac/index.d.ts +7 -1
- package/dist/drivers/mac/index.d.ts.map +1 -1
- package/dist/drivers/mac/screenshot.d.ts +1 -1
- package/dist/drivers/mac/screenshot.d.ts.map +1 -1
- package/dist/index.d.ts +7 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +371 -36
- package/dist/lib/integrations.d.ts +30 -0
- package/dist/lib/integrations.d.ts.map +1 -0
- package/dist/lib/terminal-image.d.ts +29 -0
- package/dist/lib/terminal-image.d.ts.map +1 -0
- package/dist/mcp/index.js +347 -36
- package/dist/server/index.js +156 -36
- package/dist/types/index.d.ts +5 -0
- package/dist/types/index.d.ts.map +1 -1
- package/helpers/accessibility +0 -0
- package/helpers/accessibility.swift +161 -0
- package/package.json +3 -1
package/dist/mcp/index.js
CHANGED
|
@@ -14,6 +14,7 @@ var __export = (target, all) => {
|
|
|
14
14
|
set: __exportSetter.bind(all, name)
|
|
15
15
|
});
|
|
16
16
|
};
|
|
17
|
+
var __require = import.meta.require;
|
|
17
18
|
|
|
18
19
|
// src/mcp/index.ts
|
|
19
20
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
@@ -4000,10 +4001,15 @@ import { mkdir } from "fs/promises";
|
|
|
4000
4001
|
import { tmpdir } from "os";
|
|
4001
4002
|
import { join } from "path";
|
|
4002
4003
|
import { readFile, unlink } from "fs/promises";
|
|
4003
|
-
async function captureScreenshot() {
|
|
4004
|
+
async function captureScreenshot(displayNumber) {
|
|
4004
4005
|
const timestamp = Date.now();
|
|
4005
4006
|
const tmpPath = join(tmpdir(), `computer-screenshot-${timestamp}.png`);
|
|
4006
|
-
const
|
|
4007
|
+
const args = ["screencapture", "-x", "-C", "-t", "png"];
|
|
4008
|
+
if (displayNumber) {
|
|
4009
|
+
args.push(`-D${displayNumber}`);
|
|
4010
|
+
}
|
|
4011
|
+
args.push(tmpPath);
|
|
4012
|
+
const proc = Bun.spawn(args, {
|
|
4007
4013
|
stdout: "pipe",
|
|
4008
4014
|
stderr: "pipe"
|
|
4009
4015
|
});
|
|
@@ -4226,19 +4232,23 @@ function getScrollHelperPath() {
|
|
|
4226
4232
|
|
|
4227
4233
|
// src/drivers/mac/index.ts
|
|
4228
4234
|
class MacDriver {
|
|
4235
|
+
displayNumber;
|
|
4236
|
+
constructor(opts) {
|
|
4237
|
+
this.displayNumber = opts?.displayNumber;
|
|
4238
|
+
}
|
|
4229
4239
|
async getScreenSize() {
|
|
4230
4240
|
return getScreenSize();
|
|
4231
4241
|
}
|
|
4232
4242
|
async screenshot() {
|
|
4233
|
-
return captureScreenshot();
|
|
4243
|
+
return captureScreenshot(this.displayNumber);
|
|
4234
4244
|
}
|
|
4235
4245
|
async execute(action) {
|
|
4236
4246
|
return executeAction(action);
|
|
4237
4247
|
}
|
|
4238
4248
|
async dispose() {}
|
|
4239
4249
|
}
|
|
4240
|
-
function createMacDriver() {
|
|
4241
|
-
return new MacDriver;
|
|
4250
|
+
function createMacDriver(opts) {
|
|
4251
|
+
return new MacDriver(opts);
|
|
4242
4252
|
}
|
|
4243
4253
|
|
|
4244
4254
|
// node_modules/@anthropic-ai/sdk/version.mjs
|
|
@@ -13530,7 +13540,7 @@ var __export2 = (target, all) => {
|
|
|
13530
13540
|
set: __exportSetter2.bind(all, name)
|
|
13531
13541
|
});
|
|
13532
13542
|
};
|
|
13533
|
-
var
|
|
13543
|
+
var __require2 = /* @__PURE__ */ createRequire(import.meta.url);
|
|
13534
13544
|
var require_postgres_array = __commonJS((exports) => {
|
|
13535
13545
|
exports.parse = function(source, transform) {
|
|
13536
13546
|
return new ArrayParser(source, transform).parse();
|
|
@@ -14470,7 +14480,7 @@ var require_defaults = __commonJS((exports, module) => {
|
|
|
14470
14480
|
});
|
|
14471
14481
|
var require_utils = __commonJS((exports, module) => {
|
|
14472
14482
|
var defaults2 = require_defaults();
|
|
14473
|
-
var util3 =
|
|
14483
|
+
var util3 = __require2("util");
|
|
14474
14484
|
var { isDate } = util3.types || util3;
|
|
14475
14485
|
function escapeElement(elementRepresentation) {
|
|
14476
14486
|
const escaped = elementRepresentation.replace(/\\/g, "\\\\").replace(/"/g, "\\\"");
|
|
@@ -14626,7 +14636,7 @@ var require_utils = __commonJS((exports, module) => {
|
|
|
14626
14636
|
};
|
|
14627
14637
|
});
|
|
14628
14638
|
var require_utils_legacy = __commonJS((exports, module) => {
|
|
14629
|
-
var nodeCrypto =
|
|
14639
|
+
var nodeCrypto = __require2("crypto");
|
|
14630
14640
|
function md5(string) {
|
|
14631
14641
|
return nodeCrypto.createHash("md5").update(string, "utf-8").digest("hex");
|
|
14632
14642
|
}
|
|
@@ -14659,7 +14669,7 @@ var require_utils_legacy = __commonJS((exports, module) => {
|
|
|
14659
14669
|
};
|
|
14660
14670
|
});
|
|
14661
14671
|
var require_utils_webcrypto = __commonJS((exports, module) => {
|
|
14662
|
-
var nodeCrypto =
|
|
14672
|
+
var nodeCrypto = __require2("crypto");
|
|
14663
14673
|
module.exports = {
|
|
14664
14674
|
postgresMd5PasswordHash,
|
|
14665
14675
|
randomBytes,
|
|
@@ -15069,7 +15079,7 @@ var require_pg_connection_string = __commonJS((exports, module) => {
|
|
|
15069
15079
|
if (config.sslcert || config.sslkey || config.sslrootcert || config.sslmode) {
|
|
15070
15080
|
config.ssl = {};
|
|
15071
15081
|
}
|
|
15072
|
-
const fs = config.sslcert || config.sslkey || config.sslrootcert ?
|
|
15082
|
+
const fs = config.sslcert || config.sslkey || config.sslrootcert ? __require2("fs") : null;
|
|
15073
15083
|
if (config.sslcert) {
|
|
15074
15084
|
config.ssl.cert = fs.readFileSync(config.sslcert).toString();
|
|
15075
15085
|
}
|
|
@@ -15192,7 +15202,7 @@ See https://www.postgresql.org/docs/current/libpq-ssl.html for libpq SSL mode de
|
|
|
15192
15202
|
parse.parseIntoClientConfig = parseIntoClientConfig;
|
|
15193
15203
|
});
|
|
15194
15204
|
var require_connection_parameters = __commonJS((exports, module) => {
|
|
15195
|
-
var dns =
|
|
15205
|
+
var dns = __require2("dns");
|
|
15196
15206
|
var defaults2 = require_defaults();
|
|
15197
15207
|
var parse = require_pg_connection_string().parse;
|
|
15198
15208
|
var val = function(key, config, envVar) {
|
|
@@ -15415,7 +15425,7 @@ var require_result = __commonJS((exports, module) => {
|
|
|
15415
15425
|
module.exports = Result;
|
|
15416
15426
|
});
|
|
15417
15427
|
var require_query = __commonJS((exports, module) => {
|
|
15418
|
-
var { EventEmitter } =
|
|
15428
|
+
var { EventEmitter } = __require2("events");
|
|
15419
15429
|
var Result = require_result();
|
|
15420
15430
|
var utils = require_utils();
|
|
15421
15431
|
|
|
@@ -16398,11 +16408,11 @@ var require_stream = __commonJS((exports, module) => {
|
|
|
16398
16408
|
};
|
|
16399
16409
|
function getNodejsStreamFuncs() {
|
|
16400
16410
|
function getStream2(ssl) {
|
|
16401
|
-
const net =
|
|
16411
|
+
const net = __require2("net");
|
|
16402
16412
|
return new net.Socket;
|
|
16403
16413
|
}
|
|
16404
16414
|
function getSecureStream2(options) {
|
|
16405
|
-
const tls =
|
|
16415
|
+
const tls = __require2("tls");
|
|
16406
16416
|
return tls.connect(options);
|
|
16407
16417
|
}
|
|
16408
16418
|
return {
|
|
@@ -16444,7 +16454,7 @@ var require_stream = __commonJS((exports, module) => {
|
|
|
16444
16454
|
}
|
|
16445
16455
|
});
|
|
16446
16456
|
var require_connection = __commonJS((exports, module) => {
|
|
16447
|
-
var EventEmitter =
|
|
16457
|
+
var EventEmitter = __require2("events").EventEmitter;
|
|
16448
16458
|
var { parse, serialize } = require_dist();
|
|
16449
16459
|
var { getStream, getSecureStream } = require_stream();
|
|
16450
16460
|
var flushBuffer = serialize.flush();
|
|
@@ -16517,7 +16527,7 @@ var require_connection = __commonJS((exports, module) => {
|
|
|
16517
16527
|
options.key = self.ssl.key;
|
|
16518
16528
|
}
|
|
16519
16529
|
}
|
|
16520
|
-
const net =
|
|
16530
|
+
const net = __require2("net");
|
|
16521
16531
|
if (net.isIP && net.isIP(host) === 0) {
|
|
16522
16532
|
options.servername = host;
|
|
16523
16533
|
}
|
|
@@ -16620,8 +16630,8 @@ var require_connection = __commonJS((exports, module) => {
|
|
|
16620
16630
|
module.exports = Connection;
|
|
16621
16631
|
});
|
|
16622
16632
|
var require_split2 = __commonJS((exports, module) => {
|
|
16623
|
-
var { Transform } =
|
|
16624
|
-
var { StringDecoder } =
|
|
16633
|
+
var { Transform } = __require2("stream");
|
|
16634
|
+
var { StringDecoder } = __require2("string_decoder");
|
|
16625
16635
|
var kLast = Symbol("last");
|
|
16626
16636
|
var kDecoder = Symbol("decoder");
|
|
16627
16637
|
function transform(chunk, enc, cb) {
|
|
@@ -16717,10 +16727,10 @@ var require_split2 = __commonJS((exports, module) => {
|
|
|
16717
16727
|
module.exports = split;
|
|
16718
16728
|
});
|
|
16719
16729
|
var require_helper = __commonJS((exports, module) => {
|
|
16720
|
-
var path =
|
|
16721
|
-
var Stream3 =
|
|
16730
|
+
var path = __require2("path");
|
|
16731
|
+
var Stream3 = __require2("stream").Stream;
|
|
16722
16732
|
var split = require_split2();
|
|
16723
|
-
var util3 =
|
|
16733
|
+
var util3 = __require2("util");
|
|
16724
16734
|
var defaultPort = 5432;
|
|
16725
16735
|
var isWin = process.platform === "win32";
|
|
16726
16736
|
var warnStream = process.stderr;
|
|
@@ -16879,8 +16889,8 @@ var require_helper = __commonJS((exports, module) => {
|
|
|
16879
16889
|
};
|
|
16880
16890
|
});
|
|
16881
16891
|
var require_lib = __commonJS((exports, module) => {
|
|
16882
|
-
var path =
|
|
16883
|
-
var fs =
|
|
16892
|
+
var path = __require2("path");
|
|
16893
|
+
var fs = __require2("fs");
|
|
16884
16894
|
var helper = require_helper();
|
|
16885
16895
|
module.exports = function(connInfo, cb) {
|
|
16886
16896
|
var file = helper.getFileName();
|
|
@@ -16895,9 +16905,9 @@ var require_lib = __commonJS((exports, module) => {
|
|
|
16895
16905
|
module.exports.warnTo = helper.warnTo;
|
|
16896
16906
|
});
|
|
16897
16907
|
var require_client = __commonJS((exports, module) => {
|
|
16898
|
-
var EventEmitter =
|
|
16908
|
+
var EventEmitter = __require2("events").EventEmitter;
|
|
16899
16909
|
var utils = require_utils();
|
|
16900
|
-
var nodeUtils =
|
|
16910
|
+
var nodeUtils = __require2("util");
|
|
16901
16911
|
var sasl = require_sasl();
|
|
16902
16912
|
var TypeOverrides = require_type_overrides();
|
|
16903
16913
|
var ConnectionParameters = require_connection_parameters();
|
|
@@ -17474,7 +17484,7 @@ var require_client = __commonJS((exports, module) => {
|
|
|
17474
17484
|
module.exports = Client;
|
|
17475
17485
|
});
|
|
17476
17486
|
var require_pg_pool = __commonJS((exports, module) => {
|
|
17477
|
-
var EventEmitter =
|
|
17487
|
+
var EventEmitter = __require2("events").EventEmitter;
|
|
17478
17488
|
var NOOP = function() {};
|
|
17479
17489
|
var removeWhere = (list, predicate) => {
|
|
17480
17490
|
const i = list.findIndex(predicate);
|
|
@@ -17884,8 +17894,8 @@ var require_pg_pool = __commonJS((exports, module) => {
|
|
|
17884
17894
|
module.exports = Pool;
|
|
17885
17895
|
});
|
|
17886
17896
|
var require_query2 = __commonJS((exports, module) => {
|
|
17887
|
-
var EventEmitter =
|
|
17888
|
-
var util3 =
|
|
17897
|
+
var EventEmitter = __require2("events").EventEmitter;
|
|
17898
|
+
var util3 = __require2("util");
|
|
17889
17899
|
var utils = require_utils();
|
|
17890
17900
|
var NativeQuery = module.exports = function(config, values, callback) {
|
|
17891
17901
|
EventEmitter.call(this);
|
|
@@ -18018,7 +18028,7 @@ var require_query2 = __commonJS((exports, module) => {
|
|
|
18018
18028
|
};
|
|
18019
18029
|
});
|
|
18020
18030
|
var require_client2 = __commonJS((exports, module) => {
|
|
18021
|
-
var nodeUtils =
|
|
18031
|
+
var nodeUtils = __require2("util");
|
|
18022
18032
|
var Native;
|
|
18023
18033
|
try {
|
|
18024
18034
|
Native = (() => {
|
|
@@ -18028,8 +18038,8 @@ var require_client2 = __commonJS((exports, module) => {
|
|
|
18028
18038
|
throw e;
|
|
18029
18039
|
}
|
|
18030
18040
|
var TypeOverrides = require_type_overrides();
|
|
18031
|
-
var EventEmitter =
|
|
18032
|
-
var util3 =
|
|
18041
|
+
var EventEmitter = __require2("events").EventEmitter;
|
|
18042
|
+
var util3 = __require2("util");
|
|
18033
18043
|
var ConnectionParameters = require_connection_parameters();
|
|
18034
18044
|
var NativeQuery = require_query2();
|
|
18035
18045
|
var queryQueueLengthDeprecationNotice = nodeUtils.deprecate(() => {}, "Calling client.query() when the client is already executing a query is deprecated and will be removed in pg@9.0. Use async/await or an external async flow control mechanism instead.");
|
|
@@ -23245,6 +23255,7 @@ function getDb() {
|
|
|
23245
23255
|
total_tokens_in INTEGER NOT NULL DEFAULT 0,
|
|
23246
23256
|
total_tokens_out INTEGER NOT NULL DEFAULT 0,
|
|
23247
23257
|
total_duration_ms INTEGER NOT NULL DEFAULT 0,
|
|
23258
|
+
tags TEXT,
|
|
23248
23259
|
error TEXT,
|
|
23249
23260
|
created_at TEXT NOT NULL,
|
|
23250
23261
|
completed_at TEXT
|
|
@@ -23300,9 +23311,9 @@ function getDb() {
|
|
|
23300
23311
|
async function createSession(session) {
|
|
23301
23312
|
const d = getDb();
|
|
23302
23313
|
d.prepare(`
|
|
23303
|
-
INSERT INTO sessions (id, task, provider, model, status, steps, total_tokens_in, total_tokens_out, total_duration_ms, error, created_at, completed_at)
|
|
23304
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
23305
|
-
`).run(session.id, session.task, session.provider, session.model, session.status, session.steps, session.total_tokens_in, session.total_tokens_out, session.total_duration_ms, session.error ?? null, session.created_at, session.completed_at ?? null);
|
|
23314
|
+
INSERT INTO sessions (id, task, provider, model, status, steps, total_tokens_in, total_tokens_out, total_duration_ms, tags, error, created_at, completed_at)
|
|
23315
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
23316
|
+
`).run(session.id, session.task, session.provider, session.model, session.status, session.steps, session.total_tokens_in, session.total_tokens_out, session.total_duration_ms, session.tags?.length ? JSON.stringify(session.tags) : null, session.error ?? null, session.created_at, session.completed_at ?? null);
|
|
23306
23317
|
}
|
|
23307
23318
|
async function updateSession(session) {
|
|
23308
23319
|
const d = getDb();
|
|
@@ -23330,10 +23341,18 @@ function listSessions(opts) {
|
|
|
23330
23341
|
const d = getDb();
|
|
23331
23342
|
let sql = "SELECT * FROM sessions";
|
|
23332
23343
|
const params = [];
|
|
23344
|
+
const conditions = [];
|
|
23333
23345
|
if (opts?.status) {
|
|
23334
|
-
|
|
23346
|
+
conditions.push("status = ?");
|
|
23335
23347
|
params.push(opts.status);
|
|
23336
23348
|
}
|
|
23349
|
+
if (opts?.tag) {
|
|
23350
|
+
conditions.push("tags LIKE ?");
|
|
23351
|
+
params.push(`%"${opts.tag}"%`);
|
|
23352
|
+
}
|
|
23353
|
+
if (conditions.length > 0) {
|
|
23354
|
+
sql += " WHERE " + conditions.join(" AND ");
|
|
23355
|
+
}
|
|
23337
23356
|
sql += " ORDER BY created_at DESC";
|
|
23338
23357
|
if (opts?.limit) {
|
|
23339
23358
|
sql += " LIMIT ?";
|
|
@@ -23434,12 +23453,104 @@ function rowToSession(row) {
|
|
|
23434
23453
|
total_tokens_in: row.total_tokens_in,
|
|
23435
23454
|
total_tokens_out: row.total_tokens_out,
|
|
23436
23455
|
total_duration_ms: row.total_duration_ms,
|
|
23456
|
+
tags: row.tags ? JSON.parse(row.tags) : undefined,
|
|
23437
23457
|
error: row.error,
|
|
23438
23458
|
created_at: row.created_at,
|
|
23439
23459
|
completed_at: row.completed_at
|
|
23440
23460
|
};
|
|
23441
23461
|
}
|
|
23442
23462
|
|
|
23463
|
+
// src/lib/integrations.ts
|
|
23464
|
+
async function saveToRecordings(session, logs) {
|
|
23465
|
+
try {
|
|
23466
|
+
const { saveRecording } = await import("@hasna/recordings");
|
|
23467
|
+
await saveRecording({
|
|
23468
|
+
title: `Computer Use: ${session.task.slice(0, 100)}`,
|
|
23469
|
+
type: "computer-use",
|
|
23470
|
+
source: "computer",
|
|
23471
|
+
duration_ms: session.total_duration_ms,
|
|
23472
|
+
metadata: {
|
|
23473
|
+
session_id: session.id,
|
|
23474
|
+
provider: session.provider,
|
|
23475
|
+
model: session.model,
|
|
23476
|
+
steps: session.steps,
|
|
23477
|
+
tokens_in: session.total_tokens_in,
|
|
23478
|
+
tokens_out: session.total_tokens_out,
|
|
23479
|
+
status: session.status,
|
|
23480
|
+
tags: session.tags
|
|
23481
|
+
},
|
|
23482
|
+
transcript: logs.map((l) => ({
|
|
23483
|
+
step: l.step,
|
|
23484
|
+
action: l.action.type,
|
|
23485
|
+
reasoning: l.reasoning?.slice(0, 200),
|
|
23486
|
+
success: l.success,
|
|
23487
|
+
timestamp: l.created_at
|
|
23488
|
+
}))
|
|
23489
|
+
});
|
|
23490
|
+
return true;
|
|
23491
|
+
} catch {
|
|
23492
|
+
return false;
|
|
23493
|
+
}
|
|
23494
|
+
}
|
|
23495
|
+
async function registerWithSessions(session) {
|
|
23496
|
+
try {
|
|
23497
|
+
const mod = await import("@hasna/sessions");
|
|
23498
|
+
const registerSession = mod.registerSession ?? mod.createSession ?? mod.saveSession;
|
|
23499
|
+
if (typeof registerSession !== "function")
|
|
23500
|
+
return false;
|
|
23501
|
+
await registerSession({
|
|
23502
|
+
id: session.id,
|
|
23503
|
+
type: "computer-use",
|
|
23504
|
+
source: "computer",
|
|
23505
|
+
status: session.status,
|
|
23506
|
+
metadata: {
|
|
23507
|
+
task: session.task,
|
|
23508
|
+
provider: session.provider,
|
|
23509
|
+
model: session.model,
|
|
23510
|
+
steps: session.steps
|
|
23511
|
+
},
|
|
23512
|
+
started_at: session.created_at,
|
|
23513
|
+
ended_at: session.completed_at
|
|
23514
|
+
});
|
|
23515
|
+
return true;
|
|
23516
|
+
} catch {
|
|
23517
|
+
return false;
|
|
23518
|
+
}
|
|
23519
|
+
}
|
|
23520
|
+
async function pushToLogs(session, logs) {
|
|
23521
|
+
try {
|
|
23522
|
+
const mod = await import("@hasna/logs");
|
|
23523
|
+
const pushBatch = mod.logPushBatch ?? mod.pushBatch;
|
|
23524
|
+
if (typeof pushBatch !== "function")
|
|
23525
|
+
return false;
|
|
23526
|
+
await pushBatch(logs.map((l) => ({
|
|
23527
|
+
level: l.success ? "info" : "error",
|
|
23528
|
+
source: "computer",
|
|
23529
|
+
message: `[${l.action.type}] ${l.reasoning?.slice(0, 100) ?? ""}`,
|
|
23530
|
+
metadata: {
|
|
23531
|
+
session_id: session.id,
|
|
23532
|
+
step: l.step,
|
|
23533
|
+
action_type: l.action.type,
|
|
23534
|
+
success: l.success,
|
|
23535
|
+
error: l.error,
|
|
23536
|
+
duration_ms: l.duration_ms
|
|
23537
|
+
},
|
|
23538
|
+
timestamp: l.created_at
|
|
23539
|
+
})));
|
|
23540
|
+
return true;
|
|
23541
|
+
} catch {
|
|
23542
|
+
return false;
|
|
23543
|
+
}
|
|
23544
|
+
}
|
|
23545
|
+
async function runPostSessionIntegrations(session, logs) {
|
|
23546
|
+
const [recordings, sessions, logsPushed] = await Promise.all([
|
|
23547
|
+
saveToRecordings(session, logs),
|
|
23548
|
+
registerWithSessions(session),
|
|
23549
|
+
pushToLogs(session, logs)
|
|
23550
|
+
]);
|
|
23551
|
+
return { recordings, sessions, logs: logsPushed };
|
|
23552
|
+
}
|
|
23553
|
+
|
|
23443
23554
|
// src/agent/loop.ts
|
|
23444
23555
|
var DEFAULT_MAX_STEPS = 50;
|
|
23445
23556
|
async function runTask(options) {
|
|
@@ -23453,10 +23564,12 @@ async function runTask(options) {
|
|
|
23453
23564
|
systemPrompt,
|
|
23454
23565
|
screenshotMaxWidth,
|
|
23455
23566
|
dryRun = false,
|
|
23567
|
+
tags,
|
|
23568
|
+
displayNumber,
|
|
23456
23569
|
onStep,
|
|
23457
23570
|
onDone
|
|
23458
23571
|
} = options;
|
|
23459
|
-
const driver = createMacDriver();
|
|
23572
|
+
const driver = createMacDriver({ displayNumber });
|
|
23460
23573
|
const provider = createProvider(providerName, { model });
|
|
23461
23574
|
const config = loadConfig();
|
|
23462
23575
|
const safetyConfig = config.safety;
|
|
@@ -23467,6 +23580,7 @@ async function runTask(options) {
|
|
|
23467
23580
|
provider: providerName,
|
|
23468
23581
|
model: model ?? (providerName === "anthropic" ? "claude-sonnet-4-5-20250514" : "computer-use-preview"),
|
|
23469
23582
|
status: "running",
|
|
23583
|
+
tags,
|
|
23470
23584
|
steps: 0,
|
|
23471
23585
|
total_tokens_in: 0,
|
|
23472
23586
|
total_tokens_out: 0,
|
|
@@ -23519,6 +23633,8 @@ async function runTask(options) {
|
|
|
23519
23633
|
});
|
|
23520
23634
|
await updateSession(session);
|
|
23521
23635
|
onStep?.(step, response, { success: true, duration_ms: 0 });
|
|
23636
|
+
const logs = getActionLogs(sessionId);
|
|
23637
|
+
await runPostSessionIntegrations(session, logs).catch(() => {});
|
|
23522
23638
|
onDone?.(session);
|
|
23523
23639
|
await driver.dispose();
|
|
23524
23640
|
return session;
|
|
@@ -23584,6 +23700,8 @@ async function runTask(options) {
|
|
|
23584
23700
|
session.completed_at = new Date().toISOString();
|
|
23585
23701
|
session.error = `Reached max steps (${maxSteps})`;
|
|
23586
23702
|
await updateSession(session);
|
|
23703
|
+
const endLogs = getActionLogs(sessionId);
|
|
23704
|
+
await runPostSessionIntegrations(session, endLogs).catch(() => {});
|
|
23587
23705
|
onDone?.(session);
|
|
23588
23706
|
await driver.dispose();
|
|
23589
23707
|
return session;
|
|
@@ -23593,6 +23711,8 @@ async function runTask(options) {
|
|
|
23593
23711
|
session.total_duration_ms = Date.now() - startTime;
|
|
23594
23712
|
session.completed_at = new Date().toISOString();
|
|
23595
23713
|
await updateSession(session);
|
|
23714
|
+
const errLogs = getActionLogs(sessionId);
|
|
23715
|
+
await runPostSessionIntegrations(session, errLogs).catch(() => {});
|
|
23596
23716
|
onDone?.(session);
|
|
23597
23717
|
await driver.dispose();
|
|
23598
23718
|
return session;
|
|
@@ -23620,6 +23740,154 @@ function remapCoordinates(action, from, to) {
|
|
|
23620
23740
|
}
|
|
23621
23741
|
}
|
|
23622
23742
|
|
|
23743
|
+
// src/db/agents.ts
|
|
23744
|
+
function ensureAgentsTable() {
|
|
23745
|
+
const d = getDb();
|
|
23746
|
+
d.exec(`
|
|
23747
|
+
CREATE TABLE IF NOT EXISTS agents (
|
|
23748
|
+
id TEXT PRIMARY KEY,
|
|
23749
|
+
name TEXT NOT NULL UNIQUE,
|
|
23750
|
+
description TEXT,
|
|
23751
|
+
capabilities TEXT,
|
|
23752
|
+
focus TEXT,
|
|
23753
|
+
last_heartbeat TEXT NOT NULL,
|
|
23754
|
+
created_at TEXT NOT NULL DEFAULT (datetime('now'))
|
|
23755
|
+
);
|
|
23756
|
+
`);
|
|
23757
|
+
}
|
|
23758
|
+
function registerAgent(agent) {
|
|
23759
|
+
ensureAgentsTable();
|
|
23760
|
+
const d = getDb();
|
|
23761
|
+
const now = new Date().toISOString();
|
|
23762
|
+
const id = agent.name;
|
|
23763
|
+
const existing = d.prepare("SELECT * FROM agents WHERE name = ?").get(agent.name);
|
|
23764
|
+
if (existing) {
|
|
23765
|
+
d.prepare(`
|
|
23766
|
+
UPDATE agents SET description = ?, capabilities = ?, last_heartbeat = ?
|
|
23767
|
+
WHERE name = ?
|
|
23768
|
+
`).run(agent.description ?? existing.description, agent.capabilities ? JSON.stringify(agent.capabilities) : existing.capabilities, now, agent.name);
|
|
23769
|
+
} else {
|
|
23770
|
+
d.prepare(`
|
|
23771
|
+
INSERT INTO agents (id, name, description, capabilities, last_heartbeat, created_at)
|
|
23772
|
+
VALUES (?, ?, ?, ?, ?, ?)
|
|
23773
|
+
`).run(id, agent.name, agent.description ?? null, agent.capabilities ? JSON.stringify(agent.capabilities) : null, now, now);
|
|
23774
|
+
}
|
|
23775
|
+
return getAgent(id);
|
|
23776
|
+
}
|
|
23777
|
+
function heartbeat(agentId) {
|
|
23778
|
+
ensureAgentsTable();
|
|
23779
|
+
const d = getDb();
|
|
23780
|
+
const result = d.prepare("UPDATE agents SET last_heartbeat = ? WHERE id = ?").run(new Date().toISOString(), agentId);
|
|
23781
|
+
return result.changes > 0;
|
|
23782
|
+
}
|
|
23783
|
+
function setFocus(agentId, focus) {
|
|
23784
|
+
ensureAgentsTable();
|
|
23785
|
+
const d = getDb();
|
|
23786
|
+
const result = d.prepare("UPDATE agents SET focus = ? WHERE id = ?").run(focus, agentId);
|
|
23787
|
+
return result.changes > 0;
|
|
23788
|
+
}
|
|
23789
|
+
function getAgent(id) {
|
|
23790
|
+
ensureAgentsTable();
|
|
23791
|
+
const d = getDb();
|
|
23792
|
+
const row = d.prepare("SELECT * FROM agents WHERE id = ?").get(id);
|
|
23793
|
+
if (!row)
|
|
23794
|
+
return null;
|
|
23795
|
+
return rowToAgent(row);
|
|
23796
|
+
}
|
|
23797
|
+
function listAgents() {
|
|
23798
|
+
ensureAgentsTable();
|
|
23799
|
+
const d = getDb();
|
|
23800
|
+
const rows = d.prepare("SELECT * FROM agents ORDER BY last_heartbeat DESC").all();
|
|
23801
|
+
return rows.map(rowToAgent);
|
|
23802
|
+
}
|
|
23803
|
+
function rowToAgent(row) {
|
|
23804
|
+
return {
|
|
23805
|
+
id: row.id,
|
|
23806
|
+
name: row.name,
|
|
23807
|
+
description: row.description,
|
|
23808
|
+
capabilities: row.capabilities ? JSON.parse(row.capabilities) : undefined,
|
|
23809
|
+
focus: row.focus,
|
|
23810
|
+
last_heartbeat: row.last_heartbeat,
|
|
23811
|
+
created_at: row.created_at
|
|
23812
|
+
};
|
|
23813
|
+
}
|
|
23814
|
+
|
|
23815
|
+
// src/drivers/mac/accessibility.ts
|
|
23816
|
+
import { join as join6, dirname as dirname2 } from "path";
|
|
23817
|
+
import { existsSync as existsSync4 } from "fs";
|
|
23818
|
+
import { fileURLToPath as fileURLToPath2 } from "url";
|
|
23819
|
+
async function queryAccessibilityTree(opts) {
|
|
23820
|
+
const helperPath = getAccessibilityHelperPath();
|
|
23821
|
+
const args = [helperPath];
|
|
23822
|
+
if (opts?.app) {
|
|
23823
|
+
args.push("--app", opts.app);
|
|
23824
|
+
}
|
|
23825
|
+
if (opts?.focusedOnly) {
|
|
23826
|
+
args.push("--focused");
|
|
23827
|
+
}
|
|
23828
|
+
if (opts?.depth !== undefined) {
|
|
23829
|
+
args.push("--depth", String(opts.depth));
|
|
23830
|
+
}
|
|
23831
|
+
const proc = Bun.spawn(args, {
|
|
23832
|
+
stdout: "pipe",
|
|
23833
|
+
stderr: "pipe"
|
|
23834
|
+
});
|
|
23835
|
+
await proc.exited;
|
|
23836
|
+
if (proc.exitCode !== 0) {
|
|
23837
|
+
const stderr = await new Response(proc.stderr).text();
|
|
23838
|
+
throw new Error(`accessibility query failed: ${stderr}`);
|
|
23839
|
+
}
|
|
23840
|
+
const stdout = await new Response(proc.stdout).text();
|
|
23841
|
+
try {
|
|
23842
|
+
return JSON.parse(stdout);
|
|
23843
|
+
} catch {
|
|
23844
|
+
return [];
|
|
23845
|
+
}
|
|
23846
|
+
}
|
|
23847
|
+
function summarizeAccessibilityTree(elements) {
|
|
23848
|
+
if (elements.length === 0)
|
|
23849
|
+
return "No accessibility information available.";
|
|
23850
|
+
const lines = ["UI Elements:"];
|
|
23851
|
+
for (const el of elements) {
|
|
23852
|
+
const parts = [];
|
|
23853
|
+
if (el.role)
|
|
23854
|
+
parts.push(el.role);
|
|
23855
|
+
if (el.title)
|
|
23856
|
+
parts.push(`"${el.title}"`);
|
|
23857
|
+
if (el.label)
|
|
23858
|
+
parts.push(`(${el.label})`);
|
|
23859
|
+
if (el.value)
|
|
23860
|
+
parts.push(`value="${el.value}"`);
|
|
23861
|
+
const pos = `at (${el.x + Math.round(el.width / 2)}, ${el.y + Math.round(el.height / 2)})`;
|
|
23862
|
+
const size = `${el.width}x${el.height}`;
|
|
23863
|
+
const flags = [];
|
|
23864
|
+
if (!el.enabled)
|
|
23865
|
+
flags.push("disabled");
|
|
23866
|
+
if (el.focused)
|
|
23867
|
+
flags.push("focused");
|
|
23868
|
+
lines.push(` - ${parts.join(" ")} ${pos} [${size}]${flags.length ? ` (${flags.join(", ")})` : ""}`);
|
|
23869
|
+
}
|
|
23870
|
+
return lines.join(`
|
|
23871
|
+
`);
|
|
23872
|
+
}
|
|
23873
|
+
var _helperPath = null;
|
|
23874
|
+
function getAccessibilityHelperPath() {
|
|
23875
|
+
if (_helperPath)
|
|
23876
|
+
return _helperPath;
|
|
23877
|
+
const candidates = [
|
|
23878
|
+
join6(dirname2(fileURLToPath2(import.meta.url)), "..", "..", "..", "helpers", "accessibility"),
|
|
23879
|
+
join6(dirname2(fileURLToPath2(import.meta.url)), "..", "helpers", "accessibility"),
|
|
23880
|
+
join6(process.env.HOME ?? "~", ".hasna", "computer", "helpers", "accessibility")
|
|
23881
|
+
];
|
|
23882
|
+
for (const candidate of candidates) {
|
|
23883
|
+
if (existsSync4(candidate)) {
|
|
23884
|
+
_helperPath = candidate;
|
|
23885
|
+
return candidate;
|
|
23886
|
+
}
|
|
23887
|
+
}
|
|
23888
|
+
throw new Error("Accessibility helper not found. Run `swiftc helpers/accessibility.swift -o helpers/accessibility -framework AppKit` from the project root.");
|
|
23889
|
+
}
|
|
23890
|
+
|
|
23623
23891
|
// src/mcp/index.ts
|
|
23624
23892
|
var server = new McpServer({
|
|
23625
23893
|
name: "computer",
|
|
@@ -23805,6 +24073,49 @@ server.tool("computer_stats", "Get usage statistics for computer use", {}, async
|
|
|
23805
24073
|
const stats = getStats();
|
|
23806
24074
|
return { content: [{ type: "text", text: JSON.stringify(stats, null, 2) }] };
|
|
23807
24075
|
});
|
|
24076
|
+
server.tool("computer_accessibility", "Query the macOS accessibility tree \u2014 get structured UI elements (buttons, fields, labels) with positions. Much more precise than pixel-guessing from screenshots.", {
|
|
24077
|
+
app: exports_external.string().optional().describe("App name to query (default: frontmost)"),
|
|
24078
|
+
focused_only: exports_external.boolean().default(false).describe("Only get focused element's subtree"),
|
|
24079
|
+
depth: exports_external.number().default(3).describe("Max tree traversal depth"),
|
|
24080
|
+
format: exports_external.enum(["json", "summary"]).default("summary").describe("Output format")
|
|
24081
|
+
}, async (params) => {
|
|
24082
|
+
try {
|
|
24083
|
+
const elements = await queryAccessibilityTree({
|
|
24084
|
+
app: params.app,
|
|
24085
|
+
focusedOnly: params.focused_only,
|
|
24086
|
+
depth: params.depth
|
|
24087
|
+
});
|
|
24088
|
+
const text = params.format === "json" ? JSON.stringify(elements, null, 2) : summarizeAccessibilityTree(elements);
|
|
24089
|
+
return { content: [{ type: "text", text }] };
|
|
24090
|
+
} catch (err) {
|
|
24091
|
+
return { content: [{ type: "text", text: `Accessibility query failed: ${err instanceof Error ? err.message : err}` }] };
|
|
24092
|
+
}
|
|
24093
|
+
});
|
|
24094
|
+
server.tool("computer_register_agent", "Register an agent for multi-agent coordination", {
|
|
24095
|
+
name: exports_external.string().describe("Agent name"),
|
|
24096
|
+
description: exports_external.string().optional().describe("Agent description"),
|
|
24097
|
+
capabilities: exports_external.array(exports_external.string()).optional().describe("Agent capabilities")
|
|
24098
|
+
}, async (params) => {
|
|
24099
|
+
const agent = registerAgent(params);
|
|
24100
|
+
return { content: [{ type: "text", text: JSON.stringify(agent, null, 2) }] };
|
|
24101
|
+
});
|
|
24102
|
+
server.tool("computer_heartbeat", "Send a heartbeat to mark an agent as active", {
|
|
24103
|
+
agent_id: exports_external.string().describe("Agent ID")
|
|
24104
|
+
}, async (params) => {
|
|
24105
|
+
const ok = heartbeat(params.agent_id);
|
|
24106
|
+
return { content: [{ type: "text", text: ok ? "Heartbeat received" : "Agent not found" }] };
|
|
24107
|
+
});
|
|
24108
|
+
server.tool("computer_set_focus", "Set what an agent is currently focused on", {
|
|
24109
|
+
agent_id: exports_external.string().describe("Agent ID"),
|
|
24110
|
+
focus: exports_external.string().describe("Current focus description")
|
|
24111
|
+
}, async (params) => {
|
|
24112
|
+
const ok = setFocus(params.agent_id, params.focus);
|
|
24113
|
+
return { content: [{ type: "text", text: ok ? "Focus updated" : "Agent not found" }] };
|
|
24114
|
+
});
|
|
24115
|
+
server.tool("computer_list_agents", "List all registered agents", {}, async () => {
|
|
24116
|
+
const agents = listAgents();
|
|
24117
|
+
return { content: [{ type: "text", text: JSON.stringify(agents, null, 2) }] };
|
|
24118
|
+
});
|
|
23808
24119
|
registerCloudTools(server, "computer");
|
|
23809
24120
|
async function main() {
|
|
23810
24121
|
const transport = new StdioServerTransport;
|