@leg3ndy/otto-bridge 0.4.2 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/executors/native_macos.js +389 -55
- package/dist/http.js +29 -0
- package/dist/main.js +0 -0
- package/dist/runtime.js +1 -1
- package/dist/types.js +1 -1
- package/package.json +1 -1
|
@@ -4,6 +4,7 @@ import os from "node:os";
|
|
|
4
4
|
import path from "node:path";
|
|
5
5
|
import process from "node:process";
|
|
6
6
|
import { JobCancelledError } from "./shared.js";
|
|
7
|
+
import { postDeviceJson, uploadDeviceJobArtifact } from "../http.js";
|
|
7
8
|
const KNOWN_APPS = [
|
|
8
9
|
{ canonical: "Safari", patterns: [/\bsafari\b/i] },
|
|
9
10
|
{ canonical: "Google Chrome", patterns: [/\bgoogle chrome\b/i, /\bchrome\b/i] },
|
|
@@ -87,6 +88,22 @@ function humanizeUrl(url) {
|
|
|
87
88
|
return normalized;
|
|
88
89
|
}
|
|
89
90
|
}
|
|
91
|
+
function mimeTypeFromPath(filePath) {
|
|
92
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
93
|
+
if (ext === ".png")
|
|
94
|
+
return "image/png";
|
|
95
|
+
if (ext === ".jpg" || ext === ".jpeg")
|
|
96
|
+
return "image/jpeg";
|
|
97
|
+
if (ext === ".webp")
|
|
98
|
+
return "image/webp";
|
|
99
|
+
if (ext === ".gif")
|
|
100
|
+
return "image/gif";
|
|
101
|
+
if (ext === ".txt" || ext === ".md")
|
|
102
|
+
return "text/plain";
|
|
103
|
+
if (ext === ".json")
|
|
104
|
+
return "application/json";
|
|
105
|
+
return "application/octet-stream";
|
|
106
|
+
}
|
|
90
107
|
function expandUserPath(value) {
|
|
91
108
|
const trimmed = value.trim();
|
|
92
109
|
if (!trimmed) {
|
|
@@ -109,6 +126,9 @@ function clipText(value, maxLength) {
|
|
|
109
126
|
}
|
|
110
127
|
return `${value.slice(0, maxLength)}...`;
|
|
111
128
|
}
|
|
129
|
+
function delay(ms) {
|
|
130
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
131
|
+
}
|
|
112
132
|
function escapeHtml(value) {
|
|
113
133
|
return value
|
|
114
134
|
.replace(/&/g, "&")
|
|
@@ -155,50 +175,7 @@ function isSafeShellCommand(command) {
|
|
|
155
175
|
if (!trimmed) {
|
|
156
176
|
return false;
|
|
157
177
|
}
|
|
158
|
-
|
|
159
|
-
/(^|[;&|])\s*sudo\b/i,
|
|
160
|
-
/\brm\b/i,
|
|
161
|
-
/\bmv\b/i,
|
|
162
|
-
/\bcp\b/i,
|
|
163
|
-
/\bchmod\b/i,
|
|
164
|
-
/\bchown\b/i,
|
|
165
|
-
/\bshutdown\b/i,
|
|
166
|
-
/\breboot\b/i,
|
|
167
|
-
/\bmkfs\b/i,
|
|
168
|
-
/\bdd\b/i,
|
|
169
|
-
/\bkill(?:all)?\b/i,
|
|
170
|
-
/>/,
|
|
171
|
-
/>>/,
|
|
172
|
-
];
|
|
173
|
-
if (forbiddenPatterns.some((pattern) => pattern.test(trimmed))) {
|
|
174
|
-
return false;
|
|
175
|
-
}
|
|
176
|
-
const normalized = trimmed.replace(/\s+/g, " ");
|
|
177
|
-
const allowedPrefixes = [
|
|
178
|
-
"pwd",
|
|
179
|
-
"ls",
|
|
180
|
-
"cat ",
|
|
181
|
-
"cat",
|
|
182
|
-
"sed ",
|
|
183
|
-
"rg ",
|
|
184
|
-
"find ",
|
|
185
|
-
"git status",
|
|
186
|
-
"git log",
|
|
187
|
-
"git diff",
|
|
188
|
-
"head ",
|
|
189
|
-
"tail ",
|
|
190
|
-
"wc ",
|
|
191
|
-
"stat ",
|
|
192
|
-
"file ",
|
|
193
|
-
"mdls ",
|
|
194
|
-
"whoami",
|
|
195
|
-
"date",
|
|
196
|
-
"uname",
|
|
197
|
-
"python3 --version",
|
|
198
|
-
"node -v",
|
|
199
|
-
"npm -v",
|
|
200
|
-
];
|
|
201
|
-
return allowedPrefixes.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix} `));
|
|
178
|
+
return true;
|
|
202
179
|
}
|
|
203
180
|
function extractConfirmationOptions(job, actions) {
|
|
204
181
|
const payload = asRecord(job.payload);
|
|
@@ -307,6 +284,13 @@ function parseStructuredActions(job) {
|
|
|
307
284
|
actions.push({ type: "take_screenshot", path: savePath || undefined });
|
|
308
285
|
continue;
|
|
309
286
|
}
|
|
287
|
+
if (type === "read_frontmost_page" || type === "read_page" || type === "read_webpage") {
|
|
288
|
+
actions.push({
|
|
289
|
+
type: "read_frontmost_page",
|
|
290
|
+
app: asString(action.app) || asString(action.application) || "Safari",
|
|
291
|
+
});
|
|
292
|
+
continue;
|
|
293
|
+
}
|
|
310
294
|
if (type === "read_file" || type === "read_local_file") {
|
|
311
295
|
const filePath = asString(action.path);
|
|
312
296
|
if (filePath) {
|
|
@@ -327,6 +311,25 @@ function parseStructuredActions(job) {
|
|
|
327
311
|
if (command) {
|
|
328
312
|
actions.push({ type: "run_shell", command, cwd: cwd || undefined });
|
|
329
313
|
}
|
|
314
|
+
continue;
|
|
315
|
+
}
|
|
316
|
+
if (type === "set_volume" || type === "volume") {
|
|
317
|
+
const rawLevel = Number(action.level);
|
|
318
|
+
if (Number.isFinite(rawLevel)) {
|
|
319
|
+
actions.push({ type: "set_volume", level: Math.max(0, Math.min(Math.round(rawLevel), 100)) });
|
|
320
|
+
}
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
if (type === "click_visual_target" || type === "click_target") {
|
|
324
|
+
const description = asString(action.description) || asString(action.target);
|
|
325
|
+
if (description) {
|
|
326
|
+
actions.push({
|
|
327
|
+
type: "click_visual_target",
|
|
328
|
+
description,
|
|
329
|
+
app: asString(action.app) || undefined,
|
|
330
|
+
});
|
|
331
|
+
}
|
|
332
|
+
continue;
|
|
330
333
|
}
|
|
331
334
|
}
|
|
332
335
|
return actions;
|
|
@@ -361,6 +364,30 @@ function deriveActionsFromText(job) {
|
|
|
361
364
|
const task = extractTaskText(job);
|
|
362
365
|
const detectedApp = detectKnownApp(task);
|
|
363
366
|
const detectedUrl = detectUrl(task);
|
|
367
|
+
const normalizedTask = normalizeText(task);
|
|
368
|
+
if (/\b(volume|som|audio)\b/i.test(task)) {
|
|
369
|
+
const percentMatch = task.match(/(\d{1,3})\s*%/);
|
|
370
|
+
let level = 50;
|
|
371
|
+
if (percentMatch?.[1]) {
|
|
372
|
+
level = Math.max(0, Math.min(Number(percentMatch[1]), 100));
|
|
373
|
+
}
|
|
374
|
+
else if (/\b(mudo|mute|silencia)\b/i.test(task)) {
|
|
375
|
+
level = 0;
|
|
376
|
+
}
|
|
377
|
+
else if (/\b(aumenta|aumente|mais alto)\b/i.test(task)) {
|
|
378
|
+
level = 80;
|
|
379
|
+
}
|
|
380
|
+
else if (/\b(diminui|abaixa|mais baixo)\b/i.test(task)) {
|
|
381
|
+
level = 25;
|
|
382
|
+
}
|
|
383
|
+
return [{ type: "set_volume", level }];
|
|
384
|
+
}
|
|
385
|
+
if ((normalizedTask.includes("leia") || normalizedTask.includes("ler")) && detectedUrl) {
|
|
386
|
+
return [
|
|
387
|
+
{ type: "open_url", url: detectedUrl, app: detectedApp || "Safari" },
|
|
388
|
+
{ type: "read_frontmost_page", app: detectedApp || "Safari" },
|
|
389
|
+
];
|
|
390
|
+
}
|
|
364
391
|
if (detectedUrl) {
|
|
365
392
|
return [{
|
|
366
393
|
type: "open_url",
|
|
@@ -384,8 +411,12 @@ function extractActions(job) {
|
|
|
384
411
|
return deriveActionsFromText(job);
|
|
385
412
|
}
|
|
386
413
|
export class NativeMacOSJobExecutor {
|
|
414
|
+
bridgeConfig;
|
|
387
415
|
cancelledJobs = new Set();
|
|
388
416
|
activeChild = null;
|
|
417
|
+
constructor(bridgeConfig) {
|
|
418
|
+
this.bridgeConfig = bridgeConfig;
|
|
419
|
+
}
|
|
389
420
|
async run(job, reporter) {
|
|
390
421
|
if (process.platform !== "darwin") {
|
|
391
422
|
throw new Error("The native-macos executor only runs on macOS");
|
|
@@ -407,6 +438,13 @@ export class NativeMacOSJobExecutor {
|
|
|
407
438
|
}
|
|
408
439
|
try {
|
|
409
440
|
const completionNotes = [];
|
|
441
|
+
const artifacts = [];
|
|
442
|
+
const resultPayload = {
|
|
443
|
+
executor: "native-macos",
|
|
444
|
+
actions,
|
|
445
|
+
artifacts,
|
|
446
|
+
action_summaries: completionNotes,
|
|
447
|
+
};
|
|
410
448
|
for (let index = 0; index < actions.length; index += 1) {
|
|
411
449
|
this.assertNotCancelled(job.job_id);
|
|
412
450
|
const action = actions[index];
|
|
@@ -414,11 +452,13 @@ export class NativeMacOSJobExecutor {
|
|
|
414
452
|
if (action.type === "open_app") {
|
|
415
453
|
await reporter.progress(progressPercent, `Abrindo ${action.app} no macOS`);
|
|
416
454
|
await this.openApp(action.app);
|
|
455
|
+
completionNotes.push(`${action.app} foi aberto no macOS.`);
|
|
417
456
|
continue;
|
|
418
457
|
}
|
|
419
458
|
if (action.type === "focus_app") {
|
|
420
459
|
await reporter.progress(progressPercent, `Trazendo ${action.app} para frente`);
|
|
421
460
|
await this.focusApp(action.app);
|
|
461
|
+
completionNotes.push(`${action.app} ficou em foco no macOS.`);
|
|
422
462
|
continue;
|
|
423
463
|
}
|
|
424
464
|
if (action.type === "press_shortcut") {
|
|
@@ -440,7 +480,57 @@ export class NativeMacOSJobExecutor {
|
|
|
440
480
|
if (action.type === "take_screenshot") {
|
|
441
481
|
await reporter.progress(progressPercent, "Capturando screenshot do Mac");
|
|
442
482
|
const screenshotPath = await this.takeScreenshot(action.path);
|
|
443
|
-
|
|
483
|
+
const uploadable = await this.buildUploadableImage(screenshotPath);
|
|
484
|
+
const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
|
|
485
|
+
kind: "screenshot",
|
|
486
|
+
mimeTypeOverride: uploadable.mimeType,
|
|
487
|
+
fileNameOverride: uploadable.filename,
|
|
488
|
+
metadata: {
|
|
489
|
+
width: uploadable.dimensions?.width || undefined,
|
|
490
|
+
height: uploadable.dimensions?.height || undefined,
|
|
491
|
+
original_width: uploadable.originalDimensions?.width || undefined,
|
|
492
|
+
original_height: uploadable.originalDimensions?.height || undefined,
|
|
493
|
+
resized_for_upload: uploadable.resized,
|
|
494
|
+
},
|
|
495
|
+
});
|
|
496
|
+
if (screenshotArtifact) {
|
|
497
|
+
artifacts.push(screenshotArtifact);
|
|
498
|
+
completionNotes.push("Capturei a tela do Mac e anexei a imagem aqui no chat.");
|
|
499
|
+
}
|
|
500
|
+
else {
|
|
501
|
+
completionNotes.push(`Screenshot salvo em ${screenshotPath}`);
|
|
502
|
+
}
|
|
503
|
+
resultPayload.screenshot_path = screenshotPath;
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
if (action.type === "read_frontmost_page") {
|
|
507
|
+
await reporter.progress(progressPercent, `Lendo a pagina ativa em ${action.app || "Safari"}`);
|
|
508
|
+
const page = await this.readFrontmostPage(action.app || "Safari");
|
|
509
|
+
if (!page.text && this.bridgeConfig?.apiBaseUrl && this.bridgeConfig?.deviceToken) {
|
|
510
|
+
await reporter.progress(progressPercent, "Safari bloqueou leitura direta; vou analisar a pagina pela tela");
|
|
511
|
+
const screenshotPath = await this.takeScreenshot();
|
|
512
|
+
const uploadable = await this.buildUploadableImage(screenshotPath);
|
|
513
|
+
const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
|
|
514
|
+
kind: "screenshot",
|
|
515
|
+
mimeTypeOverride: uploadable.mimeType,
|
|
516
|
+
fileNameOverride: uploadable.filename,
|
|
517
|
+
metadata: {
|
|
518
|
+
purpose: "page_read_fallback",
|
|
519
|
+
width: uploadable.dimensions?.width || undefined,
|
|
520
|
+
height: uploadable.dimensions?.height || undefined,
|
|
521
|
+
original_width: uploadable.originalDimensions?.width || undefined,
|
|
522
|
+
original_height: uploadable.originalDimensions?.height || undefined,
|
|
523
|
+
resized_for_upload: uploadable.resized,
|
|
524
|
+
},
|
|
525
|
+
});
|
|
526
|
+
if (artifact?.storage_path) {
|
|
527
|
+
artifacts.push(artifact);
|
|
528
|
+
const answer = await this.analyzeUploadedArtifact(job.job_id, artifact.storage_path, "Leia o que esta visivel nesta pagina da web e resuma em portugues brasileiro o conteudo principal. Inclua titulos, chamadas e o que parecer mais importante na tela.", artifact.mime_type);
|
|
529
|
+
page.text = answer || page.text;
|
|
530
|
+
}
|
|
531
|
+
}
|
|
532
|
+
resultPayload.page = page;
|
|
533
|
+
completionNotes.push(`Li a pagina ${page.title || page.url || "ativa"} no navegador.`);
|
|
444
534
|
continue;
|
|
445
535
|
}
|
|
446
536
|
if (action.type === "read_file") {
|
|
@@ -461,19 +551,71 @@ export class NativeMacOSJobExecutor {
|
|
|
461
551
|
completionNotes.push(`Saida de \`${action.command}\`:\n${shellOutput}`);
|
|
462
552
|
continue;
|
|
463
553
|
}
|
|
554
|
+
if (action.type === "set_volume") {
|
|
555
|
+
await reporter.progress(progressPercent, `Ajustando volume para ${action.level}%`);
|
|
556
|
+
await this.setVolume(action.level);
|
|
557
|
+
completionNotes.push(`Volume ajustado para ${action.level}% no macOS.`);
|
|
558
|
+
continue;
|
|
559
|
+
}
|
|
560
|
+
if (action.type === "click_visual_target") {
|
|
561
|
+
if (action.app) {
|
|
562
|
+
await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do clique`);
|
|
563
|
+
await this.focusApp(action.app);
|
|
564
|
+
}
|
|
565
|
+
await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.description}`);
|
|
566
|
+
const screenshotPath = await this.takeScreenshot();
|
|
567
|
+
const uploadable = await this.buildUploadableImage(screenshotPath);
|
|
568
|
+
const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
|
|
569
|
+
kind: "screenshot",
|
|
570
|
+
mimeTypeOverride: uploadable.mimeType,
|
|
571
|
+
fileNameOverride: uploadable.filename,
|
|
572
|
+
metadata: {
|
|
573
|
+
purpose: "visual_click",
|
|
574
|
+
target: action.description,
|
|
575
|
+
width: uploadable.dimensions?.width || undefined,
|
|
576
|
+
height: uploadable.dimensions?.height || undefined,
|
|
577
|
+
original_width: uploadable.originalDimensions?.width || undefined,
|
|
578
|
+
original_height: uploadable.originalDimensions?.height || undefined,
|
|
579
|
+
resized_for_upload: uploadable.resized,
|
|
580
|
+
},
|
|
581
|
+
});
|
|
582
|
+
if (!artifact?.storage_path) {
|
|
583
|
+
throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
|
|
584
|
+
}
|
|
585
|
+
artifacts.push(artifact);
|
|
586
|
+
const artifactMetadata = artifact.metadata || {};
|
|
587
|
+
const width = Number(artifactMetadata.width || 0);
|
|
588
|
+
const height = Number(artifactMetadata.height || 0);
|
|
589
|
+
const originalWidth = Number(artifactMetadata.original_width || width || 0);
|
|
590
|
+
const originalHeight = Number(artifactMetadata.original_height || height || 0);
|
|
591
|
+
const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, action.description, width, height, artifact.mime_type);
|
|
592
|
+
if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
|
|
593
|
+
throw new Error(`Nao consegui localizar ${action.description} com confianca suficiente na tela.`);
|
|
594
|
+
}
|
|
595
|
+
await reporter.progress(progressPercent, `Clicando em ${action.description}`);
|
|
596
|
+
const scaledX = width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x;
|
|
597
|
+
const scaledY = height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y;
|
|
598
|
+
await this.clickPoint(scaledX, scaledY);
|
|
599
|
+
completionNotes.push(`Localizei e cliquei em ${action.description}.`);
|
|
600
|
+
resultPayload.last_click = {
|
|
601
|
+
...location,
|
|
602
|
+
x: scaledX,
|
|
603
|
+
y: scaledY,
|
|
604
|
+
};
|
|
605
|
+
continue;
|
|
606
|
+
}
|
|
464
607
|
await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
|
|
465
608
|
await this.openUrl(action.url, action.app);
|
|
609
|
+
await delay(1200);
|
|
610
|
+
completionNotes.push(`${humanizeUrl(action.url)} foi aberto${action.app ? ` em ${action.app}` : ""}.`);
|
|
466
611
|
}
|
|
467
612
|
const summary = completionNotes.length > 0
|
|
468
613
|
? completionNotes.join("\n\n")
|
|
469
614
|
: (actions.length === 1
|
|
470
615
|
? this.describeAction(actions[0])
|
|
471
616
|
: `${actions.length} ações executadas no macOS`);
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
summary,
|
|
475
|
-
actions,
|
|
476
|
-
});
|
|
617
|
+
resultPayload.summary = summary;
|
|
618
|
+
await reporter.completed(resultPayload);
|
|
477
619
|
}
|
|
478
620
|
finally {
|
|
479
621
|
this.cancelledJobs.delete(job.job_id);
|
|
@@ -578,6 +720,193 @@ end tell
|
|
|
578
720
|
await this.runCommand("screencapture", ["-x", screenshotPath]);
|
|
579
721
|
return screenshotPath;
|
|
580
722
|
}
|
|
723
|
+
async uploadArtifactForJob(jobId, localPath, options) {
|
|
724
|
+
if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
|
|
725
|
+
return null;
|
|
726
|
+
}
|
|
727
|
+
const bytes = await readFile(localPath);
|
|
728
|
+
const fileName = options?.fileNameOverride || path.basename(localPath);
|
|
729
|
+
const mimeType = options?.mimeTypeOverride || mimeTypeFromPath(fileName);
|
|
730
|
+
const dimensions = mimeType.startsWith("image/") ? await this.getImageDimensions(localPath) : null;
|
|
731
|
+
const metadata = {
|
|
732
|
+
...(options?.metadata || {}),
|
|
733
|
+
...(dimensions || {}),
|
|
734
|
+
};
|
|
735
|
+
const response = await uploadDeviceJobArtifact(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, jobId, {
|
|
736
|
+
filename: fileName,
|
|
737
|
+
contentType: mimeType,
|
|
738
|
+
bytes,
|
|
739
|
+
kind: options?.kind || "file",
|
|
740
|
+
metadata,
|
|
741
|
+
});
|
|
742
|
+
return response.artifact || null;
|
|
743
|
+
}
|
|
744
|
+
async analyzeUploadedArtifact(jobId, storagePath, question, mimeType) {
|
|
745
|
+
if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
|
|
746
|
+
return "";
|
|
747
|
+
}
|
|
748
|
+
const response = await postDeviceJson(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, `/v1/devices/jobs/${encodeURIComponent(jobId)}/vision/analyze`, {
|
|
749
|
+
storage_path: storagePath,
|
|
750
|
+
question,
|
|
751
|
+
mime_type: mimeType || "image/jpeg",
|
|
752
|
+
});
|
|
753
|
+
return String(response.answer || "").trim();
|
|
754
|
+
}
|
|
755
|
+
async readFrontmostPage(app) {
|
|
756
|
+
const targetApp = app || "Safari";
|
|
757
|
+
if (targetApp !== "Safari") {
|
|
758
|
+
throw new Error("Leitura de pagina frontmost esta disponivel apenas para Safari no momento.");
|
|
759
|
+
}
|
|
760
|
+
const script = `
|
|
761
|
+
tell application "Safari"
|
|
762
|
+
activate
|
|
763
|
+
if (count of windows) = 0 then error "Safari nao possui janelas abertas."
|
|
764
|
+
delay 1
|
|
765
|
+
set pageJson to do JavaScript "(function(){const title=document.title||''; const url=location.href||''; const text=((document.body&&document.body.innerText)||'').trim().slice(0, 12000); return JSON.stringify({title:title,url:url,text:text});})();" in current tab of front window
|
|
766
|
+
end tell
|
|
767
|
+
return pageJson
|
|
768
|
+
`;
|
|
769
|
+
try {
|
|
770
|
+
const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
|
|
771
|
+
const parsed = JSON.parse(stdout.trim() || "{}");
|
|
772
|
+
return {
|
|
773
|
+
title: asString(parsed.title) || "",
|
|
774
|
+
url: asString(parsed.url) || "",
|
|
775
|
+
text: asString(parsed.text) || "",
|
|
776
|
+
};
|
|
777
|
+
}
|
|
778
|
+
catch (error) {
|
|
779
|
+
const detail = error instanceof Error ? error.message : String(error);
|
|
780
|
+
if (!detail.toLowerCase().includes("allow javascript from apple events")) {
|
|
781
|
+
throw error;
|
|
782
|
+
}
|
|
783
|
+
const metadataScript = `
|
|
784
|
+
tell application "Safari"
|
|
785
|
+
activate
|
|
786
|
+
if (count of windows) = 0 then error "Safari nao possui janelas abertas."
|
|
787
|
+
set pageTitle to name of current tab of front window
|
|
788
|
+
set pageUrl to URL of current tab of front window
|
|
789
|
+
end tell
|
|
790
|
+
return pageTitle & linefeed & pageUrl
|
|
791
|
+
`;
|
|
792
|
+
const { stdout } = await this.runCommandCapture("osascript", ["-e", metadataScript]);
|
|
793
|
+
const [title, url] = stdout.split("\n");
|
|
794
|
+
return {
|
|
795
|
+
title: String(title || "").trim(),
|
|
796
|
+
url: String(url || "").trim(),
|
|
797
|
+
text: "",
|
|
798
|
+
};
|
|
799
|
+
}
|
|
800
|
+
}
|
|
801
|
+
async setVolume(level) {
|
|
802
|
+
const bounded = Math.max(0, Math.min(Math.round(level), 100));
|
|
803
|
+
await this.runCommand("osascript", ["-e", `set volume output volume ${bounded}`]);
|
|
804
|
+
}
|
|
805
|
+
async locateVisualTarget(jobId, storagePath, target, width, height, mimeType) {
|
|
806
|
+
if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
|
|
807
|
+
throw new Error("Otto Bridge nao possui configuracao para usar visao no backend.");
|
|
808
|
+
}
|
|
809
|
+
const response = await postDeviceJson(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, `/v1/devices/jobs/${encodeURIComponent(jobId)}/vision/locate`, {
|
|
810
|
+
storage_path: storagePath,
|
|
811
|
+
target,
|
|
812
|
+
image_width: Math.max(1, width),
|
|
813
|
+
image_height: Math.max(1, height),
|
|
814
|
+
mime_type: mimeType || "image/png",
|
|
815
|
+
});
|
|
816
|
+
return response.location || {};
|
|
817
|
+
}
|
|
818
|
+
async clickPoint(x, y) {
|
|
819
|
+
const script = `
|
|
820
|
+
import Cocoa
|
|
821
|
+
import ApplicationServices
|
|
822
|
+
|
|
823
|
+
let x = Double(CommandLine.arguments[1]) ?? 0
|
|
824
|
+
let y = Double(CommandLine.arguments[2]) ?? 0
|
|
825
|
+
let point = CGPoint(x: x, y: y)
|
|
826
|
+
|
|
827
|
+
func post(_ type: CGEventType) {
|
|
828
|
+
guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
|
|
829
|
+
fputs("failed to create mouse event\\n", stderr)
|
|
830
|
+
exit(1)
|
|
831
|
+
}
|
|
832
|
+
event.post(tap: .cghidEventTap)
|
|
833
|
+
}
|
|
834
|
+
|
|
835
|
+
post(.mouseMoved)
|
|
836
|
+
usleep(120000)
|
|
837
|
+
post(.leftMouseDown)
|
|
838
|
+
usleep(80000)
|
|
839
|
+
post(.leftMouseUp)
|
|
840
|
+
`;
|
|
841
|
+
await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
|
|
842
|
+
}
|
|
843
|
+
async getImageDimensions(filePath) {
|
|
844
|
+
try {
|
|
845
|
+
const { stdout } = await this.runCommandCapture("sips", ["-g", "pixelWidth", "-g", "pixelHeight", filePath]);
|
|
846
|
+
const widthMatch = stdout.match(/pixelWidth:\s*(\d+)/i);
|
|
847
|
+
const heightMatch = stdout.match(/pixelHeight:\s*(\d+)/i);
|
|
848
|
+
if (!widthMatch || !heightMatch) {
|
|
849
|
+
return null;
|
|
850
|
+
}
|
|
851
|
+
return {
|
|
852
|
+
width: Number(widthMatch[1]),
|
|
853
|
+
height: Number(heightMatch[1]),
|
|
854
|
+
};
|
|
855
|
+
}
|
|
856
|
+
catch {
|
|
857
|
+
return null;
|
|
858
|
+
}
|
|
859
|
+
}
|
|
860
|
+
async buildUploadableImage(localPath) {
|
|
861
|
+
const originalDimensions = await this.getImageDimensions(localPath);
|
|
862
|
+
const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
|
|
863
|
+
await mkdir(artifactsDir, { recursive: true });
|
|
864
|
+
let sourcePath = localPath;
|
|
865
|
+
let mimeType = mimeTypeFromPath(localPath);
|
|
866
|
+
let filename = path.basename(localPath);
|
|
867
|
+
let resized = false;
|
|
868
|
+
const conversionSteps = [
|
|
869
|
+
{ width: 1280, quality: 42 },
|
|
870
|
+
{ width: 1024, quality: 35 },
|
|
871
|
+
{ width: 900, quality: 30 },
|
|
872
|
+
{ width: 768, quality: 26 },
|
|
873
|
+
{ width: 640, quality: 22 },
|
|
874
|
+
{ width: 540, quality: 18 },
|
|
875
|
+
{ width: 480, quality: 16 },
|
|
876
|
+
];
|
|
877
|
+
for (const step of conversionSteps) {
|
|
878
|
+
const candidatePath = path.join(artifactsDir, `${path.basename(localPath, path.extname(localPath))}-${step.width}w-q${step.quality}.jpg`);
|
|
879
|
+
await this.runCommand("sips", [
|
|
880
|
+
"-s",
|
|
881
|
+
"format",
|
|
882
|
+
"jpeg",
|
|
883
|
+
"-s",
|
|
884
|
+
"formatOptions",
|
|
885
|
+
String(step.quality),
|
|
886
|
+
"--resampleWidth",
|
|
887
|
+
String(step.width),
|
|
888
|
+
localPath,
|
|
889
|
+
"--out",
|
|
890
|
+
candidatePath,
|
|
891
|
+
]);
|
|
892
|
+
const candidateStat = await stat(candidatePath);
|
|
893
|
+
sourcePath = candidatePath;
|
|
894
|
+
mimeType = "image/jpeg";
|
|
895
|
+
filename = path.basename(candidatePath);
|
|
896
|
+
resized = true;
|
|
897
|
+
if (candidateStat.size <= 220_000) {
|
|
898
|
+
break;
|
|
899
|
+
}
|
|
900
|
+
}
|
|
901
|
+
return {
|
|
902
|
+
path: sourcePath,
|
|
903
|
+
mimeType,
|
|
904
|
+
filename,
|
|
905
|
+
dimensions: await this.getImageDimensions(sourcePath),
|
|
906
|
+
originalDimensions,
|
|
907
|
+
resized,
|
|
908
|
+
};
|
|
909
|
+
}
|
|
581
910
|
async readLocalFile(filePath, maxChars = 4000) {
|
|
582
911
|
const resolved = expandUserPath(filePath);
|
|
583
912
|
const content = await readFile(resolved, "utf8");
|
|
@@ -602,7 +931,7 @@ end tell
|
|
|
602
931
|
}
|
|
603
932
|
async runShellCommand(command, cwd) {
|
|
604
933
|
if (!isSafeShellCommand(command)) {
|
|
605
|
-
throw new Error("
|
|
934
|
+
throw new Error("Nenhum comando shell foi informado para execucao local.");
|
|
606
935
|
}
|
|
607
936
|
const resolvedCwd = cwd ? expandUserPath(cwd) : process.cwd();
|
|
608
937
|
const { stdout, stderr } = await this.runCommandCapture("/bin/zsh", ["-lc", command], {
|
|
@@ -642,6 +971,9 @@ end tell
|
|
|
642
971
|
if (action.type === "take_screenshot") {
|
|
643
972
|
return "Screenshot capturado no macOS";
|
|
644
973
|
}
|
|
974
|
+
if (action.type === "read_frontmost_page") {
|
|
975
|
+
return `Pagina ativa lida em ${action.app || "Safari"}`;
|
|
976
|
+
}
|
|
645
977
|
if (action.type === "read_file") {
|
|
646
978
|
return `${action.path} foi lido no macOS`;
|
|
647
979
|
}
|
|
@@ -651,6 +983,12 @@ end tell
|
|
|
651
983
|
if (action.type === "run_shell") {
|
|
652
984
|
return `Comando ${action.command} executado no macOS`;
|
|
653
985
|
}
|
|
986
|
+
if (action.type === "set_volume") {
|
|
987
|
+
return `Volume ajustado para ${action.level}% no macOS`;
|
|
988
|
+
}
|
|
989
|
+
if (action.type === "click_visual_target") {
|
|
990
|
+
return `Clique guiado executado para ${action.description}`;
|
|
991
|
+
}
|
|
654
992
|
const target = humanizeUrl(action.url);
|
|
655
993
|
return `${target} foi aberto${action.app ? ` em ${action.app}` : ""}`;
|
|
656
994
|
}
|
|
@@ -695,10 +1033,6 @@ end tell
|
|
|
695
1033
|
if (stderrText) {
|
|
696
1034
|
console.warn(`[otto-bridge] ${command} stderr=${stderrText}`);
|
|
697
1035
|
}
|
|
698
|
-
const stdoutText = stdout.trim();
|
|
699
|
-
if (stdoutText) {
|
|
700
|
-
console.log(`[otto-bridge] ${command} stdout=${stdoutText}`);
|
|
701
|
-
}
|
|
702
1036
|
return { stdout, stderr };
|
|
703
1037
|
}
|
|
704
1038
|
catch (error) {
|
package/dist/http.js
CHANGED
|
@@ -38,3 +38,32 @@ export async function postJson(apiBaseUrl, pathname, body) {
|
|
|
38
38
|
body: JSON.stringify(body),
|
|
39
39
|
});
|
|
40
40
|
}
|
|
41
|
+
function buildDeviceAuthHeaders(deviceToken, headers) {
|
|
42
|
+
const next = new Headers(headers || {});
|
|
43
|
+
if (deviceToken) {
|
|
44
|
+
next.set("Authorization", `Bearer ${deviceToken}`);
|
|
45
|
+
}
|
|
46
|
+
return next;
|
|
47
|
+
}
|
|
48
|
+
export async function postDeviceJson(apiBaseUrl, deviceToken, pathname, body) {
|
|
49
|
+
return await requestJson(apiBaseUrl, pathname, {
|
|
50
|
+
method: "POST",
|
|
51
|
+
headers: buildDeviceAuthHeaders(deviceToken, {
|
|
52
|
+
"Content-Type": "application/json",
|
|
53
|
+
}),
|
|
54
|
+
body: JSON.stringify(body),
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
export async function uploadDeviceJobArtifact(apiBaseUrl, deviceToken, jobId, params) {
|
|
58
|
+
const form = new FormData();
|
|
59
|
+
form.append("file", new Blob([Buffer.from(params.bytes)], { type: params.contentType || "application/octet-stream" }), params.filename);
|
|
60
|
+
form.append("kind", String(params.kind || "file"));
|
|
61
|
+
if (params.metadata && Object.keys(params.metadata).length > 0) {
|
|
62
|
+
form.append("metadata", JSON.stringify(params.metadata));
|
|
63
|
+
}
|
|
64
|
+
return await requestJson(apiBaseUrl, `/v1/devices/jobs/${encodeURIComponent(jobId)}/artifacts`, {
|
|
65
|
+
method: "POST",
|
|
66
|
+
headers: buildDeviceAuthHeaders(deviceToken),
|
|
67
|
+
body: form,
|
|
68
|
+
});
|
|
69
|
+
}
|
package/dist/main.js
CHANGED
|
File without changes
|
package/dist/runtime.js
CHANGED
|
@@ -280,7 +280,7 @@ export class BridgeRuntime {
|
|
|
280
280
|
return new ClawdCursorJobExecutor(config.executor);
|
|
281
281
|
}
|
|
282
282
|
if (config.executor.type === "native-macos") {
|
|
283
|
-
return new NativeMacOSJobExecutor();
|
|
283
|
+
return new NativeMacOSJobExecutor(config);
|
|
284
284
|
}
|
|
285
285
|
return new MockJobExecutor();
|
|
286
286
|
}
|
package/dist/types.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
export const BRIDGE_CONFIG_VERSION = 1;
|
|
2
|
-
export const BRIDGE_VERSION = "0.
|
|
2
|
+
export const BRIDGE_VERSION = "0.5.1";
|
|
3
3
|
export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
|
|
4
4
|
export const DEFAULT_API_BASE_URL = "http://localhost:8000";
|
|
5
5
|
export const DEFAULT_POLL_INTERVAL_MS = 3000;
|