@leg3ndy/otto-bridge 0.4.2 → 0.5.0

package/dist/executors/native_macos.js

@@ -4,6 +4,7 @@ import os from "node:os";
 import path from "node:path";
 import process from "node:process";
 import { JobCancelledError } from "./shared.js";
+import { postDeviceJson, uploadDeviceJobArtifact } from "../http.js";
 const KNOWN_APPS = [
     { canonical: "Safari", patterns: [/\bsafari\b/i] },
     { canonical: "Google Chrome", patterns: [/\bgoogle chrome\b/i, /\bchrome\b/i] },
@@ -87,6 +88,22 @@ function humanizeUrl(url) {
         return normalized;
     }
 }
+function mimeTypeFromPath(filePath) {
+    const ext = path.extname(filePath).toLowerCase();
+    if (ext === ".png")
+        return "image/png";
+    if (ext === ".jpg" || ext === ".jpeg")
+        return "image/jpeg";
+    if (ext === ".webp")
+        return "image/webp";
+    if (ext === ".gif")
+        return "image/gif";
+    if (ext === ".txt" || ext === ".md")
+        return "text/plain";
+    if (ext === ".json")
+        return "application/json";
+    return "application/octet-stream";
+}
 function expandUserPath(value) {
     const trimmed = value.trim();
     if (!trimmed) {
@@ -109,6 +126,9 @@ function clipText(value, maxLength) {
     }
     return `${value.slice(0, maxLength)}...`;
 }
+function delay(ms) {
+    return new Promise((resolve) => setTimeout(resolve, ms));
+}
 function escapeHtml(value) {
     return value
         .replace(/&/g, "&")
@@ -307,6 +327,13 @@ function parseStructuredActions(job) {
             actions.push({ type: "take_screenshot", path: savePath || undefined });
             continue;
         }
+        if (type === "read_frontmost_page" || type === "read_page" || type === "read_webpage") {
+            actions.push({
+                type: "read_frontmost_page",
+                app: asString(action.app) || asString(action.application) || "Safari",
+            });
+            continue;
+        }
         if (type === "read_file" || type === "read_local_file") {
             const filePath = asString(action.path);
             if (filePath) {
@@ -327,6 +354,25 @@ function parseStructuredActions(job) {
             if (command) {
                 actions.push({ type: "run_shell", command, cwd: cwd || undefined });
             }
+            continue;
+        }
+        if (type === "set_volume" || type === "volume") {
+            const rawLevel = Number(action.level);
+            if (Number.isFinite(rawLevel)) {
+                actions.push({ type: "set_volume", level: Math.max(0, Math.min(Math.round(rawLevel), 100)) });
+            }
+            continue;
+        }
+        if (type === "click_visual_target" || type === "click_target") {
+            const description = asString(action.description) || asString(action.target);
+            if (description) {
+                actions.push({
+                    type: "click_visual_target",
+                    description,
+                    app: asString(action.app) || undefined,
+                });
+            }
+            continue;
         }
     }
     return actions;
@@ -361,6 +407,30 @@ function deriveActionsFromText(job) {
     const task = extractTaskText(job);
     const detectedApp = detectKnownApp(task);
     const detectedUrl = detectUrl(task);
+    const normalizedTask = normalizeText(task);
+    if (/\b(volume|som|audio)\b/i.test(task)) {
+        const percentMatch = task.match(/(\d{1,3})\s*%/);
+        let level = 50;
+        if (percentMatch?.[1]) {
+            level = Math.max(0, Math.min(Number(percentMatch[1]), 100));
+        }
+        else if (/\b(mudo|mute|silencia)\b/i.test(task)) {
+            level = 0;
+        }
+        else if (/\b(aumenta|aumente|mais alto)\b/i.test(task)) {
+            level = 80;
+        }
+        else if (/\b(diminui|abaixa|mais baixo)\b/i.test(task)) {
+            level = 25;
+        }
+        return [{ type: "set_volume", level }];
+    }
+    if ((normalizedTask.includes("leia") || normalizedTask.includes("ler")) && detectedUrl) {
+        return [
+            { type: "open_url", url: detectedUrl, app: detectedApp || "Safari" },
+            { type: "read_frontmost_page", app: detectedApp || "Safari" },
+        ];
+    }
     if (detectedUrl) {
         return [{
                 type: "open_url",
@@ -384,8 +454,12 @@ function extractActions(job) {
     return deriveActionsFromText(job);
 }
 export class NativeMacOSJobExecutor {
+    bridgeConfig;
     cancelledJobs = new Set();
     activeChild = null;
+    constructor(bridgeConfig) {
+        this.bridgeConfig = bridgeConfig;
+    }
     async run(job, reporter) {
         if (process.platform !== "darwin") {
             throw new Error("The native-macos executor only runs on macOS");
@@ -407,6 +481,13 @@ export class NativeMacOSJobExecutor {
         }
         try {
             const completionNotes = [];
+            const artifacts = [];
+            const resultPayload = {
+                executor: "native-macos",
+                actions,
+                artifacts,
+                action_summaries: completionNotes,
+            };
             for (let index = 0; index < actions.length; index += 1) {
                 this.assertNotCancelled(job.job_id);
                 const action = actions[index];
@@ -440,7 +521,24 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "take_screenshot") {
                     await reporter.progress(progressPercent, "Capturando screenshot do Mac");
                     const screenshotPath = await this.takeScreenshot(action.path);
-                    completionNotes.push(`Screenshot salvo em ${screenshotPath}`);
+                    const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
+                        kind: "screenshot",
+                    });
+                    if (screenshotArtifact) {
+                        artifacts.push(screenshotArtifact);
+                        completionNotes.push("Capturei a tela do Mac e anexei a imagem aqui no chat.");
+                    }
+                    else {
+                        completionNotes.push(`Screenshot salvo em ${screenshotPath}`);
+                    }
+                    resultPayload.screenshot_path = screenshotPath;
+                    continue;
+                }
+                if (action.type === "read_frontmost_page") {
+                    await reporter.progress(progressPercent, `Lendo a pagina ativa em ${action.app || "Safari"}`);
+                    const page = await this.readFrontmostPage(action.app || "Safari");
+                    resultPayload.page = page;
+                    completionNotes.push(`Li a pagina ${page.title || page.url || "ativa"} no navegador.`);
                     continue;
                 }
                 if (action.type === "read_file") {
@@ -461,19 +559,51 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Saida de \`${action.command}\`:\n${shellOutput}`);
                     continue;
                 }
+                if (action.type === "set_volume") {
+                    await reporter.progress(progressPercent, `Ajustando volume para ${action.level}%`);
+                    await this.setVolume(action.level);
+                    completionNotes.push(`Volume ajustado para ${action.level}% no macOS.`);
+                    continue;
+                }
+                if (action.type === "click_visual_target") {
+                    if (action.app) {
+                        await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do clique`);
+                        await this.focusApp(action.app);
+                    }
+                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.description}`);
+                    const screenshotPath = await this.takeScreenshot();
+                    const artifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
+                        kind: "screenshot",
+                        metadata: { purpose: "visual_click", target: action.description },
+                    });
+                    if (!artifact?.storage_path) {
+                        throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
+                    }
+                    artifacts.push(artifact);
+                    const artifactMetadata = artifact.metadata || {};
+                    const width = Number(artifactMetadata.width || 0);
+                    const height = Number(artifactMetadata.height || 0);
+                    const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, action.description, width, height, artifact.mime_type);
+                    if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+                        throw new Error(`Nao consegui localizar ${action.description} com confianca suficiente na tela.`);
+                    }
+                    await reporter.progress(progressPercent, `Clicando em ${action.description}`);
+                    await this.clickPoint(location.x, location.y);
+                    completionNotes.push(`Localizei e cliquei em ${action.description}.`);
+                    resultPayload.last_click = location;
+                    continue;
+                }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
+                await delay(1200);
             }
             const summary = completionNotes.length > 0
                 ? completionNotes.join("\n\n")
                 : (actions.length === 1
                     ? this.describeAction(actions[0])
                     : `${actions.length} ações executadas no macOS`);
-            await reporter.completed({
-                executor: "native-macos",
-                summary,
-                actions,
-            });
+            resultPayload.summary = summary;
+            await reporter.completed(resultPayload);
         }
         finally {
             this.cancelledJobs.delete(job.job_id);
@@ -578,6 +708,108 @@ end tell
         await this.runCommand("screencapture", ["-x", screenshotPath]);
         return screenshotPath;
     }
+    async uploadArtifactForJob(jobId, localPath, options) {
+        if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
+            return null;
+        }
+        const bytes = await readFile(localPath);
+        const fileName = path.basename(localPath);
+        const mimeType = mimeTypeFromPath(fileName);
+        const dimensions = mimeType.startsWith("image/") ? await this.getImageDimensions(localPath) : null;
+        const metadata = {
+            ...(options?.metadata || {}),
+            ...(dimensions || {}),
+        };
+        const response = await uploadDeviceJobArtifact(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, jobId, {
+            filename: fileName,
+            contentType: mimeType,
+            bytes,
+            kind: options?.kind || "file",
+            metadata,
+        });
+        return response.artifact || null;
+    }
+    async readFrontmostPage(app) {
+        const targetApp = app || "Safari";
+        if (targetApp !== "Safari") {
+            throw new Error("Leitura de pagina frontmost esta disponivel apenas para Safari no momento.");
+        }
+        const script = `
+tell application "Safari"
+  activate
+  if (count of windows) = 0 then error "Safari nao possui janelas abertas."
+  delay 1
+  set pageJson to do JavaScript "(function(){const title=document.title||''; const url=location.href||''; const text=((document.body&&document.body.innerText)||'').trim().slice(0, 12000); return JSON.stringify({title:title,url:url,text:text});})();" in current tab of front window
+end tell
+return pageJson
+`;
+        const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
+        const parsed = JSON.parse(stdout.trim() || "{}");
+        return {
+            title: asString(parsed.title) || "",
+            url: asString(parsed.url) || "",
+            text: asString(parsed.text) || "",
+        };
+    }
+    async setVolume(level) {
+        const bounded = Math.max(0, Math.min(Math.round(level), 100));
+        await this.runCommand("osascript", ["-e", `set volume output volume ${bounded}`]);
+    }
+    async locateVisualTarget(jobId, storagePath, target, width, height, mimeType) {
+        if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
+            throw new Error("Otto Bridge nao possui configuracao para usar visao no backend.");
+        }
+        const response = await postDeviceJson(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, `/v1/devices/jobs/${encodeURIComponent(jobId)}/vision/locate`, {
+            storage_path: storagePath,
+            target,
+            image_width: Math.max(1, width),
+            image_height: Math.max(1, height),
+            mime_type: mimeType || "image/png",
+        });
+        return response.location || {};
+    }
+    async clickPoint(x, y) {
+        const script = `
+import Cocoa
+import ApplicationServices
+
+let x = Double(CommandLine.arguments[1]) ?? 0
+let y = Double(CommandLine.arguments[2]) ?? 0
+let point = CGPoint(x: x, y: y)
+
+func post(_ type: CGEventType) {
+  guard let event = CGEvent(mouseEventSource: nil, mouseType: type, mouseCursorPosition: point, mouseButton: .left) else {
+    fputs("failed to create mouse event\\n", stderr)
+    exit(1)
+  }
+  event.post(tap: .cghidEventTap)
+}
+
+post(.mouseMoved)
+usleep(120000)
+post(.leftMouseDown)
+usleep(80000)
+post(.leftMouseUp)
+`;
+        await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
+    }
+    async getImageDimensions(filePath) {
+        try {
+            const { stdout } = await this.runCommandCapture("sips", ["-g", "pixelWidth", "-g", "pixelHeight", filePath]);
+            const widthMatch = stdout.match(/pixelWidth:\s*(\d+)/i);
+            const heightMatch = stdout.match(/pixelHeight:\s*(\d+)/i);
+            if (!widthMatch || !heightMatch) {
+                return null;
+            }
+            return {
+                width: Number(widthMatch[1]),
+                height: Number(heightMatch[1]),
+            };
+        }
+        catch {
+            return null;
+        }
+    }
     async readLocalFile(filePath, maxChars = 4000) {
         const resolved = expandUserPath(filePath);
         const content = await readFile(resolved, "utf8");
@@ -642,6 +874,9 @@ end tell
         if (action.type === "take_screenshot") {
             return "Screenshot capturado no macOS";
         }
+        if (action.type === "read_frontmost_page") {
+            return `Pagina ativa lida em ${action.app || "Safari"}`;
+        }
         if (action.type === "read_file") {
             return `${action.path} foi lido no macOS`;
         }
@@ -651,6 +886,12 @@ end tell
         if (action.type === "run_shell") {
             return `Comando ${action.command} executado no macOS`;
         }
+        if (action.type === "set_volume") {
+            return `Volume ajustado para ${action.level}% no macOS`;
+        }
+        if (action.type === "click_visual_target") {
+            return `Clique guiado executado para ${action.description}`;
+        }
         const target = humanizeUrl(action.url);
         return `${target} foi aberto${action.app ? ` em ${action.app}` : ""}`;
     }

package/dist/http.js

@@ -38,3 +38,32 @@ export async function postJson(apiBaseUrl, pathname, body) {
         body: JSON.stringify(body),
     });
 }
+function buildDeviceAuthHeaders(deviceToken, headers) {
+    const next = new Headers(headers || {});
+    if (deviceToken) {
+        next.set("Authorization", `Bearer ${deviceToken}`);
+    }
+    return next;
+}
+export async function postDeviceJson(apiBaseUrl, deviceToken, pathname, body) {
+    return await requestJson(apiBaseUrl, pathname, {
+        method: "POST",
+        headers: buildDeviceAuthHeaders(deviceToken, {
+            "Content-Type": "application/json",
+        }),
+        body: JSON.stringify(body),
+    });
+}
+export async function uploadDeviceJobArtifact(apiBaseUrl, deviceToken, jobId, params) {
+    const form = new FormData();
+    form.append("file", new Blob([Buffer.from(params.bytes)], { type: params.contentType || "application/octet-stream" }), params.filename);
+    form.append("kind", String(params.kind || "file"));
+    if (params.metadata && Object.keys(params.metadata).length > 0) {
+        form.append("metadata", JSON.stringify(params.metadata));
+    }
+    return await requestJson(apiBaseUrl, `/v1/devices/jobs/${encodeURIComponent(jobId)}/artifacts`, {
+        method: "POST",
+        headers: buildDeviceAuthHeaders(deviceToken),
+        body: form,
+    });
+}

package/dist/runtime.js

@@ -280,7 +280,7 @@ export class BridgeRuntime {
             return new ClawdCursorJobExecutor(config.executor);
         }
         if (config.executor.type === "native-macos") {
-            return new NativeMacOSJobExecutor();
+            return new NativeMacOSJobExecutor(config);
         }
         return new MockJobExecutor();
     }

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.4.2";
+export const BRIDGE_VERSION = "0.5.0";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.4.2",
+  "version": "0.5.0",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",