@leg3ndy/otto-bridge 0.5.0 → 0.5.2

package/README.md

@@ -11,6 +11,8 @@ Companion local do Otto para:
 
 Para um passo a passo de instalacao, pareamento, uso, desconexao e desinstalacao, veja [USER_GUIDE.md](https://github.com/LGCYYL/ottoai/blob/main/otto-bridge/USER_GUIDE.md).
 
+Para o estado atual da arquitetura, capacidades entregues, limitacoes e roadmap do Otto Bridge, veja [`leg3ndy-ai-backend/docs/OTTO_BRIDGE_ARCHITECTURE.md`](../leg3ndy-ai-backend/docs/OTTO_BRIDGE_ARCHITECTURE.md).
+
 ## Distribuicao
 
 Fluxo recomendado agora:

package/dist/executors/native_macos.js

@@ -88,6 +88,39 @@ function humanizeUrl(url) {
         return normalized;
     }
 }
+function uniqueStrings(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        const text = String(value || "").trim();
+        if (!text)
+            continue;
+        const key = normalizeText(text);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        result.push(text);
+    }
+    return result;
+}
+function looksLikeAffirmativeVisualVerification(answer) {
+    const normalized = normalizeText(answer || "");
+    if (!normalized)
+        return false;
+    if (normalized.startsWith("sim"))
+        return true;
+    if (normalized.startsWith("nao") || normalized.startsWith("não"))
+        return false;
+    return (normalized.includes("tocando")
+        || normalized.includes("reproduzindo")
+        || normalized.includes("em reproducao")
+        || normalized.includes("em reprodução")
+        || normalized.includes("botao de pausa")
+        || normalized.includes("botão de pausa")
+        || normalized.includes("faixa ativa")
+        || normalized.includes("resultado selecionado")
+        || normalized.includes("foi acionado"));
+}
 function mimeTypeFromPath(filePath) {
     const ext = path.extname(filePath).toLowerCase();
     if (ext === ".png")
@@ -175,50 +208,7 @@ function isSafeShellCommand(command) {
     if (!trimmed) {
         return false;
     }
-    const forbiddenPatterns = [
-        /(^|[;&|])\s*sudo\b/i,
-        /\brm\b/i,
-        /\bmv\b/i,
-        /\bcp\b/i,
-        /\bchmod\b/i,
-        /\bchown\b/i,
-        /\bshutdown\b/i,
-        /\breboot\b/i,
-        /\bmkfs\b/i,
-        /\bdd\b/i,
-        /\bkill(?:all)?\b/i,
-        />/,
-        />>/,
-    ];
-    if (forbiddenPatterns.some((pattern) => pattern.test(trimmed))) {
-        return false;
-    }
-    const normalized = trimmed.replace(/\s+/g, " ");
-    const allowedPrefixes = [
-        "pwd",
-        "ls",
-        "cat ",
-        "cat",
-        "sed ",
-        "rg ",
-        "find ",
-        "git status",
-        "git log",
-        "git diff",
-        "head ",
-        "tail ",
-        "wc ",
-        "stat ",
-        "file ",
-        "mdls ",
-        "whoami",
-        "date",
-        "uname",
-        "python3 --version",
-        "node -v",
-        "npm -v",
-    ];
-    return allowedPrefixes.some((prefix) => normalized === prefix || normalized.startsWith(`${prefix} `));
+    return true;
 }
 function extractConfirmationOptions(job, actions) {
     const payload = asRecord(job.payload);
@@ -370,6 +360,10 @@ function parseStructuredActions(job) {
                     type: "click_visual_target",
                     description,
                     app: asString(action.app) || undefined,
+                    verification_prompt: asString(action.verification_prompt) || undefined,
+                    retry_descriptions: Array.isArray(action.retry_descriptions)
+                        ? action.retry_descriptions.map((item) => asString(item)).filter(Boolean)
+                        : undefined,
                 });
             }
             continue;
@@ -495,11 +489,13 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "open_app") {
                     await reporter.progress(progressPercent, `Abrindo ${action.app} no macOS`);
                     await this.openApp(action.app);
+                    completionNotes.push(`${action.app} foi aberto no macOS.`);
                     continue;
                 }
                 if (action.type === "focus_app") {
                     await reporter.progress(progressPercent, `Trazendo ${action.app} para frente`);
                     await this.focusApp(action.app);
+                    completionNotes.push(`${action.app} ficou em foco no macOS.`);
                     continue;
                 }
                 if (action.type === "press_shortcut") {
@@ -521,8 +517,19 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "take_screenshot") {
                     await reporter.progress(progressPercent, "Capturando screenshot do Mac");
                     const screenshotPath = await this.takeScreenshot(action.path);
-                    const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
+                    const uploadable = await this.buildUploadableImage(screenshotPath);
+                    const screenshotArtifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
                         kind: "screenshot",
+                        mimeTypeOverride: uploadable.mimeType,
+                        fileNameOverride: uploadable.filename,
+                        metadata: {
+                            visible_in_chat: true,
+                            width: uploadable.dimensions?.width || undefined,
+                            height: uploadable.dimensions?.height || undefined,
+                            original_width: uploadable.originalDimensions?.width || undefined,
+                            original_height: uploadable.originalDimensions?.height || undefined,
+                            resized_for_upload: uploadable.resized,
+                        },
                     });
                     if (screenshotArtifact) {
                         artifacts.push(screenshotArtifact);
@@ -537,6 +544,30 @@ export class NativeMacOSJobExecutor {
                 if (action.type === "read_frontmost_page") {
                     await reporter.progress(progressPercent, `Lendo a pagina ativa em ${action.app || "Safari"}`);
                     const page = await this.readFrontmostPage(action.app || "Safari");
+                    if (!page.text && this.bridgeConfig?.apiBaseUrl && this.bridgeConfig?.deviceToken) {
+                        await reporter.progress(progressPercent, "Safari bloqueou leitura direta; vou analisar a pagina pela tela");
+                        const screenshotPath = await this.takeScreenshot();
+                        const uploadable = await this.buildUploadableImage(screenshotPath);
+                        const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
+                            kind: "screenshot",
+                            mimeTypeOverride: uploadable.mimeType,
+                            fileNameOverride: uploadable.filename,
+                            metadata: {
+                                purpose: "page_read_fallback",
+                                visible_in_chat: false,
+                                width: uploadable.dimensions?.width || undefined,
+                                height: uploadable.dimensions?.height || undefined,
+                                original_width: uploadable.originalDimensions?.width || undefined,
+                                original_height: uploadable.originalDimensions?.height || undefined,
+                                resized_for_upload: uploadable.resized,
+                            },
+                        });
+                        if (artifact?.storage_path) {
+                            artifacts.push(artifact);
+                            const answer = await this.analyzeUploadedArtifact(job.job_id, artifact.storage_path, "Leia o que esta visivel nesta pagina da web e resuma em portugues brasileiro o conteudo principal. Inclua titulos, chamadas e o que parecer mais importante na tela.", artifact.mime_type);
+                            page.text = answer || page.text;
+                        }
+                    }
                     resultPayload.page = page;
                     completionNotes.push(`Li a pagina ${page.title || page.url || "ativa"} no navegador.`);
                     continue;
@@ -570,32 +601,94 @@ export class NativeMacOSJobExecutor {
                         await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do clique`);
                         await this.focusApp(action.app);
                     }
-                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.description}`);
-                    const screenshotPath = await this.takeScreenshot();
-                    const artifact = await this.uploadArtifactForJob(job.job_id, screenshotPath, {
-                        kind: "screenshot",
-                        metadata: { purpose: "visual_click", target: action.description },
-                    });
-                    if (!artifact?.storage_path) {
-                        throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
+                    const targetDescriptions = uniqueStrings([action.description, ...(action.retry_descriptions || [])]);
+                    let clickSucceeded = false;
+                    let lastFailureReason = "";
+                    for (let attempt = 0; attempt < targetDescriptions.length; attempt += 1) {
+                        const targetDescription = targetDescriptions[attempt];
+                        await reporter.progress(progressPercent, `Capturando a tela para localizar ${targetDescription}`);
+                        const screenshotPath = await this.takeScreenshot();
+                        const uploadable = await this.buildUploadableImage(screenshotPath);
+                        const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
+                            kind: "screenshot",
+                            mimeTypeOverride: uploadable.mimeType,
+                            fileNameOverride: uploadable.filename,
+                            metadata: {
+                                purpose: "visual_click",
+                                visible_in_chat: false,
+                                target: targetDescription,
+                                width: uploadable.dimensions?.width || undefined,
+                                height: uploadable.dimensions?.height || undefined,
+                                original_width: uploadable.originalDimensions?.width || undefined,
+                                original_height: uploadable.originalDimensions?.height || undefined,
+                                resized_for_upload: uploadable.resized,
+                            },
+                        });
+                        if (!artifact?.storage_path) {
+                            throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
+                        }
+                        artifacts.push(artifact);
+                        const artifactMetadata = artifact.metadata || {};
+                        const width = Number(artifactMetadata.width || 0);
+                        const height = Number(artifactMetadata.height || 0);
+                        const originalWidth = Number(artifactMetadata.original_width || width || 0);
+                        const originalHeight = Number(artifactMetadata.original_height || height || 0);
+                        const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, targetDescription, width, height, artifact.mime_type);
+                        if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+                            lastFailureReason = `Nao consegui localizar ${targetDescription} com confianca suficiente na tela.`;
+                            continue;
+                        }
+                        await reporter.progress(progressPercent, `Clicando em ${targetDescription}`);
+                        const scaledX = width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x;
+                        const scaledY = height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y;
+                        await this.clickPoint(scaledX, scaledY);
+                        resultPayload.last_click = {
+                            ...location,
+                            x: scaledX,
+                            y: scaledY,
+                        };
+                        if (action.verification_prompt) {
+                            await delay(1600);
+                            await reporter.progress(progressPercent, "Validando visualmente se a ação funcionou");
+                            const afterClickPath = await this.takeScreenshot();
+                            const afterClickUpload = await this.buildUploadableImage(afterClickPath);
+                            const afterClickArtifact = await this.uploadArtifactForJob(job.job_id, afterClickUpload.path, {
+                                kind: "screenshot",
+                                mimeTypeOverride: afterClickUpload.mimeType,
+                                fileNameOverride: afterClickUpload.filename,
+                                metadata: {
+                                    purpose: "visual_click_result",
+                                    visible_in_chat: true,
+                                    target: targetDescription,
+                                    width: afterClickUpload.dimensions?.width || undefined,
+                                    height: afterClickUpload.dimensions?.height || undefined,
+                                    original_width: afterClickUpload.originalDimensions?.width || undefined,
+                                    original_height: afterClickUpload.originalDimensions?.height || undefined,
+                                    resized_for_upload: afterClickUpload.resized,
+                                },
+                            });
+                            if (afterClickArtifact?.storage_path) {
+                                artifacts.push(afterClickArtifact);
+                                const verificationAnswer = await this.analyzeUploadedArtifact(job.job_id, afterClickArtifact.storage_path, action.verification_prompt, afterClickArtifact.mime_type);
+                                if (!looksLikeAffirmativeVisualVerification(verificationAnswer)) {
+                                    lastFailureReason = verificationAnswer || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`;
+                                    continue;
+                                }
+                            }
+                        }
+                        completionNotes.push(`Localizei e cliquei em ${targetDescription}.`);
+                        clickSucceeded = true;
+                        break;
                     }
-                    artifacts.push(artifact);
-                    const artifactMetadata = artifact.metadata || {};
-                    const width = Number(artifactMetadata.width || 0);
-                    const height = Number(artifactMetadata.height || 0);
-                    const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, action.description, width, height, artifact.mime_type);
-                    if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
-                        throw new Error(`Nao consegui localizar ${action.description} com confianca suficiente na tela.`);
+                    if (!clickSucceeded) {
+                        throw new Error(lastFailureReason || `Nao consegui concluir o clique visual para ${action.description}.`);
                     }
-                    await reporter.progress(progressPercent, `Clicando em ${action.description}`);
-                    await this.clickPoint(location.x, location.y);
-                    completionNotes.push(`Localizei e cliquei em ${action.description}.`);
-                    resultPayload.last_click = location;
                     continue;
                 }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
                 await this.openUrl(action.url, action.app);
                 await delay(1200);
+                completionNotes.push(`${humanizeUrl(action.url)} foi aberto${action.app ? ` em ${action.app}` : ""}.`);
             }
             const summary = completionNotes.length > 0
                 ? completionNotes.join("\n\n")
@@ -713,8 +806,8 @@ end tell
             return null;
         }
         const bytes = await readFile(localPath);
-        const fileName = path.basename(localPath);
-        const mimeType = mimeTypeFromPath(fileName);
+        const fileName = options?.fileNameOverride || path.basename(localPath);
+        const mimeType = options?.mimeTypeOverride || mimeTypeFromPath(fileName);
         const dimensions = mimeType.startsWith("image/") ? await this.getImageDimensions(localPath) : null;
         const metadata = {
             ...(options?.metadata || {}),
@@ -729,6 +822,17 @@ end tell
         });
         return response.artifact || null;
     }
+    async analyzeUploadedArtifact(jobId, storagePath, question, mimeType) {
+        if (!this.bridgeConfig?.apiBaseUrl || !this.bridgeConfig?.deviceToken) {
+            return "";
+        }
+        const response = await postDeviceJson(this.bridgeConfig.apiBaseUrl, this.bridgeConfig.deviceToken, `/v1/devices/jobs/${encodeURIComponent(jobId)}/vision/analyze`, {
+            storage_path: storagePath,
+            question,
+            mime_type: mimeType || "image/jpeg",
+        });
+        return String(response.answer || "").trim();
+    }
     async readFrontmostPage(app) {
         const targetApp = app || "Safari";
         if (targetApp !== "Safari") {
@@ -743,13 +847,37 @@ tell application "Safari"
 end tell
 return pageJson
 `;
-        const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
-        const parsed = JSON.parse(stdout.trim() || "{}");
-        return {
-            title: asString(parsed.title) || "",
-            url: asString(parsed.url) || "",
-            text: asString(parsed.text) || "",
-        };
+        try {
+            const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
+            const parsed = JSON.parse(stdout.trim() || "{}");
+            return {
+                title: asString(parsed.title) || "",
+                url: asString(parsed.url) || "",
+                text: asString(parsed.text) || "",
+            };
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            if (!detail.toLowerCase().includes("allow javascript from apple events")) {
+                throw error;
+            }
+            const metadataScript = `
+tell application "Safari"
+  activate
+  if (count of windows) = 0 then error "Safari nao possui janelas abertas."
+  set pageTitle to name of current tab of front window
+  set pageUrl to URL of current tab of front window
+end tell
+return pageTitle & linefeed & pageUrl
+`;
+            const { stdout } = await this.runCommandCapture("osascript", ["-e", metadataScript]);
+            const [title, url] = stdout.split("\n");
+            return {
+                title: String(title || "").trim(),
+                url: String(url || "").trim(),
+                text: "",
+            };
+        }
     }
     async setVolume(level) {
         const bounded = Math.max(0, Math.min(Math.round(level), 100));
@@ -810,6 +938,59 @@ post(.leftMouseUp)
             return null;
         }
     }
+    async buildUploadableImage(localPath) {
+        const originalDimensions = await this.getImageDimensions(localPath);
+        const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
+        await mkdir(artifactsDir, { recursive: true });
+        let sourcePath = localPath;
+        let mimeType = mimeTypeFromPath(localPath);
+        let filename = path.basename(localPath);
+        let resized = false;
+        const conversionSteps = [
+            { width: 1280, quality: 42 },
+            { width: 1024, quality: 35 },
+            { width: 900, quality: 30 },
+            { width: 768, quality: 26 },
+            { width: 640, quality: 22 },
+            { width: 540, quality: 18 },
+            { width: 480, quality: 16 },
+            { width: 420, quality: 14 },
+            { width: 360, quality: 12 },
+            { width: 320, quality: 10 },
+        ];
+        for (const step of conversionSteps) {
+            const candidatePath = path.join(artifactsDir, `${path.basename(localPath, path.extname(localPath))}-${step.width}w-q${step.quality}.jpg`);
+            await this.runCommand("sips", [
+                "-s",
+                "format",
+                "jpeg",
+                "-s",
+                "formatOptions",
+                String(step.quality),
+                "--resampleWidth",
+                String(step.width),
+                localPath,
+                "--out",
+                candidatePath,
+            ]);
+            const candidateStat = await stat(candidatePath);
+            sourcePath = candidatePath;
+            mimeType = "image/jpeg";
+            filename = path.basename(candidatePath);
+            resized = true;
+            if (candidateStat.size <= 120_000) {
+                break;
+            }
+        }
+        return {
+            path: sourcePath,
+            mimeType,
+            filename,
+            dimensions: await this.getImageDimensions(sourcePath),
+            originalDimensions,
+            resized,
+        };
+    }
     async readLocalFile(filePath, maxChars = 4000) {
         const resolved = expandUserPath(filePath);
         const content = await readFile(resolved, "utf8");
@@ -834,7 +1015,7 @@ post(.leftMouseUp)
     }
     async runShellCommand(command, cwd) {
         if (!isSafeShellCommand(command)) {
-            throw new Error("Otto Bridge permite apenas shell de consulta no momento. Use comandos de leitura como pwd, ls, cat, rg, find ou git status.");
+            throw new Error("Nenhum comando shell foi informado para execucao local.");
         }
         const resolvedCwd = cwd ? expandUserPath(cwd) : process.cwd();
         const { stdout, stderr } = await this.runCommandCapture("/bin/zsh", ["-lc", command], {
@@ -936,10 +1117,6 @@ post(.leftMouseUp)
             if (stderrText) {
                 console.warn(`[otto-bridge] ${command} stderr=${stderrText}`);
             }
-            const stdoutText = stdout.trim();
-            if (stdoutText) {
-                console.log(`[otto-bridge] ${command} stdout=${stdoutText}`);
-            }
             return { stdout, stderr };
         }
         catch (error) {

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.0";
+export const BRIDGE_VERSION = "0.5.2";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.0",
+  "version": "0.5.2",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",