@leg3ndy/otto-bridge 0.5.1 → 0.5.3

package/README.md

@@ -11,6 +11,8 @@ Companion local do Otto para:
 
 Para um passo a passo de instalacao, pareamento, uso, desconexao e desinstalacao, veja [USER_GUIDE.md](https://github.com/LGCYYL/ottoai/blob/main/otto-bridge/USER_GUIDE.md).
 
+Para o estado atual da arquitetura, capacidades entregues, limitacoes e roadmap do Otto Bridge, veja [`leg3ndy-ai-backend/docs/OTTO_BRIDGE_ARCHITECTURE.md`](../leg3ndy-ai-backend/docs/OTTO_BRIDGE_ARCHITECTURE.md).
+
 ## Distribuicao
 
 Fluxo recomendado agora:

package/dist/executors/native_macos.js

@@ -30,6 +30,61 @@ const KNOWN_SITES = [
     { label: "WhatsApp Web", url: "https://web.whatsapp.com", patterns: [/\bwhatsapp\b/i] },
     { label: "X", url: "https://x.com", patterns: [/\bx\.com\b/i, /\btwitter\b/i, /\bxis\b/i] },
 ];
+const GENERIC_VISUAL_STOP_WORDS = new Set([
+    "o",
+    "a",
+    "os",
+    "as",
+    "um",
+    "uma",
+    "uns",
+    "umas",
+    "de",
+    "da",
+    "do",
+    "das",
+    "dos",
+    "em",
+    "no",
+    "na",
+    "nos",
+    "nas",
+    "por",
+    "para",
+    "com",
+    "sem",
+    "que",
+    "visivel",
+    "visiveis",
+    "tela",
+    "pagina",
+    "site",
+    "app",
+    "janela",
+    "aba",
+    "botao",
+    "botoes",
+    "link",
+    "item",
+    "resultado",
+    "resultados",
+    "primeiro",
+    "primeira",
+    "segundo",
+    "segunda",
+    "terceiro",
+    "terceira",
+    "video",
+    "videos",
+    "musica",
+    "faixa",
+    "clicar",
+    "clique",
+    "seleciona",
+    "selecionar",
+    "abre",
+    "abrir",
+]);
 function asRecord(value) {
     return value && typeof value === "object" ? value : {};
 }
@@ -49,6 +104,80 @@ function normalizeText(value) {
 function escapeAppleScript(value) {
     return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
 }
+function normalizeComparableUrl(raw) {
+    const input = String(raw || "").trim();
+    if (!input) {
+        return "";
+    }
+    try {
+        const parsed = new URL(input);
+        parsed.hash = "";
+        return parsed.toString();
+    }
+    catch {
+        return input;
+    }
+}
+function extractQuotedPhrases(value) {
+    return Array.from(String(value || "").matchAll(/["'“”‘’]([^"'“”‘’]{2,80})["'“”‘’]/g))
+        .map((match) => normalizeText(match[1] || "").trim())
+        .filter(Boolean);
+}
+function extractMeaningfulDescriptionTokens(value) {
+    return Array.from(new Set(normalizeText(value || "")
+        .split(/[^a-z0-9]+/)
+        .map((token) => token.trim())
+        .filter((token) => token.length >= 3 && !GENERIC_VISUAL_STOP_WORDS.has(token))));
+}
+function descriptionLikelyHasTextAnchor(description) {
+    return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
+}
+function findOcrTextMatch(candidates, description) {
+    const phrases = extractQuotedPhrases(description);
+    const tokens = extractMeaningfulDescriptionTokens(description);
+    const normalizedDescription = normalizeText(description || "");
+    const wantsFirst = /\b(primeir[ao]?|first)\b/.test(normalizedDescription);
+    if (!phrases.length && !tokens.length) {
+        return null;
+    }
+    const scored = candidates
+        .map((candidate, index) => {
+        const normalizedText = normalizeText(candidate.text || "");
+        let score = 0;
+        for (const phrase of phrases) {
+            if (normalizedText.includes(phrase)) {
+                score += 120;
+            }
+        }
+        for (const token of tokens) {
+            if (normalizedText.includes(token)) {
+                score += 18;
+            }
+        }
+        if (wantsFirst) {
+            score += Math.max(0, 24 - Math.round(candidate.y / 60));
+            score += Math.max(0, 12 - index);
+        }
+        if (candidate.confidence) {
+            score += Math.round(candidate.confidence * 20);
+        }
+        return score > 0 ? {
+            candidate,
+            score,
+        } : null;
+    })
+        .filter(Boolean);
+    scored.sort((left, right) => {
+        if (right.score !== left.score) {
+            return right.score - left.score;
+        }
+        if (left.candidate.y !== right.candidate.y) {
+            return left.candidate.y - right.candidate.y;
+        }
+        return left.candidate.x - right.candidate.x;
+    });
+    return scored[0] || null;
+}
 function extractTaskText(job) {
     const payload = asRecord(job.payload);
     const candidates = [
@@ -88,6 +217,39 @@ function humanizeUrl(url) {
         return normalized;
     }
 }
+function uniqueStrings(values) {
+    const seen = new Set();
+    const result = [];
+    for (const value of values) {
+        const text = String(value || "").trim();
+        if (!text)
+            continue;
+        const key = normalizeText(text);
+        if (seen.has(key))
+            continue;
+        seen.add(key);
+        result.push(text);
+    }
+    return result;
+}
+function looksLikeAffirmativeVisualVerification(answer) {
+    const normalized = normalizeText(answer || "");
+    if (!normalized)
+        return false;
+    if (normalized.startsWith("sim"))
+        return true;
+    if (normalized.startsWith("nao") || normalized.startsWith("não"))
+        return false;
+    return (normalized.includes("tocando")
+        || normalized.includes("reproduzindo")
+        || normalized.includes("em reproducao")
+        || normalized.includes("em reprodução")
+        || normalized.includes("botao de pausa")
+        || normalized.includes("botão de pausa")
+        || normalized.includes("faixa ativa")
+        || normalized.includes("resultado selecionado")
+        || normalized.includes("foi acionado"));
+}
 function mimeTypeFromPath(filePath) {
     const ext = path.extname(filePath).toLowerCase();
     if (ext === ".png")
@@ -126,6 +288,38 @@ function clipText(value, maxLength) {
     }
     return `${value.slice(0, maxLength)}...`;
 }
+const TEXTUTIL_READABLE_EXTENSIONS = new Set([
+    ".doc",
+    ".docx",
+    ".odt",
+    ".pages",
+    ".rtf",
+    ".rtfd",
+    ".webarchive",
+]);
+function sanitizeTextForJsonTransport(value) {
+    return value
+        .replace(/\r\n/g, "\n")
+        .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "")
+        .trim();
+}
+function isLikelyBinaryBuffer(buffer) {
+    if (buffer.length === 0) {
+        return false;
+    }
+    let suspiciousBytes = 0;
+    const sampleSize = Math.min(buffer.length, 4096);
+    for (let index = 0; index < sampleSize; index += 1) {
+        const byte = buffer[index];
+        if (byte === 0) {
+            return true;
+        }
+        if (byte < 7 || (byte > 13 && byte < 32) || byte === 127) {
+            suspiciousBytes += 1;
+        }
+    }
+    return (suspiciousBytes / sampleSize) > 0.1;
+}
 function delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -327,6 +521,10 @@ function parseStructuredActions(job) {
                     type: "click_visual_target",
                     description,
                     app: asString(action.app) || undefined,
+                    verification_prompt: asString(action.verification_prompt) || undefined,
+                    retry_descriptions: Array.isArray(action.retry_descriptions)
+                        ? action.retry_descriptions.map((item) => asString(item)).filter(Boolean)
+                        : undefined,
                 });
             }
             continue;
@@ -414,6 +612,7 @@ export class NativeMacOSJobExecutor {
     bridgeConfig;
     cancelledJobs = new Set();
     activeChild = null;
+    lastActiveApp = null;
     constructor(bridgeConfig) {
         this.bridgeConfig = bridgeConfig;
     }
@@ -486,6 +685,7 @@ export class NativeMacOSJobExecutor {
                         mimeTypeOverride: uploadable.mimeType,
                         fileNameOverride: uploadable.filename,
                         metadata: {
+                            visible_in_chat: true,
                             width: uploadable.dimensions?.width || undefined,
                             height: uploadable.dimensions?.height || undefined,
                             original_width: uploadable.originalDimensions?.width || undefined,
@@ -516,6 +716,7 @@ export class NativeMacOSJobExecutor {
                             fileNameOverride: uploadable.filename,
                             metadata: {
                                 purpose: "page_read_fallback",
+                                visible_in_chat: false,
                                 width: uploadable.dimensions?.width || undefined,
                                 height: uploadable.dimensions?.height || undefined,
                                 original_width: uploadable.originalDimensions?.width || undefined,
@@ -558,50 +759,162 @@ export class NativeMacOSJobExecutor {
                     continue;
                 }
                 if (action.type === "click_visual_target") {
-                    if (action.app) {
+                    const browserApp = await this.resolveLikelyBrowserApp(action.app);
+                    if (browserApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${browserApp} para frente antes do clique`);
+                        await this.focusApp(browserApp);
+                    }
+                    else if (action.app) {
                         await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do clique`);
                         await this.focusApp(action.app);
                     }
-                    await reporter.progress(progressPercent, `Capturando a tela para localizar ${action.description}`);
-                    const screenshotPath = await this.takeScreenshot();
-                    const uploadable = await this.buildUploadableImage(screenshotPath);
-                    const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
-                        kind: "screenshot",
-                        mimeTypeOverride: uploadable.mimeType,
-                        fileNameOverride: uploadable.filename,
-                        metadata: {
-                            purpose: "visual_click",
-                            target: action.description,
-                            width: uploadable.dimensions?.width || undefined,
-                            height: uploadable.dimensions?.height || undefined,
-                            original_width: uploadable.originalDimensions?.width || undefined,
-                            original_height: uploadable.originalDimensions?.height || undefined,
-                            resized_for_upload: uploadable.resized,
-                        },
-                    });
-                    if (!artifact?.storage_path) {
-                        throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
+                    const targetDescriptions = uniqueStrings([action.description, ...(action.retry_descriptions || [])]);
+                    let clickSucceeded = false;
+                    let lastFailureReason = "";
+                    for (let attempt = 0; attempt < targetDescriptions.length; attempt += 1) {
+                        const targetDescription = targetDescriptions[attempt];
+                        const initialBrowserState = browserApp
+                            ? await this.captureBrowserPageState(browserApp).catch(() => null)
+                            : null;
+                        if (browserApp === "Safari") {
+                            await reporter.progress(progressPercent, `Tentando localizar ${targetDescription} diretamente no Safari`);
+                            const domClick = await this.trySafariDomClick(targetDescription);
+                            if (domClick?.clicked) {
+                                let validated = false;
+                                let validationReason = "";
+                                if (action.verification_prompt) {
+                                    const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "dom_click_result");
+                                    validated = verification.ok;
+                                    validationReason = verification.reason;
+                                }
+                                else {
+                                    const browserValidation = await this.confirmBrowserClick(browserApp, initialBrowserState, targetDescription, domClick.matchedHref || null);
+                                    validated = browserValidation.ok;
+                                    validationReason = browserValidation.reason;
+                                }
+                                if (validated) {
+                                    resultPayload.last_click = {
+                                        strategy: domClick.strategy || "safari_dom",
+                                        matched_text: domClick.matchedText || null,
+                                        matched_href: domClick.matchedHref || null,
+                                        score: domClick.score || null,
+                                        total_candidates: domClick.totalCandidates || null,
+                                    };
+                                    completionNotes.push(`Localizei e cliquei em ${targetDescription} diretamente no navegador.`);
+                                    clickSucceeded = true;
+                                    break;
+                                }
+                                lastFailureReason = validationReason || `Clique DOM em ${targetDescription} nao alterou a pagina como esperado.`;
+                            }
+                            else if (domClick?.reason) {
+                                lastFailureReason = domClick.reason;
+                            }
+                        }
+                        const visualBeforeState = browserApp
+                            ? await this.captureBrowserPageState(browserApp).catch(() => initialBrowserState)
+                            : initialBrowserState;
+                        await reporter.progress(progressPercent, `Capturando a tela para localizar ${targetDescription}`);
+                        let screenshotPath = await this.takeScreenshot();
+                        const ocrClick = await this.tryLocalOcrClick(screenshotPath, targetDescription);
+                        if (ocrClick.clicked) {
+                            let validated = false;
+                            let validationReason = "";
+                            if (action.verification_prompt) {
+                                const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "local_ocr_click_result");
+                                validated = verification.ok;
+                                validationReason = verification.reason;
+                            }
+                            else if (browserApp) {
+                                const browserValidation = await this.confirmBrowserClick(browserApp, visualBeforeState, targetDescription, null);
+                                validated = browserValidation.ok;
+                                validationReason = browserValidation.reason;
+                            }
+                            else {
+                                validated = true;
+                            }
+                            if (validated) {
+                                const candidate = ocrClick.candidate || null;
+                                resultPayload.last_click = {
+                                    strategy: ocrClick.strategy || "local_ocr",
+                                    score: ocrClick.score || null,
+                                    matched_text: candidate?.text || null,
+                                    x: candidate ? candidate.x + (candidate.width / 2) : null,
+                                    y: candidate ? candidate.y + (candidate.height / 2) : null,
+                                    width: candidate?.width || null,
+                                    height: candidate?.height || null,
+                                };
+                                completionNotes.push(`Localizei e cliquei em ${targetDescription} por OCR local.`);
+                                clickSucceeded = true;
+                                break;
+                            }
+                            lastFailureReason = validationReason || `O clique por OCR local em ${targetDescription} nao teve efeito confirmavel.`;
+                            await reporter.progress(progressPercent, "OCR local nao confirmou o clique; vou tentar visão remota");
+                            screenshotPath = await this.takeScreenshot();
+                        }
+                        else if (ocrClick.reason) {
+                            lastFailureReason = ocrClick.reason;
+                        }
+                        const uploadable = await this.buildUploadableImage(screenshotPath);
+                        const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
+                            kind: "screenshot",
+                            mimeTypeOverride: uploadable.mimeType,
+                            fileNameOverride: uploadable.filename,
+                            metadata: {
+                                purpose: "visual_click",
+                                visible_in_chat: false,
+                                target: targetDescription,
+                                width: uploadable.dimensions?.width || undefined,
+                                height: uploadable.dimensions?.height || undefined,
+                                original_width: uploadable.originalDimensions?.width || undefined,
+                                original_height: uploadable.originalDimensions?.height || undefined,
+                                resized_for_upload: uploadable.resized,
+                            },
+                        });
+                        if (!artifact?.storage_path) {
+                            throw new Error("Otto Bridge nao conseguiu enviar a screenshot necessaria para localizar o alvo visual.");
+                        }
+                        artifacts.push(artifact);
+                        const artifactMetadata = artifact.metadata || {};
+                        const width = Number(artifactMetadata.width || 0);
+                        const height = Number(artifactMetadata.height || 0);
+                        const originalWidth = Number(artifactMetadata.original_width || width || 0);
+                        const originalHeight = Number(artifactMetadata.original_height || height || 0);
+                        const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, targetDescription, width, height, artifact.mime_type);
+                        if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
+                            lastFailureReason = `Nao consegui localizar ${targetDescription} com confianca suficiente na tela.`;
+                            continue;
+                        }
+                        await reporter.progress(progressPercent, `Clicando em ${targetDescription}`);
+                        const scaledX = width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x;
+                        const scaledY = height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y;
+                        await this.clickPoint(scaledX, scaledY);
+                        resultPayload.last_click = {
+                            ...location,
+                            x: scaledX,
+                            y: scaledY,
+                            strategy: "visual_locator",
+                        };
+                        if (action.verification_prompt) {
+                            const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "visual_click_result");
+                            if (!verification.ok) {
+                                lastFailureReason = verification.reason || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`;
+                                continue;
+                            }
+                        }
+                        else if (browserApp) {
+                            const browserValidation = await this.confirmBrowserClick(browserApp, visualBeforeState, targetDescription, null);
+                            if (!browserValidation.ok) {
+                                lastFailureReason = browserValidation.reason || `O clique em ${targetDescription} nao alterou a pagina como esperado.`;
+                                continue;
+                            }
+                        }
+                        completionNotes.push(`Localizei e cliquei em ${targetDescription}.`);
+                        clickSucceeded = true;
+                        break;
                     }
-                    artifacts.push(artifact);
-                    const artifactMetadata = artifact.metadata || {};
-                    const width = Number(artifactMetadata.width || 0);
-                    const height = Number(artifactMetadata.height || 0);
-                    const originalWidth = Number(artifactMetadata.original_width || width || 0);
-                    const originalHeight = Number(artifactMetadata.original_height || height || 0);
-                    const location = await this.locateVisualTarget(job.job_id, artifact.storage_path, action.description, width, height, artifact.mime_type);
-                    if (!location?.found || typeof location.x !== "number" || typeof location.y !== "number") {
-                        throw new Error(`Nao consegui localizar ${action.description} com confianca suficiente na tela.`);
+                    if (!clickSucceeded) {
+                        throw new Error(lastFailureReason || `Nao consegui concluir o clique visual para ${action.description}.`);
                     }
-                    await reporter.progress(progressPercent, `Clicando em ${action.description}`);
-                    const scaledX = width > 0 && originalWidth > 0 ? (location.x / width) * originalWidth : location.x;
-                    const scaledY = height > 0 && originalHeight > 0 ? (location.y / height) * originalHeight : location.y;
-                    await this.clickPoint(scaledX, scaledY);
-                    completionNotes.push(`Localizei e cliquei em ${action.description}.`);
-                    resultPayload.last_click = {
-                        ...location,
-                        x: scaledX,
-                        y: scaledY,
-                    };
                     continue;
                 }
                 await reporter.progress(progressPercent, `Abrindo ${action.url}${action.app ? ` em ${action.app}` : ""}`);
@@ -641,6 +954,7 @@ export class NativeMacOSJobExecutor {
         if (app) {
             await this.runCommand("open", ["-a", app, url]);
             await this.focusApp(app);
+            this.lastActiveApp = app;
             return;
         }
         await this.runCommand("open", [url]);
@@ -668,6 +982,118 @@ end tell
     }
     async focusApp(app) {
         await this.runCommand("osascript", ["-e", `tell application "${escapeAppleScript(app)}" to activate`]);
+        this.lastActiveApp = app;
+    }
+    async getFrontmostAppName() {
+        try {
+            const { stdout } = await this.runCommandCapture("osascript", [
+                "-e",
+                'tell application "System Events" to get name of first application process whose frontmost is true',
+            ]);
+            const app = String(stdout || "").trim();
+            return app || null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async resolveLikelyBrowserApp(preferredApp) {
+        const candidates = [
+            preferredApp || null,
+            this.lastActiveApp,
+            await this.getFrontmostAppName(),
+        ];
+        for (const candidate of candidates) {
+            if (candidate === "Safari") {
+                return candidate;
+            }
+        }
+        return null;
+    }
+    async captureBrowserPageState(app) {
+        if (app !== "Safari") {
+            return null;
+        }
+        const page = await this.readFrontmostPage(app);
+        return {
+            app,
+            title: page.title,
+            url: page.url,
+            text: page.text,
+        };
+    }
+    resolveExpectedBrowserHref(rawHref, baseUrl) {
+        const href = String(rawHref || "").trim();
+        if (!href) {
+            return null;
+        }
+        try {
+            const absolute = baseUrl ? new URL(href, baseUrl).toString() : new URL(href).toString();
+            return normalizeComparableUrl(absolute);
+        }
+        catch {
+            return normalizeComparableUrl(href);
+        }
+    }
+    didBrowserPageStateChange(before, after, targetDescription, matchedHref) {
+        if (!after) {
+            return false;
+        }
+        const beforeUrl = normalizeComparableUrl(before?.url || "");
+        const afterUrl = normalizeComparableUrl(after.url || "");
+        const expectedHref = this.resolveExpectedBrowserHref(matchedHref || null, before?.url || after.url);
+        if (expectedHref && afterUrl) {
+            if (afterUrl === expectedHref || afterUrl.startsWith(expectedHref) || expectedHref.startsWith(afterUrl)) {
+                return true;
+            }
+        }
+        if (beforeUrl && afterUrl && beforeUrl !== afterUrl) {
+            return true;
+        }
+        const normalizedDescription = normalizeText(targetDescription || "");
+        if (normalizedDescription.includes("youtube")
+            || normalizedDescription.includes("video")
+            || normalizedDescription.includes("musica")) {
+            if (afterUrl.includes("youtube.com/watch") || afterUrl.includes("youtube.com/shorts/")) {
+                return true;
+            }
+        }
+        const beforeTitle = normalizeText(before?.title || "");
+        const afterTitle = normalizeText(after.title || "");
+        if (beforeTitle && afterTitle && beforeTitle !== afterTitle) {
+            return true;
+        }
+        const beforeText = normalizeText((before?.text || "").slice(0, 320));
+        const afterText = normalizeText((after.text || "").slice(0, 320));
+        if (beforeText && afterText && beforeText !== afterText) {
+            return true;
+        }
+        return false;
+    }
+    async confirmBrowserClick(app, before, targetDescription, matchedHref) {
+        if (app !== "Safari") {
+            return {
+                ok: true,
+                reason: "",
+                afterState: null,
+            };
+        }
+        for (let attempt = 0; attempt < 4; attempt += 1) {
+            await delay(attempt === 0 ? 900 : 700);
+            const afterState = await this.captureBrowserPageState(app).catch(() => null);
+            if (this.didBrowserPageStateChange(before, afterState, targetDescription, matchedHref)) {
+                return {
+                    ok: true,
+                    reason: "",
+                    afterState,
+                };
+            }
+        }
+        return {
+            ok: false,
+            reason: `O clique em ${targetDescription} nao mudou a pagina do navegador de forma verificavel.`,
+            afterState: null,
+        };
     }
     async pressShortcut(shortcut) {
         const { key, modifiers } = parseShortcut(shortcut);
@@ -752,6 +1178,263 @@ end tell
         });
         return String(response.answer || "").trim();
     }
+    async validateVisualClickWithVision(jobId, targetDescription, verificationPrompt, progressPercent, reporter, artifacts, purpose) {
+        await delay(1600);
+        await reporter.progress(progressPercent, "Validando visualmente se a ação funcionou");
+        const afterClickPath = await this.takeScreenshot();
+        const afterClickUpload = await this.buildUploadableImage(afterClickPath);
+        const afterClickArtifact = await this.uploadArtifactForJob(jobId, afterClickUpload.path, {
+            kind: "screenshot",
+            mimeTypeOverride: afterClickUpload.mimeType,
+            fileNameOverride: afterClickUpload.filename,
+            metadata: {
+                purpose,
+                visible_in_chat: true,
+                target: targetDescription,
+                width: afterClickUpload.dimensions?.width || undefined,
+                height: afterClickUpload.dimensions?.height || undefined,
+                original_width: afterClickUpload.originalDimensions?.width || undefined,
+                original_height: afterClickUpload.originalDimensions?.height || undefined,
+                resized_for_upload: afterClickUpload.resized,
+            },
+        });
+        if (!afterClickArtifact?.storage_path) {
+            return {
+                ok: false,
+                reason: `Nao consegui registrar a tela apos tentar clicar em ${targetDescription}.`,
+            };
+        }
+        artifacts.push(afterClickArtifact);
+        const verificationAnswer = await this.analyzeUploadedArtifact(jobId, afterClickArtifact.storage_path, verificationPrompt, afterClickArtifact.mime_type);
+        if (!looksLikeAffirmativeVisualVerification(verificationAnswer)) {
+            return {
+                ok: false,
+                reason: verificationAnswer || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`,
+            };
+        }
+        return {
+            ok: true,
+            reason: verificationAnswer,
+        };
+    }
+    async runSafariJsonScript(scriptBody, input) {
+        const wrappedScript = `
+(function(){
+  const __input = ${JSON.stringify(input || null)};
+  try {
+    const __result = (() => {
+${scriptBody}
+    })();
+    return JSON.stringify({ ok: true, result: __result === undefined ? null : __result });
+  } catch (error) {
+    return JSON.stringify({
+      ok: false,
+      error: String(error && error.message ? error.message : error)
+    });
+  }
+})()
+`;
+        const script = `
+tell application "Safari"
+  activate
+  if (count of windows) = 0 then error "Safari nao possui janelas abertas."
+  delay 0.2
+  set scriptResult to do JavaScript "${escapeAppleScript(wrappedScript)}" in current tab of front window
+end tell
+return scriptResult
+`;
+        const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
+        const parsed = JSON.parse(stdout.trim() || "{}");
+        if (parsed.ok !== true) {
+            throw new Error(asString(parsed.error) || "Safari JavaScript execution failed");
+        }
+        return parsed.result;
+    }
+    async trySafariDomClick(description) {
+        try {
+            return await this.runSafariJsonScript(`
+const rawDescription = String(__input?.description || "");
+const normalize = (value) => String(value || "")
+  .normalize("NFD")
+  .replace(/[\\u0300-\\u036f]/g, "")
+  .toLowerCase();
+const normalizedDescription = normalize(rawDescription);
+const wantsFirst = /\\b(primeir[ao]?|first)\\b/.test(normalizedDescription);
+const wantsVideo = /\\b(video|videos|musica|faixa|youtube|resultado|watch)\\b/.test(normalizedDescription) || location.hostname.includes("youtube");
+const stopWords = new Set([
+  "o", "a", "os", "as", "um", "uma", "uns", "umas", "de", "da", "do", "das", "dos",
+  "em", "no", "na", "nos", "nas", "para", "por", "com", "que", "visivel", "visiveis",
+  "visivel", "tela", "pagina", "page", "site", "link", "botao", "botao", "clicar",
+  "clique", "seleciona", "selecionar", "resultado", "resultados"
+]);
+const quotedPhrases = Array.from(rawDescription.matchAll(/["'“”‘’]([^"'“”‘’]{2,80})["'“”‘’]/g))
+  .map((match) => normalize(match[1]));
+const tokens = Array.from(new Set(
+  normalizedDescription
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length >= 3 && !stopWords.has(token))
+));
+
+const candidateSelectors = location.hostname.includes("youtube")
+  ? [
+      "ytd-video-renderer a#video-title",
+      "ytd-video-renderer ytd-thumbnail a",
+      "ytd-video-renderer a#thumbnail",
+      "ytd-rich-item-renderer a#video-title-link",
+      "ytd-rich-item-renderer a#video-title",
+      "ytd-rich-grid-media a#video-title-link",
+      "a#video-title",
+      "a[href*='/watch']",
+      "button",
+      "[role='button']",
+      "[role='link']"
+    ]
+  : [
+      "a[href]",
+      "button",
+      "[role='button']",
+      "[role='link']",
+      "input[type='button']",
+      "input[type='submit']"
+    ];
+
+const seen = new Set();
+const candidates = [];
+
+function isVisible(element) {
+  if (!(element instanceof Element)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+function deriveText(element) {
+  const ownText = [
+    element.innerText,
+    element.textContent,
+    element.getAttribute("aria-label"),
+    element.getAttribute("title"),
+    element.getAttribute("alt"),
+  ].find((value) => typeof value === "string" && value.trim());
+  if (ownText && ownText.trim()) return ownText.trim();
+
+  const richVideo = element.closest("ytd-video-renderer, ytd-rich-item-renderer, ytd-rich-grid-media, ytmusic-responsive-list-item-renderer");
+  if (richVideo && richVideo instanceof HTMLElement && richVideo.innerText.trim()) {
+    return richVideo.innerText.trim();
+  }
+
+  return "";
+}
+
+function scoreCandidate(element, rank) {
+  const text = deriveText(element);
+  const href = element instanceof HTMLAnchorElement
+    ? (element.href || "")
+    : (element.getAttribute("href") || "");
+  const normalizedText = normalize(text);
+  const normalizedHref = normalize(href);
+  let score = 0;
+
+  if (wantsFirst) score += Math.max(0, 40 - rank);
+  if (wantsVideo && normalizedHref.includes("/watch")) score += 30;
+  if (location.hostname.includes("youtube") && element.closest("ytd-video-renderer, ytd-rich-item-renderer, ytd-rich-grid-media")) score += 20;
+  if (element.id === "video-title") score += 12;
+  if (!normalizedText && normalizedHref.includes("/watch")) score += 8;
+
+  for (const phrase of quotedPhrases) {
+    if (!phrase) continue;
+    if (normalizedText.includes(phrase)) score += 120;
+    if (normalizedHref.includes(phrase)) score += 40;
+  }
+
+  for (const token of tokens) {
+    if (normalizedText.includes(token)) score += 18;
+    if (normalizedHref.includes(token)) score += 8;
+  }
+
+  return {
+    element,
+    text,
+    href,
+    score,
+    rank,
+  };
+}
+
+for (const selector of candidateSelectors) {
+  const nodes = document.querySelectorAll(selector);
+  for (const node of nodes) {
+    if (!(node instanceof HTMLElement || node instanceof HTMLAnchorElement)) continue;
+    if (!isVisible(node)) continue;
+    const key = [
+      node.tagName,
+      node.id || "",
+      node.getAttribute("href") || "",
+      deriveText(node).slice(0, 120),
+    ].join("|");
+    if (seen.has(key)) continue;
+    seen.add(key);
+    candidates.push(scoreCandidate(node, candidates.length));
+  }
+}
+
+const ranked = candidates
+  .filter((candidate) => candidate.score > 0 || (wantsFirst && normalize(candidate.href).includes("/watch")))
+  .sort((left, right) => right.score - left.score || left.rank - right.rank);
+
+if (!ranked.length) {
+  return {
+    clicked: false,
+    reason: "Nenhum elemento clicavel no DOM combinou com a descricao atual.",
+    totalCandidates: candidates.length,
+    strategy: "safari_dom",
+  };
+}
+
+const winner = ranked[0];
+winner.element.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+const rect = winner.element.getBoundingClientRect();
+for (const eventName of ["mouseover", "mousedown", "mouseup", "click"]) {
+  winner.element.dispatchEvent(new MouseEvent(eventName, {
+    bubbles: true,
+    cancelable: true,
+    view: window,
+    clientX: rect.left + (rect.width / 2),
+    clientY: rect.top + (rect.height / 2),
+  }));
+}
+if (typeof winner.element.click === "function") {
+  winner.element.click();
+}
+
+return {
+  clicked: true,
+  matchedText: String(winner.text || "").slice(0, 180),
+  matchedHref: winner.href || "",
+  score: winner.score,
+  totalCandidates: candidates.length,
+  strategy: "safari_dom",
+};
+        `, { description });
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            if (detail.toLowerCase().includes("allow javascript from apple events")) {
+                return {
+                    clicked: false,
+                    reason: "Safari ainda bloqueia JavaScript por Apple Events, entao o Otto Bridge caiu para o modo visual.",
+                    strategy: "safari_dom_blocked",
+                };
+            }
+            return {
+                clicked: false,
+                reason: detail || "Falha ao tentar clicar via DOM no Safari.",
+                strategy: "safari_dom_failed",
+            };
+        }
+    }
     async readFrontmostPage(app) {
         const targetApp = app || "Safari";
         if (targetApp !== "Safari") {
@@ -840,6 +1523,128 @@ post(.leftMouseUp)
 `;
         await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
     }
+    async runLocalOcr(filePath) {
+        const script = `
+import Foundation
+import Vision
+import ImageIO
+import CoreGraphics
+
+let fileURL = URL(fileURLWithPath: CommandLine.arguments[1])
+guard let source = CGImageSourceCreateWithURL(fileURL as CFURL, nil),
+      let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
+  fputs("failed to load image\\n", stderr)
+  exit(1)
+}
+
+let width = CGFloat(image.width)
+let height = CGFloat(image.height)
+var items: [[String: Any]] = []
+
+let request = VNRecognizeTextRequest()
+request.recognitionLevel = .accurate
+request.usesLanguageCorrection = true
+request.recognitionLanguages = ["pt-BR", "en-US"]
+request.minimumTextHeight = 0.012
+
+let handler = VNImageRequestHandler(cgImage: image, options: [:])
+try handler.perform([request])
+
+let observations = request.results ?? []
+for observation in observations {
+  guard let candidate = observation.topCandidates(1).first else { continue }
+  let text = candidate.string.trimmingCharacters(in: .whitespacesAndNewlines)
+  if text.isEmpty { continue }
+
+  let box = observation.boundingBox
+  let x = box.origin.x * width
+  let y = (1.0 - box.origin.y - box.size.height) * height
+  let w = box.size.width * width
+  let h = box.size.height * height
+
+  items.append([
+    "text": text,
+    "x": Int(round(x)),
+    "y": Int(round(y)),
+    "width": Int(round(w)),
+    "height": Int(round(h)),
+    "confidence": candidate.confidence
+  ])
+}
+
+let payload: [String: Any] = ["items": items]
+let data = try JSONSerialization.data(withJSONObject: payload, options: [])
+if let output = String(data: data, encoding: .utf8) {
+  print(output)
+}
+`;
+        try {
+            const { stdout } = await this.runCommandCapture("swift", ["-e", script, filePath]);
+            const parsed = JSON.parse(stdout.trim() || "{}");
+            const items = Array.isArray(parsed.items) ? parsed.items : [];
+            return items
+                .map((item) => {
+                const row = asRecord(item);
+                const text = asString(row.text);
+                const x = Number(row.x);
+                const y = Number(row.y);
+                const width = Number(row.width);
+                const height = Number(row.height);
+                const confidence = Number(row.confidence);
+                if (!text || !Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(width) || !Number.isFinite(height)) {
+                    return null;
+                }
+                return {
+                    text,
+                    x,
+                    y,
+                    width,
+                    height,
+                    confidence: Number.isFinite(confidence) ? confidence : undefined,
+                };
+            })
+                .filter(Boolean);
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            console.warn(`[otto-bridge] local ocr failed=${detail}`);
+            return [];
+        }
+    }
+    async tryLocalOcrClick(screenshotPath, description) {
+        if (!descriptionLikelyHasTextAnchor(description)) {
+            return {
+                clicked: false,
+                reason: "A descricao nao traz ancora textual forte para OCR local.",
+                strategy: "local_ocr_skipped",
+            };
+        }
+        const candidates = await this.runLocalOcr(screenshotPath);
+        if (!candidates.length) {
+            return {
+                clicked: false,
+                reason: "OCR local nao encontrou texto utilizavel na tela.",
+                strategy: "local_ocr_empty",
+            };
+        }
+        const match = findOcrTextMatch(candidates, description);
+        if (!match || match.score < 24) {
+            return {
+                clicked: false,
+                reason: "OCR local nao encontrou texto suficientemente compativel com a descricao.",
+                strategy: "local_ocr_no_match",
+            };
+        }
+        const clickX = match.candidate.x + (match.candidate.width / 2);
+        const clickY = match.candidate.y + (match.candidate.height / 2);
+        await this.clickPoint(clickX, clickY);
+        return {
+            clicked: true,
+            score: match.score,
+            candidate: match.candidate,
+            strategy: "local_ocr",
+        };
+    }
     async getImageDimensions(filePath) {
         try {
             const { stdout } = await this.runCommandCapture("sips", ["-g", "pixelWidth", "-g", "pixelHeight", filePath]);
@@ -873,6 +1678,9 @@ post(.leftMouseUp)
             { width: 640, quality: 22 },
             { width: 540, quality: 18 },
             { width: 480, quality: 16 },
+            { width: 420, quality: 14 },
+            { width: 360, quality: 12 },
+            { width: 320, quality: 10 },
         ];
         for (const step of conversionSteps) {
             const candidatePath = path.join(artifactsDir, `${path.basename(localPath, path.extname(localPath))}-${step.width}w-q${step.quality}.jpg`);
@@ -894,7 +1702,7 @@ post(.leftMouseUp)
             mimeType = "image/jpeg";
             filename = path.basename(candidatePath);
             resized = true;
-            if (candidateStat.size <= 220_000) {
+            if (candidateStat.size <= 120_000) {
                 break;
             }
         }
@@ -909,8 +1717,25 @@ post(.leftMouseUp)
     }
     async readLocalFile(filePath, maxChars = 4000) {
         const resolved = expandUserPath(filePath);
-        const content = await readFile(resolved, "utf8");
-        return clipText(content.trim() || "(arquivo vazio)", maxChars);
+        const extension = path.extname(resolved).toLowerCase();
+        if (TEXTUTIL_READABLE_EXTENSIONS.has(extension)) {
+            const { stdout } = await this.runCommandCapture("textutil", [
+                "-convert",
+                "txt",
+                "-stdout",
+                resolved,
+            ]);
+            const content = sanitizeTextForJsonTransport(stdout);
+            return clipText(content || "(arquivo sem texto legivel)", maxChars);
+        }
+        const raw = await readFile(resolved);
+        if (isLikelyBinaryBuffer(raw)) {
+            const filename = path.basename(resolved);
+            const detectedType = extension || "binario";
+            return clipText(`O arquivo ${filename} parece ser binario (${detectedType}) e nao pode ser lido como texto puro pelo Otto Bridge ainda.`, maxChars);
+        }
+        const content = sanitizeTextForJsonTransport(raw.toString("utf8"));
+        return clipText(content || "(arquivo vazio)", maxChars);
     }
     async listLocalFiles(directoryPath, limit = 40) {
         const resolved = expandUserPath(directoryPath);

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.1";
+export const BRIDGE_VERSION = "0.5.3";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.1",
+  "version": "0.5.3",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",