@leg3ndy/otto-bridge 0.5.2 → 0.5.3

package/dist/executors/native_macos.js

@@ -30,6 +30,61 @@ const KNOWN_SITES = [
     { label: "WhatsApp Web", url: "https://web.whatsapp.com", patterns: [/\bwhatsapp\b/i] },
     { label: "X", url: "https://x.com", patterns: [/\bx\.com\b/i, /\btwitter\b/i, /\bxis\b/i] },
 ];
+const GENERIC_VISUAL_STOP_WORDS = new Set([
+    "o",
+    "a",
+    "os",
+    "as",
+    "um",
+    "uma",
+    "uns",
+    "umas",
+    "de",
+    "da",
+    "do",
+    "das",
+    "dos",
+    "em",
+    "no",
+    "na",
+    "nos",
+    "nas",
+    "por",
+    "para",
+    "com",
+    "sem",
+    "que",
+    "visivel",
+    "visiveis",
+    "tela",
+    "pagina",
+    "site",
+    "app",
+    "janela",
+    "aba",
+    "botao",
+    "botoes",
+    "link",
+    "item",
+    "resultado",
+    "resultados",
+    "primeiro",
+    "primeira",
+    "segundo",
+    "segunda",
+    "terceiro",
+    "terceira",
+    "video",
+    "videos",
+    "musica",
+    "faixa",
+    "clicar",
+    "clique",
+    "seleciona",
+    "selecionar",
+    "abre",
+    "abrir",
+]);
 function asRecord(value) {
     return value && typeof value === "object" ? value : {};
 }
@@ -49,6 +104,80 @@ function normalizeText(value) {
 function escapeAppleScript(value) {
     return value.replace(/\\/g, "\\\\").replace(/"/g, '\\"');
 }
+function normalizeComparableUrl(raw) {
+    const input = String(raw || "").trim();
+    if (!input) {
+        return "";
+    }
+    try {
+        const parsed = new URL(input);
+        parsed.hash = "";
+        return parsed.toString();
+    }
+    catch {
+        return input;
+    }
+}
+function extractQuotedPhrases(value) {
+    return Array.from(String(value || "").matchAll(/["'“”‘’]([^"'“”‘’]{2,80})["'“”‘’]/g))
+        .map((match) => normalizeText(match[1] || "").trim())
+        .filter(Boolean);
+}
+function extractMeaningfulDescriptionTokens(value) {
+    return Array.from(new Set(normalizeText(value || "")
+        .split(/[^a-z0-9]+/)
+        .map((token) => token.trim())
+        .filter((token) => token.length >= 3 && !GENERIC_VISUAL_STOP_WORDS.has(token))));
+}
+function descriptionLikelyHasTextAnchor(description) {
+    return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
+}
+function findOcrTextMatch(candidates, description) {
+    const phrases = extractQuotedPhrases(description);
+    const tokens = extractMeaningfulDescriptionTokens(description);
+    const normalizedDescription = normalizeText(description || "");
+    const wantsFirst = /\b(primeir[ao]?|first)\b/.test(normalizedDescription);
+    if (!phrases.length && !tokens.length) {
+        return null;
+    }
+    const scored = candidates
+        .map((candidate, index) => {
+        const normalizedText = normalizeText(candidate.text || "");
+        let score = 0;
+        for (const phrase of phrases) {
+            if (normalizedText.includes(phrase)) {
+                score += 120;
+            }
+        }
+        for (const token of tokens) {
+            if (normalizedText.includes(token)) {
+                score += 18;
+            }
+        }
+        if (wantsFirst) {
+            score += Math.max(0, 24 - Math.round(candidate.y / 60));
+            score += Math.max(0, 12 - index);
+        }
+        if (candidate.confidence) {
+            score += Math.round(candidate.confidence * 20);
+        }
+        return score > 0 ? {
+            candidate,
+            score,
+        } : null;
+    })
+        .filter(Boolean);
+    scored.sort((left, right) => {
+        if (right.score !== left.score) {
+            return right.score - left.score;
+        }
+        if (left.candidate.y !== right.candidate.y) {
+            return left.candidate.y - right.candidate.y;
+        }
+        return left.candidate.x - right.candidate.x;
+    });
+    return scored[0] || null;
+}
 function extractTaskText(job) {
     const payload = asRecord(job.payload);
     const candidates = [
@@ -159,6 +288,38 @@ function clipText(value, maxLength) {
     }
     return `${value.slice(0, maxLength)}...`;
 }
+const TEXTUTIL_READABLE_EXTENSIONS = new Set([
+    ".doc",
+    ".docx",
+    ".odt",
+    ".pages",
+    ".rtf",
+    ".rtfd",
+    ".webarchive",
+]);
+function sanitizeTextForJsonTransport(value) {
+    return value
+        .replace(/\r\n/g, "\n")
+        .replace(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, "")
+        .trim();
+}
+function isLikelyBinaryBuffer(buffer) {
+    if (buffer.length === 0) {
+        return false;
+    }
+    let suspiciousBytes = 0;
+    const sampleSize = Math.min(buffer.length, 4096);
+    for (let index = 0; index < sampleSize; index += 1) {
+        const byte = buffer[index];
+        if (byte === 0) {
+            return true;
+        }
+        if (byte < 7 || (byte > 13 && byte < 32) || byte === 127) {
+            suspiciousBytes += 1;
+        }
+    }
+    return (suspiciousBytes / sampleSize) > 0.1;
+}
 function delay(ms) {
     return new Promise((resolve) => setTimeout(resolve, ms));
 }
@@ -451,6 +612,7 @@ export class NativeMacOSJobExecutor {
     bridgeConfig;
     cancelledJobs = new Set();
     activeChild = null;
+    lastActiveApp = null;
     constructor(bridgeConfig) {
         this.bridgeConfig = bridgeConfig;
     }
@@ -597,7 +759,12 @@ export class NativeMacOSJobExecutor {
                     continue;
                 }
                 if (action.type === "click_visual_target") {
-                    if (action.app) {
+                    const browserApp = await this.resolveLikelyBrowserApp(action.app);
+                    if (browserApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${browserApp} para frente antes do clique`);
+                        await this.focusApp(browserApp);
+                    }
+                    else if (action.app) {
                         await reporter.progress(progressPercent, `Trazendo ${action.app} para frente antes do clique`);
                         await this.focusApp(action.app);
                     }
@@ -606,8 +773,87 @@ export class NativeMacOSJobExecutor {
                     let lastFailureReason = "";
                     for (let attempt = 0; attempt < targetDescriptions.length; attempt += 1) {
                         const targetDescription = targetDescriptions[attempt];
+                        const initialBrowserState = browserApp
+                            ? await this.captureBrowserPageState(browserApp).catch(() => null)
+                            : null;
+                        if (browserApp === "Safari") {
+                            await reporter.progress(progressPercent, `Tentando localizar ${targetDescription} diretamente no Safari`);
+                            const domClick = await this.trySafariDomClick(targetDescription);
+                            if (domClick?.clicked) {
+                                let validated = false;
+                                let validationReason = "";
+                                if (action.verification_prompt) {
+                                    const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "dom_click_result");
+                                    validated = verification.ok;
+                                    validationReason = verification.reason;
+                                }
+                                else {
+                                    const browserValidation = await this.confirmBrowserClick(browserApp, initialBrowserState, targetDescription, domClick.matchedHref || null);
+                                    validated = browserValidation.ok;
+                                    validationReason = browserValidation.reason;
+                                }
+                                if (validated) {
+                                    resultPayload.last_click = {
+                                        strategy: domClick.strategy || "safari_dom",
+                                        matched_text: domClick.matchedText || null,
+                                        matched_href: domClick.matchedHref || null,
+                                        score: domClick.score || null,
+                                        total_candidates: domClick.totalCandidates || null,
+                                    };
+                                    completionNotes.push(`Localizei e cliquei em ${targetDescription} diretamente no navegador.`);
+                                    clickSucceeded = true;
+                                    break;
+                                }
+                                lastFailureReason = validationReason || `Clique DOM em ${targetDescription} nao alterou a pagina como esperado.`;
+                            }
+                            else if (domClick?.reason) {
+                                lastFailureReason = domClick.reason;
+                            }
+                        }
+                        const visualBeforeState = browserApp
+                            ? await this.captureBrowserPageState(browserApp).catch(() => initialBrowserState)
+                            : initialBrowserState;
                         await reporter.progress(progressPercent, `Capturando a tela para localizar ${targetDescription}`);
-                        const screenshotPath = await this.takeScreenshot();
+                        let screenshotPath = await this.takeScreenshot();
+                        const ocrClick = await this.tryLocalOcrClick(screenshotPath, targetDescription);
+                        if (ocrClick.clicked) {
+                            let validated = false;
+                            let validationReason = "";
+                            if (action.verification_prompt) {
+                                const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "local_ocr_click_result");
+                                validated = verification.ok;
+                                validationReason = verification.reason;
+                            }
+                            else if (browserApp) {
+                                const browserValidation = await this.confirmBrowserClick(browserApp, visualBeforeState, targetDescription, null);
+                                validated = browserValidation.ok;
+                                validationReason = browserValidation.reason;
+                            }
+                            else {
+                                validated = true;
+                            }
+                            if (validated) {
+                                const candidate = ocrClick.candidate || null;
+                                resultPayload.last_click = {
+                                    strategy: ocrClick.strategy || "local_ocr",
+                                    score: ocrClick.score || null,
+                                    matched_text: candidate?.text || null,
+                                    x: candidate ? candidate.x + (candidate.width / 2) : null,
+                                    y: candidate ? candidate.y + (candidate.height / 2) : null,
+                                    width: candidate?.width || null,
+                                    height: candidate?.height || null,
+                                };
+                                completionNotes.push(`Localizei e cliquei em ${targetDescription} por OCR local.`);
+                                clickSucceeded = true;
+                                break;
+                            }
+                            lastFailureReason = validationReason || `O clique por OCR local em ${targetDescription} nao teve efeito confirmavel.`;
+                            await reporter.progress(progressPercent, "OCR local nao confirmou o clique; vou tentar visão remota");
+                            screenshotPath = await this.takeScreenshot();
+                        }
+                        else if (ocrClick.reason) {
+                            lastFailureReason = ocrClick.reason;
+                        }
                         const uploadable = await this.buildUploadableImage(screenshotPath);
                         const artifact = await this.uploadArtifactForJob(job.job_id, uploadable.path, {
                             kind: "screenshot",
@@ -646,34 +892,20 @@ export class NativeMacOSJobExecutor {
                             ...location,
                             x: scaledX,
                             y: scaledY,
+                            strategy: "visual_locator",
                         };
                         if (action.verification_prompt) {
-                            await delay(1600);
-                            await reporter.progress(progressPercent, "Validando visualmente se a ação funcionou");
-                            const afterClickPath = await this.takeScreenshot();
-                            const afterClickUpload = await this.buildUploadableImage(afterClickPath);
-                            const afterClickArtifact = await this.uploadArtifactForJob(job.job_id, afterClickUpload.path, {
-                                kind: "screenshot",
-                                mimeTypeOverride: afterClickUpload.mimeType,
-                                fileNameOverride: afterClickUpload.filename,
-                                metadata: {
-                                    purpose: "visual_click_result",
-                                    visible_in_chat: true,
-                                    target: targetDescription,
-                                    width: afterClickUpload.dimensions?.width || undefined,
-                                    height: afterClickUpload.dimensions?.height || undefined,
-                                    original_width: afterClickUpload.originalDimensions?.width || undefined,
-                                    original_height: afterClickUpload.originalDimensions?.height || undefined,
-                                    resized_for_upload: afterClickUpload.resized,
-                                },
-                            });
-                            if (afterClickArtifact?.storage_path) {
-                                artifacts.push(afterClickArtifact);
-                                const verificationAnswer = await this.analyzeUploadedArtifact(job.job_id, afterClickArtifact.storage_path, action.verification_prompt, afterClickArtifact.mime_type);
-                                if (!looksLikeAffirmativeVisualVerification(verificationAnswer)) {
-                                    lastFailureReason = verificationAnswer || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`;
-                                    continue;
-                                }
+                            const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "visual_click_result");
+                            if (!verification.ok) {
+                                lastFailureReason = verification.reason || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`;
+                                continue;
+                            }
+                        }
+                        else if (browserApp) {
+                            const browserValidation = await this.confirmBrowserClick(browserApp, visualBeforeState, targetDescription, null);
+                            if (!browserValidation.ok) {
+                                lastFailureReason = browserValidation.reason || `O clique em ${targetDescription} nao alterou a pagina como esperado.`;
+                                continue;
                             }
                         }
                         completionNotes.push(`Localizei e cliquei em ${targetDescription}.`);
@@ -722,6 +954,7 @@ export class NativeMacOSJobExecutor {
         if (app) {
             await this.runCommand("open", ["-a", app, url]);
             await this.focusApp(app);
+            this.lastActiveApp = app;
             return;
         }
         await this.runCommand("open", [url]);
@@ -749,6 +982,118 @@ end tell
     }
     async focusApp(app) {
         await this.runCommand("osascript", ["-e", `tell application "${escapeAppleScript(app)}" to activate`]);
+        this.lastActiveApp = app;
+    }
+    async getFrontmostAppName() {
+        try {
+            const { stdout } = await this.runCommandCapture("osascript", [
+                "-e",
+                'tell application "System Events" to get name of first application process whose frontmost is true',
+            ]);
+            const app = String(stdout || "").trim();
+            return app || null;
+        }
+        catch {
+            return null;
+        }
+    }
+    async resolveLikelyBrowserApp(preferredApp) {
+        const candidates = [
+            preferredApp || null,
+            this.lastActiveApp,
+            await this.getFrontmostAppName(),
+        ];
+        for (const candidate of candidates) {
+            if (candidate === "Safari") {
+                return candidate;
+            }
+        }
+        return null;
+    }
+    async captureBrowserPageState(app) {
+        if (app !== "Safari") {
+            return null;
+        }
+        const page = await this.readFrontmostPage(app);
+        return {
+            app,
+            title: page.title,
+            url: page.url,
+            text: page.text,
+        };
+    }
+    resolveExpectedBrowserHref(rawHref, baseUrl) {
+        const href = String(rawHref || "").trim();
+        if (!href) {
+            return null;
+        }
+        try {
+            const absolute = baseUrl ? new URL(href, baseUrl).toString() : new URL(href).toString();
+            return normalizeComparableUrl(absolute);
+        }
+        catch {
+            return normalizeComparableUrl(href);
+        }
+    }
+    didBrowserPageStateChange(before, after, targetDescription, matchedHref) {
+        if (!after) {
+            return false;
+        }
+        const beforeUrl = normalizeComparableUrl(before?.url || "");
+        const afterUrl = normalizeComparableUrl(after.url || "");
+        const expectedHref = this.resolveExpectedBrowserHref(matchedHref || null, before?.url || after.url);
+        if (expectedHref && afterUrl) {
+            if (afterUrl === expectedHref || afterUrl.startsWith(expectedHref) || expectedHref.startsWith(afterUrl)) {
+                return true;
+            }
+        }
+        if (beforeUrl && afterUrl && beforeUrl !== afterUrl) {
+            return true;
+        }
+        const normalizedDescription = normalizeText(targetDescription || "");
+        if (normalizedDescription.includes("youtube")
+            || normalizedDescription.includes("video")
+            || normalizedDescription.includes("musica")) {
+            if (afterUrl.includes("youtube.com/watch") || afterUrl.includes("youtube.com/shorts/")) {
+                return true;
+            }
+        }
+        const beforeTitle = normalizeText(before?.title || "");
+        const afterTitle = normalizeText(after.title || "");
+        if (beforeTitle && afterTitle && beforeTitle !== afterTitle) {
+            return true;
+        }
+        const beforeText = normalizeText((before?.text || "").slice(0, 320));
+        const afterText = normalizeText((after.text || "").slice(0, 320));
+        if (beforeText && afterText && beforeText !== afterText) {
+            return true;
+        }
+        return false;
+    }
+    async confirmBrowserClick(app, before, targetDescription, matchedHref) {
+        if (app !== "Safari") {
+            return {
+                ok: true,
+                reason: "",
+                afterState: null,
+            };
+        }
+        for (let attempt = 0; attempt < 4; attempt += 1) {
+            await delay(attempt === 0 ? 900 : 700);
+            const afterState = await this.captureBrowserPageState(app).catch(() => null);
+            if (this.didBrowserPageStateChange(before, afterState, targetDescription, matchedHref)) {
+                return {
+                    ok: true,
+                    reason: "",
+                    afterState,
+                };
+            }
+        }
+        return {
+            ok: false,
+            reason: `O clique em ${targetDescription} nao mudou a pagina do navegador de forma verificavel.`,
+            afterState: null,
+        };
     }
     async pressShortcut(shortcut) {
         const { key, modifiers } = parseShortcut(shortcut);
@@ -833,6 +1178,263 @@ end tell
         });
         return String(response.answer || "").trim();
     }
+    async validateVisualClickWithVision(jobId, targetDescription, verificationPrompt, progressPercent, reporter, artifacts, purpose) {
+        await delay(1600);
+        await reporter.progress(progressPercent, "Validando visualmente se a ação funcionou");
+        const afterClickPath = await this.takeScreenshot();
+        const afterClickUpload = await this.buildUploadableImage(afterClickPath);
+        const afterClickArtifact = await this.uploadArtifactForJob(jobId, afterClickUpload.path, {
+            kind: "screenshot",
+            mimeTypeOverride: afterClickUpload.mimeType,
+            fileNameOverride: afterClickUpload.filename,
+            metadata: {
+                purpose,
+                visible_in_chat: true,
+                target: targetDescription,
+                width: afterClickUpload.dimensions?.width || undefined,
+                height: afterClickUpload.dimensions?.height || undefined,
+                original_width: afterClickUpload.originalDimensions?.width || undefined,
+                original_height: afterClickUpload.originalDimensions?.height || undefined,
+                resized_for_upload: afterClickUpload.resized,
+            },
+        });
+        if (!afterClickArtifact?.storage_path) {
+            return {
+                ok: false,
+                reason: `Nao consegui registrar a tela apos tentar clicar em ${targetDescription}.`,
+            };
+        }
+        artifacts.push(afterClickArtifact);
+        const verificationAnswer = await this.analyzeUploadedArtifact(jobId, afterClickArtifact.storage_path, verificationPrompt, afterClickArtifact.mime_type);
+        if (!looksLikeAffirmativeVisualVerification(verificationAnswer)) {
+            return {
+                ok: false,
+                reason: verificationAnswer || `Nao consegui validar visualmente se ${targetDescription} foi acionado.`,
+            };
+        }
+        return {
+            ok: true,
+            reason: verificationAnswer,
+        };
+    }
+    async runSafariJsonScript(scriptBody, input) {
+        const wrappedScript = `
+(function(){
+  const __input = ${JSON.stringify(input || null)};
+  try {
+    const __result = (() => {
+${scriptBody}
+    })();
+    return JSON.stringify({ ok: true, result: __result === undefined ? null : __result });
+  } catch (error) {
+    return JSON.stringify({
+      ok: false,
+      error: String(error && error.message ? error.message : error)
+    });
+  }
+})()
+`;
+        const script = `
+tell application "Safari"
+  activate
+  if (count of windows) = 0 then error "Safari nao possui janelas abertas."
+  delay 0.2
+  set scriptResult to do JavaScript "${escapeAppleScript(wrappedScript)}" in current tab of front window
+end tell
+return scriptResult
+`;
+        const { stdout } = await this.runCommandCapture("osascript", ["-e", script]);
+        const parsed = JSON.parse(stdout.trim() || "{}");
+        if (parsed.ok !== true) {
+            throw new Error(asString(parsed.error) || "Safari JavaScript execution failed");
+        }
+        return parsed.result;
+    }
+    async trySafariDomClick(description) {
+        try {
+            return await this.runSafariJsonScript(`
+const rawDescription = String(__input?.description || "");
+const normalize = (value) => String(value || "")
+  .normalize("NFD")
+  .replace(/[\\u0300-\\u036f]/g, "")
+  .toLowerCase();
+const normalizedDescription = normalize(rawDescription);
+const wantsFirst = /\\b(primeir[ao]?|first)\\b/.test(normalizedDescription);
+const wantsVideo = /\\b(video|videos|musica|faixa|youtube|resultado|watch)\\b/.test(normalizedDescription) || location.hostname.includes("youtube");
+const stopWords = new Set([
+  "o", "a", "os", "as", "um", "uma", "uns", "umas", "de", "da", "do", "das", "dos",
+  "em", "no", "na", "nos", "nas", "para", "por", "com", "que", "visivel", "visiveis",
+  "visivel", "tela", "pagina", "page", "site", "link", "botao", "botao", "clicar",
+  "clique", "seleciona", "selecionar", "resultado", "resultados"
+]);
+const quotedPhrases = Array.from(rawDescription.matchAll(/["'“”‘’]([^"'“”‘’]{2,80})["'“”‘’]/g))
+  .map((match) => normalize(match[1]));
+const tokens = Array.from(new Set(
+  normalizedDescription
+    .split(/[^a-z0-9]+/)
+    .filter((token) => token.length >= 3 && !stopWords.has(token))
+));
+
+const candidateSelectors = location.hostname.includes("youtube")
+  ? [
+      "ytd-video-renderer a#video-title",
+      "ytd-video-renderer ytd-thumbnail a",
+      "ytd-video-renderer a#thumbnail",
+      "ytd-rich-item-renderer a#video-title-link",
+      "ytd-rich-item-renderer a#video-title",
+      "ytd-rich-grid-media a#video-title-link",
+      "a#video-title",
+      "a[href*='/watch']",
+      "button",
+      "[role='button']",
+      "[role='link']"
+    ]
+  : [
+      "a[href]",
+      "button",
+      "[role='button']",
+      "[role='link']",
+      "input[type='button']",
+      "input[type='submit']"
+    ];
+
+const seen = new Set();
+const candidates = [];
+
+function isVisible(element) {
+  if (!(element instanceof Element)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+function deriveText(element) {
+  const ownText = [
+    element.innerText,
+    element.textContent,
+    element.getAttribute("aria-label"),
+    element.getAttribute("title"),
+    element.getAttribute("alt"),
+  ].find((value) => typeof value === "string" && value.trim());
+  if (ownText && ownText.trim()) return ownText.trim();
+
+  const richVideo = element.closest("ytd-video-renderer, ytd-rich-item-renderer, ytd-rich-grid-media, ytmusic-responsive-list-item-renderer");
+  if (richVideo && richVideo instanceof HTMLElement && richVideo.innerText.trim()) {
+    return richVideo.innerText.trim();
+  }
+
+  return "";
+}
+
+function scoreCandidate(element, rank) {
+  const text = deriveText(element);
+  const href = element instanceof HTMLAnchorElement
+    ? (element.href || "")
+    : (element.getAttribute("href") || "");
+  const normalizedText = normalize(text);
+  const normalizedHref = normalize(href);
+  let score = 0;
+
+  if (wantsFirst) score += Math.max(0, 40 - rank);
+  if (wantsVideo && normalizedHref.includes("/watch")) score += 30;
+  if (location.hostname.includes("youtube") && element.closest("ytd-video-renderer, ytd-rich-item-renderer, ytd-rich-grid-media")) score += 20;
+  if (element.id === "video-title") score += 12;
+  if (!normalizedText && normalizedHref.includes("/watch")) score += 8;
+
+  for (const phrase of quotedPhrases) {
+    if (!phrase) continue;
+    if (normalizedText.includes(phrase)) score += 120;
+    if (normalizedHref.includes(phrase)) score += 40;
+  }
+
+  for (const token of tokens) {
+    if (normalizedText.includes(token)) score += 18;
+    if (normalizedHref.includes(token)) score += 8;
+  }
+
+  return {
+    element,
+    text,
+    href,
+    score,
+    rank,
+  };
+}
+
+for (const selector of candidateSelectors) {
+  const nodes = document.querySelectorAll(selector);
+  for (const node of nodes) {
+    if (!(node instanceof HTMLElement || node instanceof HTMLAnchorElement)) continue;
+    if (!isVisible(node)) continue;
+    const key = [
+      node.tagName,
+      node.id || "",
+      node.getAttribute("href") || "",
+      deriveText(node).slice(0, 120),
+    ].join("|");
+    if (seen.has(key)) continue;
+    seen.add(key);
+    candidates.push(scoreCandidate(node, candidates.length));
+  }
+}
+
+const ranked = candidates
+  .filter((candidate) => candidate.score > 0 || (wantsFirst && normalize(candidate.href).includes("/watch")))
+  .sort((left, right) => right.score - left.score || left.rank - right.rank);
+
+if (!ranked.length) {
+  return {
+    clicked: false,
+    reason: "Nenhum elemento clicavel no DOM combinou com a descricao atual.",
+    totalCandidates: candidates.length,
+    strategy: "safari_dom",
+  };
+}
+
+const winner = ranked[0];
+winner.element.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+const rect = winner.element.getBoundingClientRect();
+for (const eventName of ["mouseover", "mousedown", "mouseup", "click"]) {
+  winner.element.dispatchEvent(new MouseEvent(eventName, {
+    bubbles: true,
+    cancelable: true,
+    view: window,
+    clientX: rect.left + (rect.width / 2),
+    clientY: rect.top + (rect.height / 2),
+  }));
+}
+if (typeof winner.element.click === "function") {
+  winner.element.click();
+}
+
+return {
+  clicked: true,
+  matchedText: String(winner.text || "").slice(0, 180),
+  matchedHref: winner.href || "",
+  score: winner.score,
+  totalCandidates: candidates.length,
+  strategy: "safari_dom",
+};
+        `, { description });
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            if (detail.toLowerCase().includes("allow javascript from apple events")) {
+                return {
+                    clicked: false,
+                    reason: "Safari ainda bloqueia JavaScript por Apple Events, entao o Otto Bridge caiu para o modo visual.",
+                    strategy: "safari_dom_blocked",
+                };
+            }
+            return {
+                clicked: false,
+                reason: detail || "Falha ao tentar clicar via DOM no Safari.",
+                strategy: "safari_dom_failed",
+            };
+        }
+    }
     async readFrontmostPage(app) {
         const targetApp = app || "Safari";
         if (targetApp !== "Safari") {
@@ -921,6 +1523,128 @@ post(.leftMouseUp)
 `;
         await this.runCommand("swift", ["-e", script, String(Math.round(x)), String(Math.round(y))]);
     }
+    async runLocalOcr(filePath) {
+        const script = `
+import Foundation
+import Vision
+import ImageIO
+import CoreGraphics
+
+let fileURL = URL(fileURLWithPath: CommandLine.arguments[1])
+guard let source = CGImageSourceCreateWithURL(fileURL as CFURL, nil),
+      let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
+  fputs("failed to load image\\n", stderr)
+  exit(1)
+}
+
+let width = CGFloat(image.width)
+let height = CGFloat(image.height)
+var items: [[String: Any]] = []
+
+let request = VNRecognizeTextRequest()
+request.recognitionLevel = .accurate
+request.usesLanguageCorrection = true
+request.recognitionLanguages = ["pt-BR", "en-US"]
+request.minimumTextHeight = 0.012
+
+let handler = VNImageRequestHandler(cgImage: image, options: [:])
+try handler.perform([request])
+
+let observations = request.results ?? []
+for observation in observations {
+  guard let candidate = observation.topCandidates(1).first else { continue }
+  let text = candidate.string.trimmingCharacters(in: .whitespacesAndNewlines)
+  if text.isEmpty { continue }
+
+  let box = observation.boundingBox
+  let x = box.origin.x * width
+  let y = (1.0 - box.origin.y - box.size.height) * height
+  let w = box.size.width * width
+  let h = box.size.height * height
+
+  items.append([
+    "text": text,
+    "x": Int(round(x)),
+    "y": Int(round(y)),
+    "width": Int(round(w)),
+    "height": Int(round(h)),
+    "confidence": candidate.confidence
+  ])
+}
+
+let payload: [String: Any] = ["items": items]
+let data = try JSONSerialization.data(withJSONObject: payload, options: [])
+if let output = String(data: data, encoding: .utf8) {
+  print(output)
+}
+`;
+        try {
+            const { stdout } = await this.runCommandCapture("swift", ["-e", script, filePath]);
+            const parsed = JSON.parse(stdout.trim() || "{}");
+            const items = Array.isArray(parsed.items) ? parsed.items : [];
+            return items
+                .map((item) => {
+                const row = asRecord(item);
+                const text = asString(row.text);
+                const x = Number(row.x);
+                const y = Number(row.y);
+                const width = Number(row.width);
+                const height = Number(row.height);
+                const confidence = Number(row.confidence);
+                if (!text || !Number.isFinite(x) || !Number.isFinite(y) || !Number.isFinite(width) || !Number.isFinite(height)) {
+                    return null;
+                }
+                return {
+                    text,
+                    x,
+                    y,
+                    width,
+                    height,
+                    confidence: Number.isFinite(confidence) ? confidence : undefined,
+                };
+            })
+                .filter(Boolean);
+        }
+        catch (error) {
+            const detail = error instanceof Error ? error.message : String(error);
+            console.warn(`[otto-bridge] local ocr failed=${detail}`);
+            return [];
+        }
+    }
+    async tryLocalOcrClick(screenshotPath, description) {
+        if (!descriptionLikelyHasTextAnchor(description)) {
+            return {
+                clicked: false,
+                reason: "A descricao nao traz ancora textual forte para OCR local.",
+                strategy: "local_ocr_skipped",
+            };
+        }
+        const candidates = await this.runLocalOcr(screenshotPath);
+        if (!candidates.length) {
+            return {
+                clicked: false,
+                reason: "OCR local nao encontrou texto utilizavel na tela.",
+                strategy: "local_ocr_empty",
+            };
+        }
+        const match = findOcrTextMatch(candidates, description);
+        if (!match || match.score < 24) {
+            return {
+                clicked: false,
+                reason: "OCR local nao encontrou texto suficientemente compativel com a descricao.",
+                strategy: "local_ocr_no_match",
+            };
+        }
+        const clickX = match.candidate.x + (match.candidate.width / 2);
+        const clickY = match.candidate.y + (match.candidate.height / 2);
+        await this.clickPoint(clickX, clickY);
+        return {
+            clicked: true,
+            score: match.score,
+            candidate: match.candidate,
+            strategy: "local_ocr",
+        };
+    }
     async getImageDimensions(filePath) {
         try {
             const { stdout } = await this.runCommandCapture("sips", ["-g", "pixelWidth", "-g", "pixelHeight", filePath]);
@@ -993,8 +1717,25 @@ post(.leftMouseUp)
     }
     async readLocalFile(filePath, maxChars = 4000) {
         const resolved = expandUserPath(filePath);
-        const content = await readFile(resolved, "utf8");
-        return clipText(content.trim() || "(arquivo vazio)", maxChars);
+        const extension = path.extname(resolved).toLowerCase();
+        if (TEXTUTIL_READABLE_EXTENSIONS.has(extension)) {
+            const { stdout } = await this.runCommandCapture("textutil", [
+                "-convert",
+                "txt",
+                "-stdout",
+                resolved,
+            ]);
+            const content = sanitizeTextForJsonTransport(stdout);
+            return clipText(content || "(arquivo sem texto legivel)", maxChars);
+        }
+        const raw = await readFile(resolved);
+        if (isLikelyBinaryBuffer(raw)) {
+            const filename = path.basename(resolved);
+            const detectedType = extension || "binario";
+            return clipText(`O arquivo ${filename} parece ser binario (${detectedType}) e nao pode ser lido como texto puro pelo Otto Bridge ainda.`, maxChars);
+        }
+        const content = sanitizeTextForJsonTransport(raw.toString("utf8"));
+        return clipText(content || "(arquivo vazio)", maxChars);
     }
     async listLocalFiles(directoryPath, limit = 40) {
         const resolved = expandUserPath(directoryPath);

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.2";
+export const BRIDGE_VERSION = "0.5.3";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.2",
+  "version": "0.5.3",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",