@leg3ndy/otto-bridge 0.5.6 → 0.5.9

package/dist/executors/native_macos.js

@@ -129,6 +129,102 @@ function extractMeaningfulDescriptionTokens(value) {
         .map((token) => token.trim())
         .filter((token) => token.length >= 3 && !GENERIC_VISUAL_STOP_WORDS.has(token))));
 }
+const MEDIA_QUERY_STOP_WORDS = new Set([
+    "youtube",
+    "music",
+    "spotify",
+    "apple",
+    "player",
+    "play",
+    "pause",
+    "pausa",
+    "pausar",
+    "tocar",
+    "toque",
+    "reproduzir",
+    "reproducao",
+    "retomar",
+    "retoma",
+    "continuar",
+    "continue",
+    "resumir",
+    "resume",
+    "next",
+    "skip",
+    "proxima",
+    "proximo",
+    "anterior",
+    "previous",
+    "voltar",
+    "volta",
+    "melhor",
+    "tocavel",
+    "relacionado",
+    "diretamente",
+    "navegador",
+    "fila",
+    "playerbar",
+]);
+function extractMediaQueryTokens(value) {
+    return extractMeaningfulDescriptionTokens(value).filter((token) => !MEDIA_QUERY_STOP_WORDS.has(token));
+}
+function countMatchingTokens(text, tokens) {
+    const normalizedText = normalizeText(text || "");
+    if (!normalizedText || !tokens.length) {
+        return 0;
+    }
+    return tokens.reduce((total, token) => total + (normalizedText.includes(token) ? 1 : 0), 0);
+}
+function typedTextLooksApplied(actual, expected) {
+    const normalizedActual = normalizeText(actual || "").replace(/\s+/g, " ").trim();
+    const normalizedExpected = normalizeText(expected || "").replace(/\s+/g, " ").trim();
+    if (!normalizedActual || !normalizedExpected) {
+        return false;
+    }
+    if (normalizedActual === normalizedExpected
+        || normalizedActual.includes(normalizedExpected)
+        || normalizedExpected.includes(normalizedActual)) {
+        return true;
+    }
+    const expectedTokens = normalizedExpected.split(/[^a-z0-9]+/).filter((token) => token.length >= 3);
+    if (!expectedTokens.length) {
+        return false;
+    }
+    const matches = expectedTokens.filter((token) => normalizedActual.includes(token)).length;
+    return matches >= Math.max(2, Math.ceil(expectedTokens.length * 0.6));
+}
+function descriptionWantsNext(description) {
+    return /\b(proxim[ao]?|next|skip|pular|avanca|avanç[ae])\b/.test(normalizeText(description || ""));
+}
+function descriptionWantsPrevious(description) {
+    return /\b(anterior|previous|volta[ar]?|back|retorna[ar]?)\b/.test(normalizeText(description || ""));
+}
+function descriptionWantsPause(description) {
+    return /\b(pausa|pause|pausar)\b/.test(normalizeText(description || ""));
+}
+function descriptionWantsResume(description) {
+    return /\b(retoma|retomar|resume|continu[ae]r|despausa|play)\b/.test(normalizeText(description || ""));
+}
+function extractNativeMediaTransportCommand(description) {
+    const normalizedDescription = normalizeText(description || "");
+    if (!normalizedDescription
+        || /\b(resultado|tocavel|relacionado|faixa|search|busca|pesquisa)\b/.test(normalizedDescription)) {
+        return null;
+    }
+    if (!/\b(player|controle|fila|music|spotify|youtube|deezer|apple)\b/.test(normalizedDescription)) {
+        return null;
+    }
+    if (descriptionWantsNext(description)) {
+        return "next";
+    }
+    if (descriptionWantsPrevious(description)) {
+        return "previous";
+    }
+    if (descriptionWantsPause(description) || descriptionWantsResume(description)) {
+        return "play_pause";
+    }
+    return null;
+}
 function descriptionLikelyHasTextAnchor(description) {
     return extractQuotedPhrases(description).length > 0 || extractMeaningfulDescriptionTokens(description).length > 0;
 }
@@ -665,6 +761,44 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "scroll_view" || type === "scroll" || type === "scroll_page") {
+            const rawDirection = (asString(action.direction) || asString(action.dir) || "down").toLowerCase();
+            const direction = rawDirection === "up" || rawDirection === "cima" ? "up" : "down";
+            const rawAmount = (asString(action.amount) || "").toLowerCase();
+            const amount = rawAmount === "small" || rawAmount === "medium" || rawAmount === "large"
+                ? rawAmount
+                : undefined;
+            const rawSteps = Number(action.steps);
+            const steps = Number.isFinite(rawSteps) ? Math.max(1, Math.min(Math.round(rawSteps), 6)) : undefined;
+            actions.push({
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: asString(action.app) || undefined,
+            });
+            continue;
+        }
+        if (type === "whatsapp_send_message") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const text = asString(action.text) || asString(action.message);
+            if (contact && text) {
+                actions.push({ type: "whatsapp_send_message", contact, text });
+            }
+            continue;
+        }
+        if (type === "whatsapp_read_chat") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const rawLimit = Number(action.limit);
+            if (contact) {
+                actions.push({
+                    type: "whatsapp_read_chat",
+                    contact,
+                    limit: Number.isFinite(rawLimit) ? Math.max(1, Math.min(Math.round(rawLimit), 30)) : undefined,
+                });
+            }
+            continue;
+        }
         if (type === "click_visual_target" || type === "click_target") {
             const description = asString(action.description) || asString(action.target);
             if (description) {
@@ -744,6 +878,23 @@ function deriveActionsFromText(job) {
         }
         return [{ type: "set_volume", level }];
     }
+    if (/\b(scroll|rola\w*|role\w*|desce\w*|sobe\w*)\b/i.test(task)) {
+        const direction = /\b(sobe\w*|suba\w*|para cima|pra cima|scroll up)\b/i.test(task) ? "up" : "down";
+        const amount = /\b(pouco|leve|small)\b/i.test(task)
+            ? "small"
+            : /\b(muito|bastante|fim|grande|large)\b/i.test(task)
+                ? "large"
+                : "medium";
+        const stepsMatch = task.match(/\b(\d{1,2})\s*(?:x|vezes?)\b/i);
+        const steps = stepsMatch?.[1] ? Math.max(1, Math.min(Number(stepsMatch[1]), 6)) : 1;
+        return [{
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: detectedApp || undefined,
+            }];
+    }
     if ((normalizedTask.includes("leia") || normalizedTask.includes("ler")) && detectedUrl) {
         return [
             { type: "open_url", url: detectedUrl, app: detectedApp || "Safari" },
@@ -777,6 +928,8 @@ export class NativeMacOSJobExecutor {
     cancelledJobs = new Set();
     activeChild = null;
     lastActiveApp = null;
+    lastVisualTargetDescription = null;
+    lastVisualTargetApp = null;
     constructor(bridgeConfig) {
         this.bridgeConfig = bridgeConfig;
     }
@@ -836,8 +989,24 @@ export class NativeMacOSJobExecutor {
                     continue;
                 }
                 if (action.type === "type_text") {
-                    await reporter.progress(progressPercent, "Digitando texto no app ativo");
-                    await this.typeText(action.text);
+                    const typingApp = this.lastActiveApp || await this.getFrontmostAppName();
+                    await reporter.progress(progressPercent, `Digitando texto em ${typingApp || "app ativo"}`);
+                    const typed = await this.guidedTypeText(action.text, typingApp || undefined);
+                    if (!typed.ok) {
+                        throw new Error(typed.reason || "Nao consegui digitar o texto no app ativo.");
+                    }
+                    resultPayload.last_typed = {
+                        strategy: typed.strategy,
+                        verified: typed.verified,
+                        app: typed.app,
+                        attempts: typed.attempts,
+                        text_preview: clipText(action.text, 180),
+                    };
+                    this.lastVisualTargetDescription = null;
+                    this.lastVisualTargetApp = null;
+                    completionNotes.push(typed.verified
+                        ? `Digitei e confirmei o texto no ${typed.app || "app ativo"}.`
+                        : `Digitei o texto no ${typed.app || "app ativo"}.`);
                     continue;
                 }
                 if (action.type === "take_screenshot") {
@@ -934,6 +1103,68 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Volume ajustado para ${action.level}% no macOS.`);
                     continue;
                 }
+                if (action.type === "scroll_view") {
+                    const scrollApp = action.app || this.lastActiveApp || await this.getFrontmostAppName();
+                    if (scrollApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${scrollApp} para frente antes de rolar a tela`);
+                        await this.focusApp(scrollApp);
+                    }
+                    const directionLabel = action.direction === "up" ? "cima" : "baixo";
+                    await reporter.progress(progressPercent, `Rolando a tela para ${directionLabel}`);
+                    await this.scrollView(action.direction, action.amount, action.steps);
+                    resultPayload.last_scroll = {
+                        direction: action.direction,
+                        amount: action.amount || "medium",
+                        steps: action.steps || 1,
+                        app: scrollApp || null,
+                    };
+                    completionNotes.push(`Rolei a tela para ${directionLabel} no macOS.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_send_message") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await reporter.progress(progressPercent, `Digitando a mensagem para ${action.contact} no WhatsApp`);
+                    await this.focusWhatsAppComposer();
+                    await this.typeText(action.text);
+                    await delay(250);
+                    await this.pressShortcut("return");
+                    await delay(900);
+                    const verification = await this.verifyWhatsAppLastMessage(action.text);
+                    if (!verification.ok) {
+                        throw new Error(verification.reason || `Nao consegui confirmar o envio da mensagem para ${action.contact} no WhatsApp.`);
+                    }
+                    resultPayload.whatsapp = {
+                        action: "send_message",
+                        contact: action.contact,
+                        text_preview: clipText(action.text, 180),
+                    };
+                    completionNotes.push(`Enviei a mensagem no WhatsApp para ${action.contact}.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_read_chat") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await delay(500);
+                    const chat = await this.readWhatsAppVisibleConversation(action.contact, action.limit || 12);
+                    resultPayload.whatsapp = {
+                        action: "read_chat",
+                        contact: action.contact,
+                        messages: chat.messages,
+                    };
+                    completionNotes.push(`Mensagens visiveis no WhatsApp com ${action.contact}:\n${chat.summary}`);
+                    continue;
+                }
                 if (action.type === "click_visual_target") {
                     const browserApp = await this.resolveLikelyBrowserApp(action.app);
                     if (browserApp) {
@@ -952,6 +1183,42 @@ export class NativeMacOSJobExecutor {
                         const initialBrowserState = browserApp
                             ? await this.captureBrowserPageState(browserApp).catch(() => null)
                             : null;
+                        const nativeMediaTransport = extractNativeMediaTransportCommand(targetDescription);
+                        if (nativeMediaTransport) {
+                            await reporter.progress(progressPercent, `Tentando controle de mídia nativo do macOS para ${targetDescription}`);
+                            try {
+                                await this.triggerMacOSMediaTransport(nativeMediaTransport);
+                                let validated = false;
+                                let validationReason = "";
+                                if (action.verification_prompt) {
+                                    const verification = await this.validateVisualClickWithVision(job.job_id, targetDescription, action.verification_prompt, progressPercent, reporter, artifacts, "native_media_transport_result");
+                                    validated = verification.ok;
+                                    validationReason = verification.reason;
+                                }
+                                else if (browserApp) {
+                                    const browserValidation = await this.confirmBrowserClick(browserApp, initialBrowserState, targetDescription, null);
+                                    validated = browserValidation.ok;
+                                    validationReason = browserValidation.reason;
+                                }
+                                else {
+                                    validated = true;
+                                }
+                                if (validated) {
+                                    resultPayload.last_click = {
+                                        strategy: "native_media_transport",
+                                        matched_text: targetDescription,
+                                    };
+                                    completionNotes.push(`Acionei ${targetDescription} usando o controle de mídia nativo do macOS.`);
+                                    clickSucceeded = true;
+                                    break;
+                                }
+                                lastFailureReason = validationReason || `O controle de mídia nativo do macOS nao confirmou ${targetDescription}.`;
+                                await reporter.progress(progressPercent, "O controle de mídia nativo nao foi suficiente; vou tentar DOM/OCR");
+                            }
+                            catch (error) {
+                                lastFailureReason = error instanceof Error ? error.message : String(error);
+                            }
+                        }
                         if (browserApp === "Safari") {
                             await reporter.progress(progressPercent, `Tentando localizar ${targetDescription} diretamente no Safari`);
                             const domClick = await this.trySafariDomClick(targetDescription);
@@ -969,6 +1236,8 @@ export class NativeMacOSJobExecutor {
                                     validationReason = browserValidation.reason;
                                 }
                                 if (validated) {
+                                    this.lastVisualTargetDescription = targetDescription;
+                                    this.lastVisualTargetApp = browserApp || action.app || this.lastActiveApp;
                                     resultPayload.last_click = {
                                         strategy: domClick.strategy || "safari_dom",
                                         matched_text: domClick.matchedText || null,
@@ -1010,6 +1279,8 @@ export class NativeMacOSJobExecutor {
                             }
                             if (validated) {
                                 const region = ocrClick.region || null;
+                                this.lastVisualTargetDescription = targetDescription;
+                                this.lastVisualTargetApp = browserApp || action.app || this.lastActiveApp;
                                 resultPayload.last_click = {
                                     strategy: ocrClick.strategy || "local_ocr",
                                     score: ocrClick.score || null,
@@ -1085,6 +1356,8 @@ export class NativeMacOSJobExecutor {
                             }
                         }
                         completionNotes.push(`Localizei e cliquei em ${targetDescription}.`);
+                        this.lastVisualTargetDescription = targetDescription;
+                        this.lastVisualTargetApp = browserApp || action.app || this.lastActiveApp;
                         clickSucceeded = true;
                         break;
                     }
@@ -1270,7 +1543,26 @@ end tell
         const beforePlayerState = normalizeText(before?.playerState || "");
         const afterPlayerState = normalizeText(after.playerState || "");
         const playerLooksActive = afterPlayerState.includes("pause") || afterPlayerState.includes("pausar");
+        const playerLooksPaused = !playerLooksActive && /play|tocar|reproduzir|continuar|retomar|resume/.test(afterPlayerState);
+        const wantsNext = descriptionWantsNext(targetDescription);
+        const wantsPrevious = descriptionWantsPrevious(targetDescription);
+        const wantsPause = descriptionWantsPause(targetDescription);
+        const wantsResume = descriptionWantsResume(targetDescription);
+        const mediaQueryTokens = extractMediaQueryTokens(targetDescription);
+        const mediaMatchCount = countMatchingTokens(after.playerTitle || "", mediaQueryTokens);
         if (afterUrl.includes("music.youtube.com")) {
+            if (wantsPause && beforePlayerState && beforePlayerState !== afterPlayerState && playerLooksPaused) {
+                return true;
+            }
+            if (wantsResume && playerLooksActive && beforePlayerState !== afterPlayerState) {
+                return true;
+            }
+            if ((wantsNext || wantsPrevious) && beforePlayerTitle && afterPlayerTitle && beforePlayerTitle !== afterPlayerTitle) {
+                return true;
+            }
+            if (mediaQueryTokens.length >= 2 && mediaMatchCount >= Math.max(2, Math.ceil(mediaQueryTokens.length * 0.5)) && playerLooksActive) {
+                return true;
+            }
             if (beforePlayerState && afterPlayerState && beforePlayerState !== afterPlayerState && playerLooksActive) {
                 return true;
             }
@@ -1323,6 +1615,23 @@ end tell
         if (!key) {
             throw new Error(`Invalid shortcut: ${shortcut}`);
         }
+        const normalizedShortcut = normalizeText(shortcut).replace(/[\s+-]+/g, "_");
+        const mediaCommandMap = {
+            media_play: "play_pause",
+            media_pause: "play_pause",
+            media_play_pause: "play_pause",
+            media_resume: "play_pause",
+            media_next: "next",
+            media_proxima: "next",
+            media_previous: "previous",
+            media_prev: "previous",
+            media_anterior: "previous",
+        };
+        const mediaCommand = mediaCommandMap[normalizedShortcut];
+        if (mediaCommand) {
+            await this.triggerMacOSMediaTransport(mediaCommand);
+            return;
+        }
         const namedKeyCodes = {
             return: 36,
             enter: 36,
@@ -1348,6 +1657,45 @@ end tell
             `tell application "System Events" to keystroke "${escapeAppleScript(key)}"${usingClause}`,
         ]);
     }
+    async triggerMacOSMediaTransport(command) {
+        const keyTypeMap = {
+            play_pause: 16,
+            next: 17,
+            previous: 18,
+        };
+        const keyType = keyTypeMap[command];
+        const swiftScript = `
+import AppKit
+import Foundation
+
+let keyType = ${keyType}
+
+func postMediaKey(_ keyType: Int32, down: Bool) {
+  let eventFlags = NSEvent.ModifierFlags(rawValue: 0xA00)
+  let state = down ? 0xA : 0xB
+  let data1 = Int((keyType << 16) | (Int32(state) << 8))
+  guard let event = NSEvent.otherEvent(
+    with: .systemDefined,
+    location: .zero,
+    modifierFlags: eventFlags,
+    timestamp: 0,
+    windowNumber: 0,
+    context: nil,
+    subtype: 8,
+    data1: data1,
+    data2: -1
+  ) else {
+    return
+  }
+  event.cgEvent?.post(tap: .cghidEventTap)
+}
+
+postMediaKey(Int32(keyType), down: true)
+usleep(90000)
+postMediaKey(Int32(keyType), down: false)
+`;
+        await this.runCommand("swift", ["-e", swiftScript]);
+    }
     async typeText(text) {
         const previousClipboard = await this.readClipboardText();
         try {
@@ -1360,6 +1708,474 @@ end tell
             }
         }
     }
+    resolveLikelySearchShortcut(app) {
+        const normalizedHint = normalizeText(this.lastVisualTargetDescription || "");
+        const looksLikeSearchTarget = /\b(busca|pesquisa|search|campo|caixa|icone|ícone)\b/.test(normalizedHint);
+        if (!looksLikeSearchTarget || !app) {
+            return null;
+        }
+        if (app === "Spotify") {
+            return "cmd+l";
+        }
+        if (app === "Music") {
+            return "cmd+f";
+        }
+        return null;
+    }
+    async guidedTypeText(text, preferredApp) {
+        const app = preferredApp || this.lastActiveApp || await this.getFrontmostAppName();
+        if (app === "Safari") {
+            const safariResult = await this.trySafariGuidedType(text);
+            if (safariResult.ok) {
+                return {
+                    ...safariResult,
+                    app,
+                };
+            }
+        }
+        const searchShortcut = this.resolveLikelySearchShortcut(app);
+        if (searchShortcut) {
+            await this.pressShortcut(searchShortcut).catch(() => undefined);
+            await delay(180);
+        }
+        await this.typeText(text);
+        return {
+            ok: true,
+            verified: false,
+            strategy: searchShortcut ? `clipboard_paste_after_${searchShortcut}` : "clipboard_paste",
+            app: app || null,
+            attempts: 1,
+        };
+    }
+    async trySafariGuidedType(text) {
+        for (let attempt = 0; attempt < 3; attempt += 1) {
+            try {
+                const result = await this.runSafariJsonScript(`
+const inputText = String(__input?.text || "");
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+function isEditable(element) {
+  return element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement || (element instanceof HTMLElement && element.isContentEditable);
+}
+function readEditableValue(element) {
+  if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
+    return String(element.value || "").trim();
+  }
+  if (element instanceof HTMLElement && element.isContentEditable) {
+    return String(element.innerText || element.textContent || "").trim();
+  }
+  return "";
+}
+function clearAndFill(element, value) {
+  if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
+    element.focus();
+    element.select?.();
+    element.value = "";
+    element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "deleteContentBackward", data: null }));
+    element.value = value;
+    element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: value }));
+    element.dispatchEvent(new Event("change", { bubbles: true }));
+    return;
+  }
+  if (element instanceof HTMLElement && element.isContentEditable) {
+    element.focus();
+    const selection = window.getSelection();
+    const range = document.createRange();
+    range.selectNodeContents(element);
+    selection?.removeAllRanges();
+    selection?.addRange(range);
+    document.execCommand("selectAll", false);
+    document.execCommand("delete", false);
+    document.execCommand("insertText", false, value);
+    element.dispatchEvent(new InputEvent("input", { bubbles: true, inputType: "insertText", data: value }));
+  }
+}
+const selectors = location.hostname.includes("music.youtube.com")
+  ? [
+      "ytmusic-search-box input#input",
+      "ytmusic-search-box input",
+      "input[placeholder*='Search']",
+      "input[placeholder*='Pesquisar']",
+      "[role='searchbox'] input"
+    ]
+  : location.hostname.includes("open.spotify.com")
+  ? [
+      "input[data-testid='search-input']",
+      "[role='searchbox'] input",
+      "input[placeholder*='Search']"
+    ]
+  : location.hostname.includes("deezer.com")
+  ? [
+      "input[type='search']",
+      "input[placeholder*='Search']",
+      "input[placeholder*='Pesquisar']",
+      "form input[type='text']"
+    ]
+  : location.hostname.includes("soundcloud.com")
+  ? [
+      "input[type='search']",
+      "input[placeholder*='Search']",
+      "form input[type='search']",
+      "form input[type='text']"
+    ]
+  : location.hostname.includes("music.amazon.com")
+  ? [
+      "input[type='search']",
+      "input[aria-label*='Search']",
+      "input[placeholder*='Search']",
+      "[role='searchbox'] input"
+    ]
+  : [
+      "textarea",
+      "input[type='search']",
+      "input[type='text']",
+      "input:not([type])",
+      "[contenteditable='true'][role='textbox']",
+      "[contenteditable='true']"
+    ];
+const active = document.activeElement;
+let target = null;
+if (active instanceof HTMLElement && isVisible(active) && isEditable(active)) {
+  target = active;
+}
+if (!target) {
+  const candidates = selectors
+    .flatMap((selector) => Array.from(document.querySelectorAll(selector)))
+    .filter((node) => node instanceof HTMLElement)
+    .filter((node) => isVisible(node))
+    .map((node, index) => {
+      const label = String(
+        node.getAttribute("aria-label")
+        || node.getAttribute("placeholder")
+        || node.getAttribute("title")
+        || node.textContent
+        || ""
+      ).toLowerCase();
+      let score = 0;
+      if (label.includes("search") || label.includes("pesquis")) score += 80;
+      if (node === document.activeElement) score += 30;
+      score += Math.max(0, 12 - index);
+      return { node, score };
+    })
+    .sort((left, right) => right.score - left.score);
+  target = candidates[0]?.node || null;
+}
+if (!(target instanceof HTMLElement) || !isEditable(target)) {
+  return { ok: false, reason: "Nao achei um campo editavel confiavel no Safari.", strategy: "safari_guided_type_no_field" };
+}
+target.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+clearAndFill(target, inputText);
+return {
+  ok: true,
+  actualText: readEditableValue(target),
+  strategy: target === active ? "safari_guided_type_active_field" : "safari_guided_type_search_field",
+};
+          `, { text });
+                if (result?.ok && typedTextLooksApplied(result.actualText || "", text)) {
+                    return {
+                        ok: true,
+                        verified: true,
+                        strategy: result.strategy || "safari_guided_type",
+                        attempts: attempt + 1,
+                    };
+                }
+                await delay(180);
+            }
+            catch (error) {
+                const detail = error instanceof Error ? error.message : String(error);
+                if (detail.toLowerCase().includes("allow javascript from apple events")) {
+                    break;
+                }
+            }
+        }
+        await this.typeText(text);
+        await delay(180);
+        try {
+            const verification = await this.runSafariJsonScript(`
+function readEditableValue(element) {
+  if (element instanceof HTMLInputElement || element instanceof HTMLTextAreaElement) {
+    return String(element.value || "").trim();
+  }
+  if (element instanceof HTMLElement && element.isContentEditable) {
+    return String(element.innerText || element.textContent || "").trim();
+  }
+  return "";
+}
+const active = document.activeElement;
+return { actualText: active ? readEditableValue(active) : "" };
+        `, {});
+            if (typedTextLooksApplied(verification.actualText || "", text)) {
+                return {
+                    ok: true,
+                    verified: true,
+                    strategy: "safari_clipboard_retry",
+                    attempts: 4,
+                };
+            }
+        }
+        catch {
+            // ignore and fall back to unverified success below
+        }
+        return {
+            ok: true,
+            verified: false,
+            strategy: "clipboard_paste",
+            attempts: 1,
+        };
+    }
+    async scrollView(direction, amount = "medium", steps = 1) {
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const lineDelta = {
+            small: 4,
+            medium: 8,
+            large: 14,
+        }[amount];
+        const signedDelta = direction === "up" ? lineDelta : -lineDelta;
+        const iterations = {
+            small: 1,
+            medium: 2,
+            large: 3,
+        }[amount];
+        const swiftScript = `
+import ApplicationServices
+import Foundation
+
+let wheelDelta: Int32 = ${signedDelta}
+let stepCount = ${clampedSteps}
+let iterations = ${iterations}
+
+for _ in 0..<stepCount {
+  for _ in 0..<iterations {
+    if let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: wheelDelta, wheel2: 0, wheel3: 0) {
+      event.post(tap: .cghidEventTap)
+    }
+    usleep(35000)
+  }
+  usleep(85000)
+}
+`;
+        try {
+            await this.runCommand("swift", ["-e", swiftScript]);
+        }
+        catch {
+            await this.scrollViewWithPageKeys(direction, clampedSteps);
+        }
+    }
+    async scrollViewWithPageKeys(direction, steps) {
+        const keyCode = direction === "up" ? 116 : 121;
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const script = `
+repeat ${clampedSteps} times
+  tell application "System Events" to key code ${keyCode}
+  delay 0.06
+end repeat
+`;
+        await this.runCommand("osascript", ["-e", script]);
+    }
+    async ensureWhatsAppWebReady() {
+        const page = await this.readFrontmostPage("Safari");
+        if (!normalizeComparableUrl(page.url || "").includes("web.whatsapp.com")) {
+            throw new Error("O Safari nao esta aberto no WhatsApp Web.");
+        }
+    }
+    async selectWhatsAppConversation(contact) {
+        const prepared = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+function focusAndReplaceContent(element, value) {
+  element.focus();
+  const range = document.createRange();
+  range.selectNodeContents(element);
+  const selection = window.getSelection();
+  selection?.removeAllRanges();
+  selection?.addRange(range);
+  document.execCommand("selectAll", false);
+  document.execCommand("delete", false);
+  document.execCommand("insertText", false, value);
+  element.dispatchEvent(new InputEvent("input", { bubbles: true, data: value, inputType: "insertText" }));
+}
+
+const candidates = Array.from(document.querySelectorAll('div[contenteditable="true"][role="textbox"], div[contenteditable="true"][data-tab], [data-testid="chat-list-search"] [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const element = node;
+    const rect = element.getBoundingClientRect();
+    const label = normalize(element.getAttribute("aria-label") || element.getAttribute("data-testid") || element.textContent || "");
+    let score = 0;
+    if (rect.left < window.innerWidth * 0.45) score += 30;
+    if (rect.top < 240) score += 30;
+    if (label.includes("search") || label.includes("pesquisar") || label.includes("procure") || label.includes("chat list")) score += 80;
+    if (element.closest('[data-testid="chat-list-search"], header')) score += 25;
+    return { element, score };
+  })
+  .sort((left, right) => right.score - left.score);
+
+if (!candidates.length) {
+  return { ok: false, reason: "Nao achei o campo de busca do WhatsApp Web." };
+}
+
+focusAndReplaceContent(candidates[0].element, query);
+return { ok: true };
+      `, { contact });
+        if (!prepared?.ok) {
+            return false;
+        }
+        await delay(900);
+        const result = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const titleNodes = Array.from(document.querySelectorAll('span[title], div[title]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const text = normalize(node.getAttribute("title") || node.textContent || "");
+    let score = 0;
+    if (text === normalizedQuery) score += 160;
+    if (text.includes(normalizedQuery)) score += 100;
+    if (normalizedQuery.includes(text) && text.length >= 3) score += 50;
+    const container = node.closest('[data-testid="cell-frame-container"], [role="listitem"], [role="gridcell"], div[tabindex]');
+    if (container instanceof HTMLElement && isVisible(container)) score += 20;
+    return { node, container, text, score };
+  })
+  .filter((item) => item.score > 0)
+  .sort((left, right) => right.score - left.score);
+
+if (!titleNodes.length) {
+  return { clicked: false, reason: "Nao achei uma conversa visivel com esse nome." };
+}
+
+const winner = titleNodes[0];
+const target = winner.container instanceof HTMLElement ? winner.container : winner.node;
+target.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+target.dispatchEvent(new MouseEvent("mousedown", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("mouseup", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true, view: window }));
+if (typeof target.click === "function") {
+  target.click();
+}
+return { clicked: true };
+      `, { contact });
+        return Boolean(result?.clicked);
+    }
+    async focusWhatsAppComposer() {
+        const result = await this.runSafariJsonScript(`
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const candidates = Array.from(document.querySelectorAll('footer div[contenteditable="true"], [data-testid="conversation-compose-box-input"], main footer [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .sort((left, right) => right.getBoundingClientRect().top - left.getBoundingClientRect().top);
+
+if (!candidates.length) {
+  return { focused: false, reason: "Nao achei o campo de mensagem do WhatsApp Web." };
+}
+
+const composer = candidates[0];
+composer.focus();
+composer.click();
+return { focused: true };
+      `);
+        if (!result?.focused) {
+            throw new Error(result?.reason || "Nao consegui focar o campo de mensagem do WhatsApp Web.");
+        }
+    }
+    async readWhatsAppVisibleConversation(contact, limit) {
+        const result = await this.runSafariJsonScript(`
+const maxMessages = Number(__input?.limit || 12);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const containers = Array.from(document.querySelectorAll('[data-testid="msg-container"], div[data-id]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node));
+
+const messages = containers.map((node) => {
+  const element = node;
+  const prePlain = element.querySelector('[data-pre-plain-text]')?.getAttribute('data-pre-plain-text') || "";
+  const authorMatch = prePlain.match(/\\]\\s*([^:]+):/);
+  const author = authorMatch?.[1]?.trim() || (element.getAttribute('data-testid')?.includes('out') ? 'Voce' : 'Contato');
+  const text = (element.innerText || "").trim().replace(/\\n{2,}/g, "\\n");
+  return { author, text };
+}).filter((item) => item.text);
+
+return { messages: messages.slice(-maxMessages) };
+      `, { contact, limit });
+        const messages = Array.isArray(result?.messages)
+            ? result.messages
+                .map((item) => ({
+                author: clipText(asString(item.author) || "Contato", 80),
+                text: clipText(asString(item.text) || "", 500),
+            }))
+                .filter((item) => item.text)
+            : [];
+        return {
+            messages,
+            summary: messages.length
+                ? messages.map((item) => `${item.author}: ${item.text}`).join("\n")
+                : `(sem mensagens visiveis na conversa com ${contact})`,
+        };
+    }
+    async verifyWhatsAppLastMessage(expectedText) {
+        const chat = await this.readWhatsAppVisibleConversation("Contato", 6);
+        if (!chat.messages.length) {
+            return {
+                ok: false,
+                reason: "Nao consegui ler as mensagens visiveis apos o envio no WhatsApp.",
+            };
+        }
+        const normalizedExpected = normalizeText(expectedText).slice(0, 60);
+        const matched = chat.messages.some((item) => normalizeText(item.text).includes(normalizedExpected));
+        if (!matched) {
+            return {
+                ok: false,
+                reason: "Nao consegui confirmar visualmente a mensagem enviada no WhatsApp.",
+            };
+        }
+        return { ok: true, reason: "" };
+    }
     async takeScreenshot(targetPath) {
         const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
         await mkdir(artifactsDir, { recursive: true });
@@ -1485,11 +2301,17 @@ const normalizedDescription = normalize(rawDescription);
 const isYouTubeMusic = location.hostname.includes("music.youtube.com");
 const wantsFirst = /\\b(primeir[ao]?|first)\\b/.test(normalizedDescription);
 const wantsVideo = /\\b(video|videos|musica|faixa|youtube|resultado|watch)\\b/.test(normalizedDescription) || location.hostname.includes("youtube");
+const wantsNext = /\\b(proxim[ao]?|next|skip|pular|avanca|avancar)\\b/.test(normalizedDescription);
+const wantsPrevious = /\\b(anterior|previous|volta[ar]?|back|retorna[ar]?)\\b/.test(normalizedDescription);
+const wantsPause = /\\b(pausa|pause|pausar)\\b/.test(normalizedDescription);
+const wantsResume = /\\b(retoma|retomar|resume|continu[ae]r|despausa|play)\\b/.test(normalizedDescription);
 const stopWords = new Set([
   "o", "a", "os", "as", "um", "uma", "uns", "umas", "de", "da", "do", "das", "dos",
   "em", "no", "na", "nos", "nas", "para", "por", "com", "que", "visivel", "visiveis",
   "visivel", "tela", "pagina", "page", "site", "link", "botao", "botao", "clicar",
-  "clique", "seleciona", "selecionar", "resultado", "resultados"
+  "clique", "seleciona", "selecionar", "resultado", "resultados", "youtube", "music",
+  "melhor", "tocavel", "relacionado", "diretamente", "navegador", "player", "fila",
+  "play", "pause", "pausa", "proxima", "proximo", "anterior"
 ]);
 const quotedPhrases = Array.from(rawDescription.matchAll(/["'“”‘’]([^"'“”‘’]{2,80})["'“”‘’]/g))
   .map((match) => normalize(match[1]));
@@ -1564,6 +2386,173 @@ function deriveText(element) {
   return "";
 }
 
+function clickElement(element, strategy, matchedText, matchedHref, score, totalCandidates) {
+  if (!(element instanceof HTMLElement || element instanceof HTMLAnchorElement)) {
+    return null;
+  }
+  element.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+  const rect = element.getBoundingClientRect();
+  for (const eventName of ["mouseover", "mousedown", "mouseup", "click"]) {
+    element.dispatchEvent(new MouseEvent(eventName, {
+      bubbles: true,
+      cancelable: true,
+      view: window,
+      clientX: rect.left + (rect.width / 2),
+      clientY: rect.top + (rect.height / 2),
+    }));
+  }
+  if (typeof element.click === "function") {
+    element.click();
+  }
+  return {
+    clicked: true,
+    matchedText: String(matchedText || "").slice(0, 180),
+    matchedHref: String(matchedHref || ""),
+    score,
+    totalCandidates,
+    strategy,
+  };
+}
+
+function attemptYouTubeMusicTransportClick() {
+  if (!isYouTubeMusic || !(wantsNext || wantsPrevious || wantsPause || wantsResume)) {
+    return null;
+  }
+
+  const playerButtons = Array.from(document.querySelectorAll(
+    "ytmusic-player-bar button, ytmusic-player-bar [role='button'], ytmusic-player-bar tp-yt-paper-icon-button"
+  ))
+    .filter((node) => node instanceof HTMLElement)
+    .filter((node) => isVisible(node));
+
+  const ranked = playerButtons
+    .map((node, index) => {
+      const label = normalize([
+        deriveText(node),
+        node.getAttribute("aria-label"),
+        node.getAttribute("title"),
+        node.id,
+      ].filter(Boolean).join(" "));
+      let score = 0;
+      if (wantsNext && /proxim|next|skip/.test(label)) score += 140;
+      if (wantsPrevious && /anterior|previous|back|volta/.test(label)) score += 140;
+      if (wantsPause && /pause|pausa|pausar/.test(label)) score += 140;
+      if (wantsResume && /play|tocar|reproduzir|resume|retomar|continuar/.test(label)) score += 140;
+      if (label.includes("player")) score += 12;
+      score += Math.max(0, 12 - index);
+      return score > 0 ? { node, label, score } : null;
+    })
+    .filter(Boolean)
+    .sort((left, right) => right.score - left.score);
+
+  if (!ranked.length) {
+    return null;
+  }
+
+  const winner = ranked[0];
+  return clickElement(winner.node, "safari_dom_ytmusic_transport", winner.label, "", winner.score, ranked.length);
+}
+
+function attemptYouTubeMusicSearchResultClick() {
+  if (!isYouTubeMusic || wantsNext || wantsPrevious || wantsPause) {
+    return null;
+  }
+  if (!quotedPhrases.length && !tokens.length) {
+    return null;
+  }
+
+  const rows = Array.from(document.querySelectorAll("ytmusic-responsive-list-item-renderer"))
+    .filter((node) => node instanceof HTMLElement)
+    .filter((node) => isVisible(node));
+
+  const rankedRows = rows
+    .map((row, index) => {
+      const titleNode = row.querySelector("#title, .title, yt-formatted-string.title");
+      const subtitleNode = row.querySelector(".subtitle, .byline, .secondary-flex-columns");
+      const titleText = String((titleNode && titleNode.textContent) || "").trim();
+      const subtitleText = String((subtitleNode && subtitleNode.textContent) || "").trim();
+      const rowText = deriveText(row);
+      const normalizedTitle = normalize(titleText);
+      const normalizedSubtitle = normalize(subtitleText);
+      const normalizedRow = normalize(rowText);
+      let score = 0;
+
+      for (const phrase of quotedPhrases) {
+        if (!phrase) continue;
+        if (normalizedTitle.includes(phrase)) score += 160;
+        else if (normalizedRow.includes(phrase)) score += 110;
+      }
+
+      for (const token of tokens) {
+        if (normalizedTitle.includes(token)) score += 28;
+        else if (normalizedSubtitle.includes(token)) score += 16;
+        else if (normalizedRow.includes(token)) score += 10;
+      }
+
+      if (tokens.length > 1) {
+        const titleMatches = tokens.filter((token) => normalizedTitle.includes(token)).length;
+        const rowMatches = tokens.filter((token) => normalizedRow.includes(token)).length;
+        if (titleMatches >= Math.max(2, Math.ceil(tokens.length * 0.5))) score += 80;
+        if (rowMatches === tokens.length) score += 40;
+      }
+
+      score += Math.max(0, 10 - index);
+
+      const clickableCandidates = Array.from(row.querySelectorAll(
+        "ytmusic-item-thumbnail-overlay-renderer button, button[aria-label], tp-yt-paper-icon-button, a[href*='watch?v=']"
+      ))
+        .filter((candidate) => candidate instanceof HTMLElement || candidate instanceof HTMLAnchorElement)
+        .filter((candidate) => isVisible(candidate))
+        .map((candidate) => {
+          const label = normalize([
+            deriveText(candidate),
+            candidate.getAttribute("aria-label"),
+            candidate.getAttribute("title"),
+          ].filter(Boolean).join(" "));
+          let candidateScore = 0;
+          if (/play|tocar|reproduzir|assistir/.test(label)) candidateScore += 30;
+          if (candidate instanceof HTMLAnchorElement && normalize(candidate.href).includes("watch?v=")) candidateScore += 18;
+          if (candidate.closest("ytmusic-item-thumbnail-overlay-renderer")) candidateScore += 14;
+          return { candidate, label, candidateScore };
+        })
+        .sort((left, right) => right.candidateScore - left.candidateScore);
+
+      return score > 0 ? {
+        row,
+        titleText,
+        href: clickableCandidates[0]?.candidate instanceof HTMLAnchorElement ? clickableCandidates[0].candidate.href : "",
+        score: score + (clickableCandidates[0]?.candidateScore || 0),
+        target: clickableCandidates[0]?.candidate || row,
+      } : null;
+    })
+    .filter(Boolean)
+    .sort((left, right) => right.score - left.score);
+
+  if (!rankedRows.length) {
+    return null;
+  }
+
+  const winner = rankedRows[0];
+  return clickElement(
+    winner.target,
+    "safari_dom_ytmusic_result",
+    winner.titleText || deriveText(winner.row),
+    winner.href || "",
+    winner.score,
+    rankedRows.length,
+  );
+}
+
+const ytmTransport = attemptYouTubeMusicTransportClick();
+if (ytmTransport) {
+  return ytmTransport;
+}
+
+const ytmResult = attemptYouTubeMusicSearchResultClick();
+if (ytmResult) {
+  return ytmResult;
+}
+
 function scoreCandidate(element, rank) {
   const text = deriveText(element);
   const href = element instanceof HTMLAnchorElement
@@ -2214,6 +3203,15 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "set_volume") {
             return `Volume ajustado para ${action.level}% no macOS`;
         }
+        if (action.type === "scroll_view") {
+            return `Tela rolada para ${action.direction === "up" ? "cima" : "baixo"} no macOS`;
+        }
+        if (action.type === "whatsapp_send_message") {
+            return `Mensagem enviada no WhatsApp para ${action.contact}`;
+        }
+        if (action.type === "whatsapp_read_chat") {
+            return `Conversa do WhatsApp lida com ${action.contact}`;
+        }
         if (action.type === "click_visual_target") {
             return `Clique guiado executado para ${action.description}`;
         }

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.6";
+export const BRIDGE_VERSION = "0.5.9";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.6",
+  "version": "0.5.9",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",