@leg3ndy/otto-bridge 0.5.6 → 0.5.8

package/dist/executors/native_macos.js

@@ -665,6 +665,44 @@ function parseStructuredActions(job) {
             }
             continue;
         }
+        if (type === "scroll_view" || type === "scroll" || type === "scroll_page") {
+            const rawDirection = (asString(action.direction) || asString(action.dir) || "down").toLowerCase();
+            const direction = rawDirection === "up" || rawDirection === "cima" ? "up" : "down";
+            const rawAmount = (asString(action.amount) || "").toLowerCase();
+            const amount = rawAmount === "small" || rawAmount === "medium" || rawAmount === "large"
+                ? rawAmount
+                : undefined;
+            const rawSteps = Number(action.steps);
+            const steps = Number.isFinite(rawSteps) ? Math.max(1, Math.min(Math.round(rawSteps), 6)) : undefined;
+            actions.push({
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: asString(action.app) || undefined,
+            });
+            continue;
+        }
+        if (type === "whatsapp_send_message") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const text = asString(action.text) || asString(action.message);
+            if (contact && text) {
+                actions.push({ type: "whatsapp_send_message", contact, text });
+            }
+            continue;
+        }
+        if (type === "whatsapp_read_chat") {
+            const contact = asString(action.contact) || asString(action.recipient);
+            const rawLimit = Number(action.limit);
+            if (contact) {
+                actions.push({
+                    type: "whatsapp_read_chat",
+                    contact,
+                    limit: Number.isFinite(rawLimit) ? Math.max(1, Math.min(Math.round(rawLimit), 30)) : undefined,
+                });
+            }
+            continue;
+        }
         if (type === "click_visual_target" || type === "click_target") {
             const description = asString(action.description) || asString(action.target);
             if (description) {
@@ -744,6 +782,23 @@ function deriveActionsFromText(job) {
         }
         return [{ type: "set_volume", level }];
     }
+    if (/\b(scroll|rola\w*|role\w*|desce\w*|sobe\w*)\b/i.test(task)) {
+        const direction = /\b(sobe\w*|suba\w*|para cima|pra cima|scroll up)\b/i.test(task) ? "up" : "down";
+        const amount = /\b(pouco|leve|small)\b/i.test(task)
+            ? "small"
+            : /\b(muito|bastante|fim|grande|large)\b/i.test(task)
+                ? "large"
+                : "medium";
+        const stepsMatch = task.match(/\b(\d{1,2})\s*(?:x|vezes?)\b/i);
+        const steps = stepsMatch?.[1] ? Math.max(1, Math.min(Number(stepsMatch[1]), 6)) : 1;
+        return [{
+                type: "scroll_view",
+                direction,
+                amount,
+                steps,
+                app: detectedApp || undefined,
+            }];
+    }
     if ((normalizedTask.includes("leia") || normalizedTask.includes("ler")) && detectedUrl) {
         return [
             { type: "open_url", url: detectedUrl, app: detectedApp || "Safari" },
@@ -934,6 +989,68 @@ export class NativeMacOSJobExecutor {
                     completionNotes.push(`Volume ajustado para ${action.level}% no macOS.`);
                     continue;
                 }
+                if (action.type === "scroll_view") {
+                    const scrollApp = action.app || this.lastActiveApp || await this.getFrontmostAppName();
+                    if (scrollApp) {
+                        await reporter.progress(progressPercent, `Trazendo ${scrollApp} para frente antes de rolar a tela`);
+                        await this.focusApp(scrollApp);
+                    }
+                    const directionLabel = action.direction === "up" ? "cima" : "baixo";
+                    await reporter.progress(progressPercent, `Rolando a tela para ${directionLabel}`);
+                    await this.scrollView(action.direction, action.amount, action.steps);
+                    resultPayload.last_scroll = {
+                        direction: action.direction,
+                        amount: action.amount || "medium",
+                        steps: action.steps || 1,
+                        app: scrollApp || null,
+                    };
+                    completionNotes.push(`Rolei a tela para ${directionLabel} no macOS.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_send_message") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await reporter.progress(progressPercent, `Digitando a mensagem para ${action.contact} no WhatsApp`);
+                    await this.focusWhatsAppComposer();
+                    await this.typeText(action.text);
+                    await delay(250);
+                    await this.pressShortcut("return");
+                    await delay(900);
+                    const verification = await this.verifyWhatsAppLastMessage(action.text);
+                    if (!verification.ok) {
+                        throw new Error(verification.reason || `Nao consegui confirmar o envio da mensagem para ${action.contact} no WhatsApp.`);
+                    }
+                    resultPayload.whatsapp = {
+                        action: "send_message",
+                        contact: action.contact,
+                        text_preview: clipText(action.text, 180),
+                    };
+                    completionNotes.push(`Enviei a mensagem no WhatsApp para ${action.contact}.`);
+                    continue;
+                }
+                if (action.type === "whatsapp_read_chat") {
+                    await reporter.progress(progressPercent, `Abrindo a conversa do WhatsApp com ${action.contact}`);
+                    await this.focusApp("Safari");
+                    await this.ensureWhatsAppWebReady();
+                    const selected = await this.selectWhatsAppConversation(action.contact);
+                    if (!selected) {
+                        throw new Error(`Nao consegui localizar a conversa do WhatsApp com ${action.contact}.`);
+                    }
+                    await delay(500);
+                    const chat = await this.readWhatsAppVisibleConversation(action.contact, action.limit || 12);
+                    resultPayload.whatsapp = {
+                        action: "read_chat",
+                        contact: action.contact,
+                        messages: chat.messages,
+                    };
+                    completionNotes.push(`Mensagens visiveis no WhatsApp com ${action.contact}:\n${chat.summary}`);
+                    continue;
+                }
                 if (action.type === "click_visual_target") {
                     const browserApp = await this.resolveLikelyBrowserApp(action.app);
                     if (browserApp) {
@@ -1360,6 +1477,253 @@ end tell
             }
         }
     }
+    async scrollView(direction, amount = "medium", steps = 1) {
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const lineDelta = {
+            small: 4,
+            medium: 8,
+            large: 14,
+        }[amount];
+        const signedDelta = direction === "up" ? lineDelta : -lineDelta;
+        const iterations = {
+            small: 1,
+            medium: 2,
+            large: 3,
+        }[amount];
+        const swiftScript = `
+import ApplicationServices
+import Foundation
+
+let wheelDelta: Int32 = ${signedDelta}
+let stepCount = ${clampedSteps}
+let iterations = ${iterations}
+
+for _ in 0..<stepCount {
+  for _ in 0..<iterations {
+    if let event = CGEvent(scrollWheelEvent2Source: nil, units: .line, wheelCount: 1, wheel1: wheelDelta, wheel2: 0, wheel3: 0) {
+      event.post(tap: .cghidEventTap)
+    }
+    usleep(35000)
+  }
+  usleep(85000)
+}
+`;
+        try {
+            await this.runCommand("swift", ["-e", swiftScript]);
+        }
+        catch {
+            await this.scrollViewWithPageKeys(direction, clampedSteps);
+        }
+    }
+    async scrollViewWithPageKeys(direction, steps) {
+        const keyCode = direction === "up" ? 116 : 121;
+        const clampedSteps = Math.max(1, Math.min(Math.round(steps || 1), 6));
+        const script = `
+repeat ${clampedSteps} times
+  tell application "System Events" to key code ${keyCode}
+  delay 0.06
+end repeat
+`;
+        await this.runCommand("osascript", ["-e", script]);
+    }
+    async ensureWhatsAppWebReady() {
+        const page = await this.readFrontmostPage("Safari");
+        if (!normalizeComparableUrl(page.url || "").includes("web.whatsapp.com")) {
+            throw new Error("O Safari nao esta aberto no WhatsApp Web.");
+        }
+    }
+    async selectWhatsAppConversation(contact) {
+        const prepared = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 4 || rect.height < 4) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+function focusAndReplaceContent(element, value) {
+  element.focus();
+  const range = document.createRange();
+  range.selectNodeContents(element);
+  const selection = window.getSelection();
+  selection?.removeAllRanges();
+  selection?.addRange(range);
+  document.execCommand("selectAll", false);
+  document.execCommand("delete", false);
+  document.execCommand("insertText", false, value);
+  element.dispatchEvent(new InputEvent("input", { bubbles: true, data: value, inputType: "insertText" }));
+}
+
+const candidates = Array.from(document.querySelectorAll('div[contenteditable="true"][role="textbox"], div[contenteditable="true"][data-tab], [data-testid="chat-list-search"] [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const element = node;
+    const rect = element.getBoundingClientRect();
+    const label = normalize(element.getAttribute("aria-label") || element.getAttribute("data-testid") || element.textContent || "");
+    let score = 0;
+    if (rect.left < window.innerWidth * 0.45) score += 30;
+    if (rect.top < 240) score += 30;
+    if (label.includes("search") || label.includes("pesquisar") || label.includes("procure") || label.includes("chat list")) score += 80;
+    if (element.closest('[data-testid="chat-list-search"], header')) score += 25;
+    return { element, score };
+  })
+  .sort((left, right) => right.score - left.score);
+
+if (!candidates.length) {
+  return { ok: false, reason: "Nao achei o campo de busca do WhatsApp Web." };
+}
+
+focusAndReplaceContent(candidates[0].element, query);
+return { ok: true };
+      `, { contact });
+        if (!prepared?.ok) {
+            return false;
+        }
+        await delay(900);
+        const result = await this.runSafariJsonScript(`
+const query = String(__input?.contact || "");
+const normalize = (value) => String(value || "").normalize("NFD").replace(/[\\u0300-\\u036f]/g, "").toLowerCase().trim();
+const normalizedQuery = normalize(query);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const titleNodes = Array.from(document.querySelectorAll('span[title], div[title]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .map((node) => {
+    const text = normalize(node.getAttribute("title") || node.textContent || "");
+    let score = 0;
+    if (text === normalizedQuery) score += 160;
+    if (text.includes(normalizedQuery)) score += 100;
+    if (normalizedQuery.includes(text) && text.length >= 3) score += 50;
+    const container = node.closest('[data-testid="cell-frame-container"], [role="listitem"], [role="gridcell"], div[tabindex]');
+    if (container instanceof HTMLElement && isVisible(container)) score += 20;
+    return { node, container, text, score };
+  })
+  .filter((item) => item.score > 0)
+  .sort((left, right) => right.score - left.score);
+
+if (!titleNodes.length) {
+  return { clicked: false, reason: "Nao achei uma conversa visivel com esse nome." };
+}
+
+const winner = titleNodes[0];
+const target = winner.container instanceof HTMLElement ? winner.container : winner.node;
+target.scrollIntoView({ block: "center", inline: "center", behavior: "auto" });
+target.dispatchEvent(new MouseEvent("mousedown", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("mouseup", { bubbles: true, cancelable: true, view: window }));
+target.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true, view: window }));
+if (typeof target.click === "function") {
+  target.click();
+}
+return { clicked: true };
+      `, { contact });
+        return Boolean(result?.clicked);
+    }
+    async focusWhatsAppComposer() {
+        const result = await this.runSafariJsonScript(`
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const candidates = Array.from(document.querySelectorAll('footer div[contenteditable="true"], [data-testid="conversation-compose-box-input"], main footer [contenteditable="true"]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node))
+  .sort((left, right) => right.getBoundingClientRect().top - left.getBoundingClientRect().top);
+
+if (!candidates.length) {
+  return { focused: false, reason: "Nao achei o campo de mensagem do WhatsApp Web." };
+}
+
+const composer = candidates[0];
+composer.focus();
+composer.click();
+return { focused: true };
+      `);
+        if (!result?.focused) {
+            throw new Error(result?.reason || "Nao consegui focar o campo de mensagem do WhatsApp Web.");
+        }
+    }
+    async readWhatsAppVisibleConversation(contact, limit) {
+        const result = await this.runSafariJsonScript(`
+const maxMessages = Number(__input?.limit || 12);
+
+function isVisible(element) {
+  if (!(element instanceof HTMLElement)) return false;
+  const rect = element.getBoundingClientRect();
+  if (rect.width < 6 || rect.height < 6) return false;
+  const style = window.getComputedStyle(element);
+  if (style.visibility === "hidden" || style.display === "none" || Number(style.opacity || "1") === 0) return false;
+  return rect.bottom >= 0 && rect.right >= 0 && rect.top <= window.innerHeight && rect.left <= window.innerWidth;
+}
+
+const containers = Array.from(document.querySelectorAll('[data-testid="msg-container"], div[data-id]'))
+  .filter((node) => node instanceof HTMLElement)
+  .filter((node) => isVisible(node));
+
+const messages = containers.map((node) => {
+  const element = node;
+  const prePlain = element.querySelector('[data-pre-plain-text]')?.getAttribute('data-pre-plain-text') || "";
+  const authorMatch = prePlain.match(/\\]\\s*([^:]+):/);
+  const author = authorMatch?.[1]?.trim() || (element.getAttribute('data-testid')?.includes('out') ? 'Voce' : 'Contato');
+  const text = (element.innerText || "").trim().replace(/\\n{2,}/g, "\\n");
+  return { author, text };
+}).filter((item) => item.text);
+
+return { messages: messages.slice(-maxMessages) };
+      `, { contact, limit });
+        const messages = Array.isArray(result?.messages)
+            ? result.messages
+                .map((item) => ({
+                author: clipText(asString(item.author) || "Contato", 80),
+                text: clipText(asString(item.text) || "", 500),
+            }))
+                .filter((item) => item.text)
+            : [];
+        return {
+            messages,
+            summary: messages.length
+                ? messages.map((item) => `${item.author}: ${item.text}`).join("\n")
+                : `(sem mensagens visiveis na conversa com ${contact})`,
+        };
+    }
+    async verifyWhatsAppLastMessage(expectedText) {
+        const chat = await this.readWhatsAppVisibleConversation("Contato", 6);
+        if (!chat.messages.length) {
+            return {
+                ok: false,
+                reason: "Nao consegui ler as mensagens visiveis apos o envio no WhatsApp.",
+            };
+        }
+        const normalizedExpected = normalizeText(expectedText).slice(0, 60);
+        const matched = chat.messages.some((item) => normalizeText(item.text).includes(normalizedExpected));
+        if (!matched) {
+            return {
+                ok: false,
+                reason: "Nao consegui confirmar visualmente a mensagem enviada no WhatsApp.",
+            };
+        }
+        return { ok: true, reason: "" };
+    }
     async takeScreenshot(targetPath) {
         const artifactsDir = path.join(os.homedir(), ".otto-bridge", "artifacts");
         await mkdir(artifactsDir, { recursive: true });
@@ -2214,6 +2578,15 @@ if let output = String(data: data, encoding: .utf8) {
         if (action.type === "set_volume") {
             return `Volume ajustado para ${action.level}% no macOS`;
         }
+        if (action.type === "scroll_view") {
+            return `Tela rolada para ${action.direction === "up" ? "cima" : "baixo"} no macOS`;
+        }
+        if (action.type === "whatsapp_send_message") {
+            return `Mensagem enviada no WhatsApp para ${action.contact}`;
+        }
+        if (action.type === "whatsapp_read_chat") {
+            return `Conversa do WhatsApp lida com ${action.contact}`;
+        }
         if (action.type === "click_visual_target") {
             return `Clique guiado executado para ${action.description}`;
         }

package/dist/types.js

@@ -1,5 +1,5 @@
 export const BRIDGE_CONFIG_VERSION = 1;
-export const BRIDGE_VERSION = "0.5.6";
+export const BRIDGE_VERSION = "0.5.8";
 export const BRIDGE_PACKAGE_NAME = "@leg3ndy/otto-bridge";
 export const DEFAULT_API_BASE_URL = "http://localhost:8000";
 export const DEFAULT_POLL_INTERVAL_MS = 3000;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@leg3ndy/otto-bridge",
-  "version": "0.5.6",
+  "version": "0.5.8",
   "private": false,
   "type": "module",
   "description": "Local companion for Otto Bridge device pairing and WebSocket runtime.",